git.meshlink.io Git - meshlink/blob - src/utcp.c

   1 /*
   2     utcp.c -- Userspace TCP
   3     Copyright (C) 2014-2017 Guus Sliepen <guus@tinc-vpn.org>
   4
   5     This program is free software; you can redistribute it and/or modify
   6     it under the terms of the GNU General Public License as published by
   7     the Free Software Foundation; either version 2 of the License, or
   8     (at your option) any later version.
   9
  10     This program is distributed in the hope that it will be useful,
  11     but WITHOUT ANY WARRANTY; without even the implied warranty of
  12     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13     GNU General Public License for more details.
  14
  15     You should have received a copy of the GNU General Public License along
  16     with this program; if not, write to the Free Software Foundation, Inc.,
  17     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "system.h"
  21 #include <time.h>
  22
  23 #include "utcp_priv.h"
  24
  25 #ifndef EBADMSG
  26 #define EBADMSG         104
  27 #endif
  28
  29 #ifndef SHUT_RDWR
  30 #define SHUT_RDWR 2
  31 #endif
  32
  33 #ifdef poll
  34 #undef poll
  35 #endif
  36
  37 #ifndef UTCP_CLOCK
  38 #if defined(CLOCK_MONOTONIC_RAW) && defined(__x86_64__)
  39 #define UTCP_CLOCK CLOCK_MONOTONIC_RAW
  40 #else
  41 #define UTCP_CLOCK CLOCK_MONOTONIC
  42 #endif
  43 #endif
  44
  45 static void timespec_sub(const struct timespec *a, const struct timespec *b, struct timespec *r) {
  46         r->tv_sec = a->tv_sec - b->tv_sec;
  47         r->tv_nsec = a->tv_nsec - b->tv_nsec;
  48
  49         if(r->tv_nsec < 0) {
  50                 r->tv_sec--, r->tv_nsec += NSEC_PER_SEC;
  51         }
  52 }
  53
  54 static int32_t timespec_diff_usec(const struct timespec *a, const struct timespec *b) {
  55         return (a->tv_sec - b->tv_sec) * 1000000 + (a->tv_nsec - b->tv_nsec) / 1000;
  56 }
  57
  58 static bool timespec_lt(const struct timespec *a, const struct timespec *b) {
  59         if(a->tv_sec == b->tv_sec) {
  60                 return a->tv_nsec < b->tv_nsec;
  61         } else {
  62                 return a->tv_sec < b->tv_sec;
  63         }
  64 }
  65
  66 static void timespec_clear(struct timespec *a) {
  67         a->tv_sec = 0;
  68         a->tv_nsec = 0;
  69 }
  70
  71 static bool timespec_isset(const struct timespec *a) {
  72         return a->tv_sec;
  73 }
  74
  75 static long CLOCK_GRANULARITY; // usec
  76
  77 static inline size_t min(size_t a, size_t b) {
  78         return a < b ? a : b;
  79 }
  80
  81 static inline size_t max(size_t a, size_t b) {
  82         return a > b ? a : b;
  83 }
  84
  85 #ifdef UTCP_DEBUG
  86 #include <stdarg.h>
  87
  88 #ifndef UTCP_DEBUG_DATALEN
  89 #define UTCP_DEBUG_DATALEN 20
  90 #endif
  91
  92 static void debug(struct utcp_connection *c, const char *format, ...) {
  93         struct timespec tv;
  94         char buf[1024];
  95         int len;
  96
  97         clock_gettime(CLOCK_REALTIME, &tv);
  98         len = snprintf(buf, sizeof(buf), "%ld.%06lu %u:%u ", (long)tv.tv_sec, tv.tv_nsec / 1000, c ? c->src : 0, c ? c->dst : 0);
  99         va_list ap;
 100         va_start(ap, format);
 101         len += vsnprintf(buf + len, sizeof(buf) - len, format, ap);
 102         va_end(ap);
 103
 104         if(len > 0 && (size_t)len < sizeof(buf)) {
 105                 fwrite(buf, len, 1, stderr);
 106         }
 107 }
 108
 109 static void print_packet(struct utcp_connection *c, const char *dir, const void *pkt, size_t len) {
 110         struct hdr hdr;
 111
 112         if(len < sizeof(hdr)) {
 113                 debug(c, "%s: short packet (%lu bytes)\n", dir, (unsigned long)len);
 114                 return;
 115         }
 116
 117         memcpy(&hdr, pkt, sizeof(hdr));
 118
 119         uint32_t datalen;
 120
 121         if(len > sizeof(hdr)) {
 122                 datalen = min(len - sizeof(hdr), UTCP_DEBUG_DATALEN);
 123         } else {
 124                 datalen = 0;
 125         }
 126
 127
 128         const uint8_t *data = (uint8_t *)pkt + sizeof(hdr);
 129         char str[datalen * 2 + 1];
 130         char *p = str;
 131
 132         for(uint32_t i = 0; i < datalen; i++) {
 133                 *p++ = "0123456789ABCDEF"[data[i] >> 4];
 134                 *p++ = "0123456789ABCDEF"[data[i] & 15];
 135         }
 136
 137         *p = 0;
 138
 139         debug(c, "%s: len %lu src %u dst %u seq %u ack %u wnd %u aux %x ctl %s%s%s%s%s data %s\n",
 140               dir, (unsigned long)len, hdr.src, hdr.dst, hdr.seq, hdr.ack, hdr.wnd, hdr.aux,
 141               hdr.ctl & SYN ? "SYN" : "",
 142               hdr.ctl & RST ? "RST" : "",
 143               hdr.ctl & FIN ? "FIN" : "",
 144               hdr.ctl & ACK ? "ACK" : "",
 145               hdr.ctl & MF ? "MF" : "",
 146               str
 147              );
 148 }
 149
 150 static void debug_cwnd(struct utcp_connection *c) {
 151         debug(c, "snd.cwnd %u snd.ssthresh %u\n", c->snd.cwnd, ~c->snd.ssthresh ? c->snd.ssthresh : 0);
 152 }
 153 #else
 154 #define debug(...) do {} while(0)
 155 #define print_packet(...) do {} while(0)
 156 #define debug_cwnd(...) do {} while(0)
 157 #endif
 158
 159 static void set_state(struct utcp_connection *c, enum state state) {
 160         c->state = state;
 161
 162         if(state == ESTABLISHED) {
 163                 timespec_clear(&c->conn_timeout);
 164         }
 165
 166         debug(c, "state %s\n", strstate[state]);
 167 }
 168
 169 static bool fin_wanted(struct utcp_connection *c, uint32_t seq) {
 170         if(seq != c->snd.last) {
 171                 return false;
 172         }
 173
 174         switch(c->state) {
 175         case FIN_WAIT_1:
 176         case CLOSING:
 177         case LAST_ACK:
 178                 return true;
 179
 180         default:
 181                 return false;
 182         }
 183 }
 184
 185 static bool is_reliable(struct utcp_connection *c) {
 186         return c->flags & UTCP_RELIABLE;
 187 }
 188
 189 static int32_t seqdiff(uint32_t a, uint32_t b) {
 190         return a - b;
 191 }
 192
 193 // Buffer functions
 194 static bool buffer_wraps(struct buffer *buf) {
 195         return buf->size - buf->offset < buf->used;
 196 }
 197
 198 static bool buffer_resize(struct buffer *buf, uint32_t newsize) {
 199         char *newdata = realloc(buf->data, newsize);
 200
 201         if(!newdata) {
 202                 return false;
 203         }
 204
 205         buf->data = newdata;
 206
 207         if(buffer_wraps(buf)) {
 208                 // Shift the right part of the buffer until it hits the end of the new buffer.
 209                 // Old situation:
 210                 // [345......012]
 211                 // New situation:
 212                 // [345.........|........012]
 213                 uint32_t tailsize = buf->size - buf->offset;
 214                 uint32_t newoffset = newsize - tailsize;
 215                 memmove(buf->data + newoffset, buf->data + buf->offset, tailsize);
 216                 buf->offset = newoffset;
 217         }
 218
 219         buf->size = newsize;
 220         return true;
 221 }
 222
 223 // Store data into the buffer
 224 static ssize_t buffer_put_at(struct buffer *buf, size_t offset, const void *data, size_t len) {
 225         debug(NULL, "buffer_put_at %lu %lu %lu\n", (unsigned long)buf->used, (unsigned long)offset, (unsigned long)len);
 226
 227         // Ensure we don't store more than maxsize bytes in total
 228         size_t required = offset + len;
 229
 230         if(required > buf->maxsize) {
 231                 if(offset >= buf->maxsize) {
 232                         return 0;
 233                 }
 234
 235                 len = buf->maxsize - offset;
 236                 required = buf->maxsize;
 237         }
 238
 239         // Check if we need to resize the buffer
 240         if(required > buf->size) {
 241                 size_t newsize = buf->size;
 242
 243                 if(!newsize) {
 244                         newsize = 4096;
 245                 }
 246
 247                 do {
 248                         newsize *= 2;
 249                 } while(newsize < required);
 250
 251                 if(newsize > buf->maxsize) {
 252                         newsize = buf->maxsize;
 253                 }
 254
 255                 if(!buffer_resize(buf, newsize)) {
 256                         return -1;
 257                 }
 258         }
 259
 260         uint32_t realoffset = buf->offset + offset;
 261
 262         if(buf->size - buf->offset <= offset) {
 263                 // The offset wrapped
 264                 realoffset -= buf->size;
 265         }
 266
 267         if(buf->size - realoffset < len) {
 268                 // The new chunk of data must be wrapped
 269                 memcpy(buf->data + realoffset, data, buf->size - realoffset);
 270                 memcpy(buf->data, (char *)data + buf->size - realoffset, len - (buf->size - realoffset));
 271         } else {
 272                 memcpy(buf->data + realoffset, data, len);
 273         }
 274
 275         if(required > buf->used) {
 276                 buf->used = required;
 277         }
 278
 279         return len;
 280 }
 281
 282 static ssize_t buffer_put(struct buffer *buf, const void *data, size_t len) {
 283         return buffer_put_at(buf, buf->used, data, len);
 284 }
 285
 286 // Copy data from the buffer without removing it.
 287 static ssize_t buffer_copy(struct buffer *buf, void *data, size_t offset, size_t len) {
 288         // Ensure we don't copy more than is actually stored in the buffer
 289         if(offset >= buf->used) {
 290                 return 0;
 291         }
 292
 293         if(buf->used - offset < len) {
 294                 len = buf->used - offset;
 295         }
 296
 297         uint32_t realoffset = buf->offset + offset;
 298
 299         if(buf->size - buf->offset <= offset) {
 300                 // The offset wrapped
 301                 realoffset -= buf->size;
 302         }
 303
 304         if(buf->size - realoffset < len) {
 305                 // The data is wrapped
 306                 memcpy(data, buf->data + realoffset, buf->size - realoffset);
 307                 memcpy((char *)data + buf->size - realoffset, buf->data, len - (buf->size - realoffset));
 308         } else {
 309                 memcpy(data, buf->data + realoffset, len);
 310         }
 311
 312         return len;
 313 }
 314
 315 // Copy data from the buffer without removing it.
 316 static ssize_t buffer_call(struct utcp_connection *c, struct buffer *buf, size_t offset, size_t len) {
 317         if(!c->recv) {
 318                 return len;
 319         }
 320
 321         // Ensure we don't copy more than is actually stored in the buffer
 322         if(offset >= buf->used) {
 323                 return 0;
 324         }
 325
 326         if(buf->used - offset < len) {
 327                 len = buf->used - offset;
 328         }
 329
 330         uint32_t realoffset = buf->offset + offset;
 331
 332         if(buf->size - buf->offset <= offset) {
 333                 // The offset wrapped
 334                 realoffset -= buf->size;
 335         }
 336
 337         if(buf->size - realoffset < len) {
 338                 // The data is wrapped
 339                 ssize_t rx1 = c->recv(c, buf->data + realoffset, buf->size - realoffset);
 340
 341                 if(rx1 < buf->size - realoffset) {
 342                         return rx1;
 343                 }
 344
 345                 // The channel might have been closed by the previous callback
 346                 if(!c->recv) {
 347                         return len;
 348                 }
 349
 350                 ssize_t rx2 = c->recv(c, buf->data, len - (buf->size - realoffset));
 351
 352                 if(rx2 < 0) {
 353                         return rx2;
 354                 } else {
 355                         return rx1 + rx2;
 356                 }
 357         } else {
 358                 return c->recv(c, buf->data + realoffset, len);
 359         }
 360 }
 361
 362 // Discard data from the buffer.
 363 static ssize_t buffer_discard(struct buffer *buf, size_t len) {
 364         if(buf->used < len) {
 365                 len = buf->used;
 366         }
 367
 368         if(buf->size - buf->offset <= len) {
 369                 buf->offset -= buf->size;
 370         }
 371
 372         if(buf->used == len) {
 373                 buf->offset = 0;
 374         } else {
 375                 buf->offset += len;
 376         }
 377
 378         buf->used -= len;
 379
 380         return len;
 381 }
 382
 383 static void buffer_clear(struct buffer *buf) {
 384         buf->used = 0;
 385         buf->offset = 0;
 386 }
 387
 388 static bool buffer_set_size(struct buffer *buf, uint32_t minsize, uint32_t maxsize) {
 389         if(maxsize < minsize) {
 390                 maxsize = minsize;
 391         }
 392
 393         buf->maxsize = maxsize;
 394
 395         return buf->size >= minsize || buffer_resize(buf, minsize);
 396 }
 397
 398 static void buffer_transfer(struct buffer *buf, char *newdata, size_t newsize) {
 399         if(buffer_wraps(buf)) {
 400                 // Old situation:
 401                 // [345......012]
 402                 // New situation:
 403                 // [012345......]
 404                 uint32_t tailsize = buf->size - buf->offset;
 405                 memcpy(newdata, buf->data + buf->offset, tailsize);
 406                 memcpy(newdata + tailsize, buf->data, buf->used - tailsize);
 407         } else {
 408                 // Old situation:
 409                 // [....012345..]
 410                 // New situation:
 411                 // [012345......]
 412                 memcpy(newdata, buf->data + buf->offset, buf->used);
 413         }
 414
 415         buf->offset = 0;
 416         buf->size = newsize;
 417 }
 418
 419 static void set_buffer_storage(struct buffer *buf, char *data, size_t size) {
 420         if(size > UINT32_MAX) {
 421                 size = UINT32_MAX;
 422         }
 423
 424         buf->maxsize = size;
 425
 426         if(data) {
 427                 if(buf->external) {
 428                         // Don't allow resizing an external buffer
 429                         abort();
 430                 }
 431
 432                 if(size < buf->used) {
 433                         // Ignore requests for an external buffer if we are already using more than it can store
 434                         return;
 435                 }
 436
 437                 // Transition from internal to external buffer
 438                 buffer_transfer(buf, data, size);
 439                 free(buf->data);
 440                 buf->data = data;
 441                 buf->external = true;
 442         } else if(buf->external) {
 443                 // Transition from external to internal buf
 444                 size_t minsize = buf->used < DEFAULT_SNDBUFSIZE ? DEFAULT_SNDBUFSIZE : buf->used;
 445                 data = malloc(minsize);
 446
 447                 if(!data) {
 448                         // Cannot handle this
 449                         abort();
 450                 }
 451
 452                 buffer_transfer(buf, data, minsize);
 453                 buf->data = data;
 454                 buf->external = false;
 455         }
 456 }
 457
 458 static void buffer_exit(struct buffer *buf) {
 459         if(!buf->external) {
 460                 free(buf->data);
 461         }
 462
 463         memset(buf, 0, sizeof(*buf));
 464 }
 465
 466 static uint32_t buffer_free(const struct buffer *buf) {
 467         return buf->maxsize > buf->used ? buf->maxsize - buf->used : 0;
 468 }
 469
 470 // Connections are stored in a sorted list.
 471 // This gives O(log(N)) lookup time, O(N log(N)) insertion time and O(N) deletion time.
 472
 473 static int compare(const void *va, const void *vb) {
 474         assert(va && vb);
 475
 476         const struct utcp_connection *a = *(struct utcp_connection **)va;
 477         const struct utcp_connection *b = *(struct utcp_connection **)vb;
 478
 479         assert(a && b);
 480
 481         int c = (int)a->src - (int)b->src;
 482
 483         if(c) {
 484                 return c;
 485         }
 486
 487         c = (int)a->dst - (int)b->dst;
 488         return c;
 489 }
 490
 491 static struct utcp_connection *find_connection(const struct utcp *utcp, uint16_t src, uint16_t dst) {
 492         if(!utcp->nconnections) {
 493                 return NULL;
 494         }
 495
 496         struct utcp_connection key = {
 497                 .src = src,
 498                 .dst = dst,
 499         }, *keyp = &key;
 500         struct utcp_connection **match = bsearch(&keyp, utcp->connections, utcp->nconnections, sizeof(*utcp->connections), compare);
 501         return match ? *match : NULL;
 502 }
 503
 504 static void free_connection(struct utcp_connection *c) {
 505         struct utcp *utcp = c->utcp;
 506         struct utcp_connection **cp = bsearch(&c, utcp->connections, utcp->nconnections, sizeof(*utcp->connections), compare);
 507
 508         assert(cp);
 509
 510         int i = cp - utcp->connections;
 511         memmove(cp, cp + 1, (utcp->nconnections - i - 1) * sizeof(*cp));
 512         utcp->nconnections--;
 513
 514         buffer_exit(&c->rcvbuf);
 515         buffer_exit(&c->sndbuf);
 516         free(c);
 517 }
 518
 519 static struct utcp_connection *allocate_connection(struct utcp *utcp, uint16_t src, uint16_t dst) {
 520         // Check whether this combination of src and dst is free
 521
 522         if(src) {
 523                 if(find_connection(utcp, src, dst)) {
 524                         errno = EADDRINUSE;
 525                         return NULL;
 526                 }
 527         } else { // If src == 0, generate a random port number with the high bit set
 528                 if(utcp->nconnections >= 32767) {
 529                         errno = ENOMEM;
 530                         return NULL;
 531                 }
 532
 533                 src = rand() | 0x8000;
 534
 535                 while(find_connection(utcp, src, dst)) {
 536                         src++;
 537                 }
 538         }
 539
 540         // Allocate memory for the new connection
 541
 542         if(utcp->nconnections >= utcp->nallocated) {
 543                 if(!utcp->nallocated) {
 544                         utcp->nallocated = 4;
 545                 } else {
 546                         utcp->nallocated *= 2;
 547                 }
 548
 549                 struct utcp_connection **new_array = realloc(utcp->connections, utcp->nallocated * sizeof(*utcp->connections));
 550
 551                 if(!new_array) {
 552                         return NULL;
 553                 }
 554
 555                 utcp->connections = new_array;
 556         }
 557
 558         struct utcp_connection *c = calloc(1, sizeof(*c));
 559
 560         if(!c) {
 561                 return NULL;
 562         }
 563
 564         if(!buffer_set_size(&c->sndbuf, DEFAULT_SNDBUFSIZE, DEFAULT_MAXSNDBUFSIZE)) {
 565                 free(c);
 566                 return NULL;
 567         }
 568
 569         if(!buffer_set_size(&c->rcvbuf, DEFAULT_RCVBUFSIZE, DEFAULT_MAXRCVBUFSIZE)) {
 570                 buffer_exit(&c->sndbuf);
 571                 free(c);
 572                 return NULL;
 573         }
 574
 575         // Fill in the details
 576
 577         c->src = src;
 578         c->dst = dst;
 579 #ifdef UTCP_DEBUG
 580         c->snd.iss = 0;
 581 #else
 582         c->snd.iss = rand();
 583 #endif
 584         c->snd.una = c->snd.iss;
 585         c->snd.nxt = c->snd.iss + 1;
 586         c->snd.last = c->snd.nxt;
 587         c->snd.cwnd = (utcp->mss > 2190 ? 2 : utcp->mss > 1095 ? 3 : 4) * utcp->mss;
 588         c->snd.ssthresh = ~0;
 589         debug_cwnd(c);
 590         c->srtt = 0;
 591         c->rttvar = 0;
 592         c->rto = START_RTO;
 593         c->utcp = utcp;
 594
 595         // Add it to the sorted list of connections
 596
 597         utcp->connections[utcp->nconnections++] = c;
 598         qsort(utcp->connections, utcp->nconnections, sizeof(*utcp->connections), compare);
 599
 600         return c;
 601 }
 602
 603 static inline uint32_t absdiff(uint32_t a, uint32_t b) {
 604         if(a > b) {
 605                 return a - b;
 606         } else {
 607                 return b - a;
 608         }
 609 }
 610
 611 // Update RTT variables. See RFC 6298.
 612 static void update_rtt(struct utcp_connection *c, uint32_t rtt) {
 613         if(!rtt) {
 614                 debug(c, "invalid rtt\n");
 615                 return;
 616         }
 617
 618         if(!c->srtt) {
 619                 c->srtt = rtt;
 620                 c->rttvar = rtt / 2;
 621         } else {
 622                 c->rttvar = (c->rttvar * 3 + absdiff(c->srtt, rtt)) / 4;
 623                 c->srtt = (c->srtt * 7 + rtt) / 8;
 624         }
 625
 626         c->rto = c->srtt + max(4 * c->rttvar, CLOCK_GRANULARITY);
 627
 628         if(c->rto > MAX_RTO) {
 629                 c->rto = MAX_RTO;
 630         }
 631
 632         debug(c, "rtt %u srtt %u rttvar %u rto %u\n", rtt, c->srtt, c->rttvar, c->rto);
 633 }
 634
 635 static void start_retransmit_timer(struct utcp_connection *c) {
 636         clock_gettime(UTCP_CLOCK, &c->rtrx_timeout);
 637
 638         uint32_t rto = c->rto;
 639
 640         while(rto > USEC_PER_SEC) {
 641                 c->rtrx_timeout.tv_sec++;
 642                 rto -= USEC_PER_SEC;
 643         }
 644
 645         c->rtrx_timeout.tv_nsec += rto * 1000;
 646
 647         if(c->rtrx_timeout.tv_nsec >= NSEC_PER_SEC) {
 648                 c->rtrx_timeout.tv_nsec -= NSEC_PER_SEC;
 649                 c->rtrx_timeout.tv_sec++;
 650         }
 651
 652         debug(c, "rtrx_timeout %ld.%06lu\n", c->rtrx_timeout.tv_sec, c->rtrx_timeout.tv_nsec);
 653 }
 654
 655 static void stop_retransmit_timer(struct utcp_connection *c) {
 656         timespec_clear(&c->rtrx_timeout);
 657         debug(c, "rtrx_timeout cleared\n");
 658 }
 659
 660 struct utcp_connection *utcp_connect_ex(struct utcp *utcp, uint16_t dst, utcp_recv_t recv, void *priv, uint32_t flags) {
 661         struct utcp_connection *c = allocate_connection(utcp, 0, dst);
 662
 663         if(!c) {
 664                 return NULL;
 665         }
 666
 667         assert((flags & ~0x1f) == 0);
 668
 669         c->flags = flags;
 670         c->recv = recv;
 671         c->priv = priv;
 672
 673         struct {
 674                 struct hdr hdr;
 675                 uint8_t init[4];
 676         } pkt;
 677
 678         pkt.hdr.src = c->src;
 679         pkt.hdr.dst = c->dst;
 680         pkt.hdr.seq = c->snd.iss;
 681         pkt.hdr.ack = 0;
 682         pkt.hdr.wnd = c->rcvbuf.maxsize;
 683         pkt.hdr.ctl = SYN;
 684         pkt.hdr.aux = 0x0101;
 685         pkt.init[0] = 1;
 686         pkt.init[1] = 0;
 687         pkt.init[2] = 0;
 688         pkt.init[3] = flags & 0x7;
 689
 690         set_state(c, SYN_SENT);
 691
 692         print_packet(c, "send", &pkt, sizeof(pkt));
 693         utcp->send(utcp, &pkt, sizeof(pkt));
 694
 695         clock_gettime(UTCP_CLOCK, &c->conn_timeout);
 696         c->conn_timeout.tv_sec += utcp->timeout;
 697
 698         start_retransmit_timer(c);
 699
 700         return c;
 701 }
 702
 703 struct utcp_connection *utcp_connect(struct utcp *utcp, uint16_t dst, utcp_recv_t recv, void *priv) {
 704         return utcp_connect_ex(utcp, dst, recv, priv, UTCP_TCP);
 705 }
 706
 707 void utcp_accept(struct utcp_connection *c, utcp_recv_t recv, void *priv) {
 708         if(c->reapable || c->state != SYN_RECEIVED) {
 709                 debug(c, "accept() called on invalid connection in state %s\n", c, strstate[c->state]);
 710                 return;
 711         }
 712
 713         debug(c, "accepted %p %p\n", c, recv, priv);
 714         c->recv = recv;
 715         c->priv = priv;
 716         c->do_poll = true;
 717         set_state(c, ESTABLISHED);
 718 }
 719
 720 static void ack(struct utcp_connection *c, bool sendatleastone) {
 721         int32_t left = seqdiff(c->snd.last, c->snd.nxt);
 722         int32_t cwndleft = is_reliable(c) ? min(c->snd.cwnd, c->snd.wnd) - seqdiff(c->snd.nxt, c->snd.una) : MAX_UNRELIABLE_SIZE;
 723
 724         assert(left >= 0);
 725
 726         if(cwndleft <= 0) {
 727                 left = 0;
 728         } else if(cwndleft < left) {
 729                 left = cwndleft;
 730
 731                 if(!sendatleastone || cwndleft > c->utcp->mss) {
 732                         left -= left % c->utcp->mss;
 733                 }
 734         }
 735
 736         debug(c, "cwndleft %d left %d\n", cwndleft, left);
 737
 738         if(!left && !sendatleastone) {
 739                 return;
 740         }
 741
 742         struct {
 743                 struct hdr hdr;
 744                 uint8_t data[];
 745         } *pkt = c->utcp->pkt;
 746
 747         pkt->hdr.src = c->src;
 748         pkt->hdr.dst = c->dst;
 749         pkt->hdr.ack = c->rcv.nxt;
 750         pkt->hdr.wnd = is_reliable(c) ? c->rcvbuf.maxsize : 0;
 751         pkt->hdr.ctl = ACK;
 752         pkt->hdr.aux = 0;
 753
 754         do {
 755                 uint32_t seglen = left > c->utcp->mss ? c->utcp->mss : left;
 756                 pkt->hdr.seq = c->snd.nxt;
 757
 758                 buffer_copy(&c->sndbuf, pkt->data, seqdiff(c->snd.nxt, c->snd.una), seglen);
 759
 760                 c->snd.nxt += seglen;
 761                 left -= seglen;
 762
 763                 if(!is_reliable(c)) {
 764                         if(left) {
 765                                 pkt->hdr.ctl |= MF;
 766                         } else {
 767                                 pkt->hdr.ctl &= ~MF;
 768                         }
 769                 }
 770
 771                 if(seglen && fin_wanted(c, c->snd.nxt)) {
 772                         seglen--;
 773                         pkt->hdr.ctl |= FIN;
 774                 }
 775
 776                 if(!c->rtt_start.tv_sec) {
 777                         // Start RTT measurement
 778                         clock_gettime(UTCP_CLOCK, &c->rtt_start);
 779                         c->rtt_seq = pkt->hdr.seq + seglen;
 780                         debug(c, "starting RTT measurement, expecting ack %u\n", c->rtt_seq);
 781                 }
 782
 783                 print_packet(c, "send", pkt, sizeof(pkt->hdr) + seglen);
 784                 c->utcp->send(c->utcp, pkt, sizeof(pkt->hdr) + seglen);
 785
 786                 if(left && !is_reliable(c)) {
 787                         pkt->hdr.wnd += seglen;
 788                 }
 789         } while(left);
 790 }
 791
 792 ssize_t utcp_send(struct utcp_connection *c, const void *data, size_t len) {
 793         if(c->reapable) {
 794                 debug(c, "send() called on closed connection\n");
 795                 errno = EBADF;
 796                 return -1;
 797         }
 798
 799         switch(c->state) {
 800         case CLOSED:
 801         case LISTEN:
 802                 debug(c, "send() called on unconnected connection\n");
 803                 errno = ENOTCONN;
 804                 return -1;
 805
 806         case SYN_SENT:
 807         case SYN_RECEIVED:
 808         case ESTABLISHED:
 809         case CLOSE_WAIT:
 810                 break;
 811
 812         case FIN_WAIT_1:
 813         case FIN_WAIT_2:
 814         case CLOSING:
 815         case LAST_ACK:
 816         case TIME_WAIT:
 817                 debug(c, "send() called on closed connection\n");
 818                 errno = EPIPE;
 819                 return -1;
 820         }
 821
 822         // Exit early if we have nothing to send.
 823
 824         if(!len) {
 825                 return 0;
 826         }
 827
 828         if(!data) {
 829                 errno = EFAULT;
 830                 return -1;
 831         }
 832
 833         // Check if we need to be able to buffer all data
 834
 835         if(c->flags & UTCP_NO_PARTIAL) {
 836                 if(len > buffer_free(&c->sndbuf)) {
 837                         if(len > c->sndbuf.maxsize) {
 838                                 errno = EMSGSIZE;
 839                                 return -1;
 840                         } else {
 841                                 errno = EWOULDBLOCK;
 842                                 return 0;
 843                         }
 844                 }
 845         }
 846
 847         // Add data to send buffer.
 848
 849         if(is_reliable(c)) {
 850                 len = buffer_put(&c->sndbuf, data, len);
 851         } else if(c->state != SYN_SENT && c->state != SYN_RECEIVED) {
 852                 if(len > MAX_UNRELIABLE_SIZE || buffer_put(&c->sndbuf, data, len) != (ssize_t)len) {
 853                         errno = EMSGSIZE;
 854                         return -1;
 855                 }
 856         } else {
 857                 return 0;
 858         }
 859
 860         if(len <= 0) {
 861                 if(is_reliable(c)) {
 862                         errno = EWOULDBLOCK;
 863                         return 0;
 864                 } else {
 865                         return len;
 866                 }
 867         }
 868
 869         c->snd.last += len;
 870
 871         // Don't send anything yet if the connection has not fully established yet
 872
 873         if(c->state == SYN_SENT || c->state == SYN_RECEIVED) {
 874                 return len;
 875         }
 876
 877         ack(c, false);
 878
 879         if(!is_reliable(c)) {
 880                 c->snd.una = c->snd.nxt = c->snd.last;
 881                 buffer_discard(&c->sndbuf, c->sndbuf.used);
 882         }
 883
 884         if(is_reliable(c) && !timespec_isset(&c->rtrx_timeout)) {
 885                 start_retransmit_timer(c);
 886         }
 887
 888         if(is_reliable(c) && !timespec_isset(&c->conn_timeout)) {
 889                 clock_gettime(UTCP_CLOCK, &c->conn_timeout);
 890                 c->conn_timeout.tv_sec += c->utcp->timeout;
 891         }
 892
 893         return len;
 894 }
 895
 896 static void swap_ports(struct hdr *hdr) {
 897         uint16_t tmp = hdr->src;
 898         hdr->src = hdr->dst;
 899         hdr->dst = tmp;
 900 }
 901
 902 static void fast_retransmit(struct utcp_connection *c) {
 903         if(c->state == CLOSED || c->snd.last == c->snd.una) {
 904                 debug(c, "fast_retransmit() called but nothing to retransmit!\n");
 905                 return;
 906         }
 907
 908         struct utcp *utcp = c->utcp;
 909
 910         struct {
 911                 struct hdr hdr;
 912                 uint8_t data[];
 913         } *pkt = c->utcp->pkt;
 914
 915         pkt->hdr.src = c->src;
 916         pkt->hdr.dst = c->dst;
 917         pkt->hdr.wnd = c->rcvbuf.maxsize;
 918         pkt->hdr.aux = 0;
 919
 920         switch(c->state) {
 921         case ESTABLISHED:
 922         case FIN_WAIT_1:
 923         case CLOSE_WAIT:
 924         case CLOSING:
 925         case LAST_ACK:
 926                 // Send unacked data again.
 927                 pkt->hdr.seq = c->snd.una;
 928                 pkt->hdr.ack = c->rcv.nxt;
 929                 pkt->hdr.ctl = ACK;
 930                 uint32_t len = min(seqdiff(c->snd.last, c->snd.una), utcp->mss);
 931
 932                 if(fin_wanted(c, c->snd.una + len)) {
 933                         len--;
 934                         pkt->hdr.ctl |= FIN;
 935                 }
 936
 937                 buffer_copy(&c->sndbuf, pkt->data, 0, len);
 938                 print_packet(c, "rtrx", pkt, sizeof(pkt->hdr) + len);
 939                 utcp->send(utcp, pkt, sizeof(pkt->hdr) + len);
 940                 break;
 941
 942         default:
 943                 break;
 944         }
 945 }
 946
 947 static void retransmit(struct utcp_connection *c) {
 948         if(c->state == CLOSED || c->snd.last == c->snd.una) {
 949                 debug(c, "retransmit() called but nothing to retransmit!\n");
 950                 stop_retransmit_timer(c);
 951                 return;
 952         }
 953
 954         struct utcp *utcp = c->utcp;
 955
 956         if(utcp->retransmit) {
 957                 utcp->retransmit(c);
 958         }
 959
 960         struct {
 961                 struct hdr hdr;
 962                 uint8_t data[];
 963         } *pkt = c->utcp->pkt;
 964
 965         pkt->hdr.src = c->src;
 966         pkt->hdr.dst = c->dst;
 967         pkt->hdr.wnd = c->rcvbuf.maxsize;
 968         pkt->hdr.aux = 0;
 969
 970         switch(c->state) {
 971         case SYN_SENT:
 972                 // Send our SYN again
 973                 pkt->hdr.seq = c->snd.iss;
 974                 pkt->hdr.ack = 0;
 975                 pkt->hdr.ctl = SYN;
 976                 pkt->hdr.aux = 0x0101;
 977                 pkt->data[0] = 1;
 978                 pkt->data[1] = 0;
 979                 pkt->data[2] = 0;
 980                 pkt->data[3] = c->flags & 0x7;
 981                 print_packet(c, "rtrx", pkt, sizeof(pkt->hdr) + 4);
 982                 utcp->send(utcp, pkt, sizeof(pkt->hdr) + 4);
 983                 break;
 984
 985         case SYN_RECEIVED:
 986                 // Send SYNACK again
 987                 pkt->hdr.seq = c->snd.nxt;
 988                 pkt->hdr.ack = c->rcv.nxt;
 989                 pkt->hdr.ctl = SYN | ACK;
 990                 print_packet(c, "rtrx", pkt, sizeof(pkt->hdr));
 991                 utcp->send(utcp, pkt, sizeof(pkt->hdr));
 992                 break;
 993
 994         case ESTABLISHED:
 995         case FIN_WAIT_1:
 996         case CLOSE_WAIT:
 997         case CLOSING:
 998         case LAST_ACK:
 999                 // Send unacked data again.
1000                 pkt->hdr.seq = c->snd.una;
1001                 pkt->hdr.ack = c->rcv.nxt;
1002                 pkt->hdr.ctl = ACK;
1003                 uint32_t len = min(seqdiff(c->snd.last, c->snd.una), utcp->mss);
1004
1005                 if(fin_wanted(c, c->snd.una + len)) {
1006                         len--;
1007                         pkt->hdr.ctl |= FIN;
1008                 }
1009
1010                 // RFC 5681 slow start after timeout
1011                 uint32_t flightsize = seqdiff(c->snd.nxt, c->snd.una);
1012                 c->snd.ssthresh = max(flightsize / 2, utcp->mss * 2); // eq. 4
1013                 c->snd.cwnd = utcp->mss;
1014                 debug_cwnd(c);
1015
1016                 buffer_copy(&c->sndbuf, pkt->data, 0, len);
1017                 print_packet(c, "rtrx", pkt, sizeof(pkt->hdr) + len);
1018                 utcp->send(utcp, pkt, sizeof(pkt->hdr) + len);
1019
1020                 c->snd.nxt = c->snd.una + len;
1021                 break;
1022
1023         case CLOSED:
1024         case LISTEN:
1025         case TIME_WAIT:
1026         case FIN_WAIT_2:
1027                 // We shouldn't need to retransmit anything in this state.
1028 #ifdef UTCP_DEBUG
1029                 abort();
1030 #endif
1031                 stop_retransmit_timer(c);
1032                 goto cleanup;
1033         }
1034
1035         start_retransmit_timer(c);
1036         c->rto *= 2;
1037
1038         if(c->rto > MAX_RTO) {
1039                 c->rto = MAX_RTO;
1040         }
1041
1042         c->rtt_start.tv_sec = 0; // invalidate RTT timer
1043         c->dupack = 0; // cancel any ongoing fast recovery
1044
1045 cleanup:
1046         return;
1047 }
1048
1049 /* Update receive buffer and SACK entries after consuming data.
1050  *
1051  * Situation:
1052  *
1053  * |.....0000..1111111111.....22222......3333|
1054  * |---------------^
1055  *
1056  * 0..3 represent the SACK entries. The ^ indicates up to which point we want
1057  * to remove data from the receive buffer. The idea is to substract "len"
1058  * from the offset of all the SACK entries, and then remove/cut down entries
1059  * that are shifted to before the start of the receive buffer.
1060  *
1061  * There are three cases:
1062  * - the SACK entry is after ^, in that case just change the offset.
1063  * - the SACK entry starts before and ends after ^, so we have to
1064  *   change both its offset and size.
1065  * - the SACK entry is completely before ^, in that case delete it.
1066  */
1067 static void sack_consume(struct utcp_connection *c, size_t len) {
1068         debug(c, "sack_consume %lu\n", (unsigned long)len);
1069
1070         if(len > c->rcvbuf.used) {
1071                 debug(c, "all SACK entries consumed\n");
1072                 c->sacks[0].len = 0;
1073                 return;
1074         }
1075
1076         buffer_discard(&c->rcvbuf, len);
1077
1078         for(int i = 0; i < NSACKS && c->sacks[i].len;) {
1079                 if(len < c->sacks[i].offset) {
1080                         c->sacks[i].offset -= len;
1081                         i++;
1082                 } else if(len < c->sacks[i].offset + c->sacks[i].len) {
1083                         c->sacks[i].len -= len - c->sacks[i].offset;
1084                         c->sacks[i].offset = 0;
1085                         i++;
1086                 } else {
1087                         if(i < NSACKS - 1) {
1088                                 memmove(&c->sacks[i], &c->sacks[i + 1], (NSACKS - 1 - i) * sizeof(c->sacks)[i]);
1089                                 c->sacks[NSACKS - 1].len = 0;
1090                         } else {
1091                                 c->sacks[i].len = 0;
1092                                 break;
1093                         }
1094                 }
1095         }
1096
1097         for(int i = 0; i < NSACKS && c->sacks[i].len; i++) {
1098                 debug(c, "SACK[%d] offset %u len %u\n", i, c->sacks[i].offset, c->sacks[i].len);
1099         }
1100 }
1101
1102 static void handle_out_of_order(struct utcp_connection *c, uint32_t offset, const void *data, size_t len) {
1103         debug(c, "out of order packet, offset %u\n", offset);
1104         // Packet loss or reordering occured. Store the data in the buffer.
1105         ssize_t rxd = buffer_put_at(&c->rcvbuf, offset, data, len);
1106
1107         if(rxd <= 0) {
1108                 debug(c, "packet outside receive buffer, dropping\n");
1109                 return;
1110         }
1111
1112         if((size_t)rxd < len) {
1113                 debug(c, "packet partially outside receive buffer\n");
1114                 len = rxd;
1115         }
1116
1117         // Make note of where we put it.
1118         for(int i = 0; i < NSACKS; i++) {
1119                 if(!c->sacks[i].len) { // nothing to merge, add new entry
1120                         debug(c, "new SACK entry %d\n", i);
1121                         c->sacks[i].offset = offset;
1122                         c->sacks[i].len = rxd;
1123                         break;
1124                 } else if(offset < c->sacks[i].offset) {
1125                         if(offset + rxd < c->sacks[i].offset) { // insert before
1126                                 if(!c->sacks[NSACKS - 1].len) { // only if room left
1127                                         debug(c, "insert SACK entry at %d\n", i);
1128                                         memmove(&c->sacks[i + 1], &c->sacks[i], (NSACKS - i - 1) * sizeof(c->sacks)[i]);
1129                                         c->sacks[i].offset = offset;
1130                                         c->sacks[i].len = rxd;
1131                                 } else {
1132                                         debug(c, "SACK entries full, dropping packet\n");
1133                                 }
1134
1135                                 break;
1136                         } else { // merge
1137                                 debug(c, "merge with start of SACK entry at %d\n", i);
1138                                 c->sacks[i].offset = offset;
1139                                 break;
1140                         }
1141                 } else if(offset <= c->sacks[i].offset + c->sacks[i].len) {
1142                         if(offset + rxd > c->sacks[i].offset + c->sacks[i].len) { // merge
1143                                 debug(c, "merge with end of SACK entry at %d\n", i);
1144                                 c->sacks[i].len = offset + rxd - c->sacks[i].offset;
1145                                 // TODO: handle potential merge with next entry
1146                         }
1147
1148                         break;
1149                 }
1150         }
1151
1152         for(int i = 0; i < NSACKS && c->sacks[i].len; i++) {
1153                 debug(c, "SACK[%d] offset %u len %u\n", i, c->sacks[i].offset, c->sacks[i].len);
1154         }
1155 }
1156
1157 static void handle_in_order(struct utcp_connection *c, const void *data, size_t len) {
1158         if(c->recv) {
1159                 ssize_t rxd = c->recv(c, data, len);
1160
1161                 if(rxd != (ssize_t)len) {
1162                         // TODO: handle the application not accepting all data.
1163                         abort();
1164                 }
1165         }
1166
1167         // Check if we can process out-of-order data now.
1168         if(c->sacks[0].len && len >= c->sacks[0].offset) {
1169                 debug(c, "incoming packet len %lu connected with SACK at %u\n", (unsigned long)len, c->sacks[0].offset);
1170
1171                 if(len < c->sacks[0].offset + c->sacks[0].len) {
1172                         size_t offset = len;
1173                         len = c->sacks[0].offset + c->sacks[0].len;
1174                         size_t remainder = len - offset;
1175
1176                         ssize_t rxd = buffer_call(c, &c->rcvbuf, offset, remainder);
1177
1178                         if(rxd != (ssize_t)remainder) {
1179                                 // TODO: handle the application not accepting all data.
1180                                 abort();
1181                         }
1182                 }
1183         }
1184
1185         if(c->rcvbuf.used) {
1186                 sack_consume(c, len);
1187         }
1188
1189         c->rcv.nxt += len;
1190 }
1191
1192 static void handle_unreliable(struct utcp_connection *c, const struct hdr *hdr, const void *data, size_t len) {
1193         // Fast path for unfragmented packets
1194         if(!hdr->wnd && !(hdr->ctl & MF)) {
1195                 if(c->recv) {
1196                         c->recv(c, data, len);
1197                 }
1198
1199                 c->rcv.nxt = hdr->seq + len;
1200                 return;
1201         }
1202
1203         // Ensure reassembled packet are not larger than 64 kiB
1204         if(hdr->wnd >= MAX_UNRELIABLE_SIZE || hdr->wnd + len > MAX_UNRELIABLE_SIZE) {
1205                 return;
1206         }
1207
1208         // Don't accept out of order fragments
1209         if(hdr->wnd && hdr->seq != c->rcv.nxt) {
1210                 return;
1211         }
1212
1213         // Reset the receive buffer for the first fragment
1214         if(!hdr->wnd) {
1215                 buffer_clear(&c->rcvbuf);
1216         }
1217
1218         ssize_t rxd = buffer_put_at(&c->rcvbuf, hdr->wnd, data, len);
1219
1220         if(rxd != (ssize_t)len) {
1221                 return;
1222         }
1223
1224         // Send the packet if it's the final fragment
1225         if(!(hdr->ctl & MF)) {
1226                 buffer_call(c, &c->rcvbuf, 0, hdr->wnd + len);
1227         }
1228
1229         c->rcv.nxt = hdr->seq + len;
1230 }
1231
1232 static void handle_incoming_data(struct utcp_connection *c, const struct hdr *hdr, const void *data, size_t len) {
1233         if(!is_reliable(c)) {
1234                 handle_unreliable(c, hdr, data, len);
1235                 return;
1236         }
1237
1238         uint32_t offset = seqdiff(hdr->seq, c->rcv.nxt);
1239
1240         if(offset) {
1241                 handle_out_of_order(c, offset, data, len);
1242         } else {
1243                 handle_in_order(c, data, len);
1244         }
1245 }
1246
1247
1248 ssize_t utcp_recv(struct utcp *utcp, const void *data, size_t len) {
1249         const uint8_t *ptr = data;
1250
1251         if(!utcp) {
1252                 errno = EFAULT;
1253                 return -1;
1254         }
1255
1256         if(!len) {
1257                 return 0;
1258         }
1259
1260         if(!data) {
1261                 errno = EFAULT;
1262                 return -1;
1263         }
1264
1265         // Drop packets smaller than the header
1266
1267         struct hdr hdr;
1268
1269         if(len < sizeof(hdr)) {
1270                 print_packet(NULL, "recv", data, len);
1271                 errno = EBADMSG;
1272                 return -1;
1273         }
1274
1275         // Make a copy from the potentially unaligned data to a struct hdr
1276
1277         memcpy(&hdr, ptr, sizeof(hdr));
1278
1279         // Try to match the packet to an existing connection
1280
1281         struct utcp_connection *c = find_connection(utcp, hdr.dst, hdr.src);
1282         print_packet(c, "recv", data, len);
1283
1284         // Process the header
1285
1286         ptr += sizeof(hdr);
1287         len -= sizeof(hdr);
1288
1289         // Drop packets with an unknown CTL flag
1290
1291         if(hdr.ctl & ~(SYN | ACK | RST | FIN | MF)) {
1292                 print_packet(NULL, "recv", data, len);
1293                 errno = EBADMSG;
1294                 return -1;
1295         }
1296
1297         // Check for auxiliary headers
1298
1299         const uint8_t *init = NULL;
1300
1301         uint16_t aux = hdr.aux;
1302
1303         while(aux) {
1304                 size_t auxlen = 4 * (aux >> 8) & 0xf;
1305                 uint8_t auxtype = aux & 0xff;
1306
1307                 if(len < auxlen) {
1308                         errno = EBADMSG;
1309                         return -1;
1310                 }
1311
1312                 switch(auxtype) {
1313                 case AUX_INIT:
1314                         if(!(hdr.ctl & SYN) || auxlen != 4) {
1315                                 errno = EBADMSG;
1316                                 return -1;
1317                         }
1318
1319                         init = ptr;
1320                         break;
1321
1322                 default:
1323                         errno = EBADMSG;
1324                         return -1;
1325                 }
1326
1327                 len -= auxlen;
1328                 ptr += auxlen;
1329
1330                 if(!(aux & 0x800)) {
1331                         break;
1332                 }
1333
1334                 if(len < 2) {
1335                         errno = EBADMSG;
1336                         return -1;
1337                 }
1338
1339                 memcpy(&aux, ptr, 2);
1340                 len -= 2;
1341                 ptr += 2;
1342         }
1343
1344         bool has_data = len || (hdr.ctl & (SYN | FIN));
1345
1346         // Is it for a new connection?
1347
1348         if(!c) {
1349                 // Ignore RST packets
1350
1351                 if(hdr.ctl & RST) {
1352                         return 0;
1353                 }
1354
1355                 // Is it a SYN packet and are we LISTENing?
1356
1357                 if(hdr.ctl & SYN && !(hdr.ctl & ACK) && utcp->accept) {
1358                         // If we don't want to accept it, send a RST back
1359                         if((utcp->listen && !utcp->listen(utcp, hdr.dst))) {
1360                                 len = 1;
1361                                 goto reset;
1362                         }
1363
1364                         // Try to allocate memory, otherwise send a RST back
1365                         c = allocate_connection(utcp, hdr.dst, hdr.src);
1366
1367                         if(!c) {
1368                                 len = 1;
1369                                 goto reset;
1370                         }
1371
1372                         // Parse auxilliary information
1373                         if(init) {
1374                                 if(init[0] < 1) {
1375                                         len = 1;
1376                                         goto reset;
1377                                 }
1378
1379                                 c->flags = init[3] & 0x7;
1380                         } else {
1381                                 c->flags = UTCP_TCP;
1382                         }
1383
1384 synack:
1385                         // Return SYN+ACK, go to SYN_RECEIVED state
1386                         c->snd.wnd = hdr.wnd;
1387                         c->rcv.irs = hdr.seq;
1388                         c->rcv.nxt = c->rcv.irs + 1;
1389                         set_state(c, SYN_RECEIVED);
1390
1391                         struct {
1392                                 struct hdr hdr;
1393                                 uint8_t data[4];
1394                         } pkt;
1395
1396                         pkt.hdr.src = c->src;
1397                         pkt.hdr.dst = c->dst;
1398                         pkt.hdr.ack = c->rcv.irs + 1;
1399                         pkt.hdr.seq = c->snd.iss;
1400                         pkt.hdr.wnd = c->rcvbuf.maxsize;
1401                         pkt.hdr.ctl = SYN | ACK;
1402
1403                         if(init) {
1404                                 pkt.hdr.aux = 0x0101;
1405                                 pkt.data[0] = 1;
1406                                 pkt.data[1] = 0;
1407                                 pkt.data[2] = 0;
1408                                 pkt.data[3] = c->flags & 0x7;
1409                                 print_packet(c, "send", &pkt, sizeof(hdr) + 4);
1410                                 utcp->send(utcp, &pkt, sizeof(hdr) + 4);
1411                         } else {
1412                                 pkt.hdr.aux = 0;
1413                                 print_packet(c, "send", &pkt, sizeof(hdr));
1414                                 utcp->send(utcp, &pkt, sizeof(hdr));
1415                         }
1416
1417                         start_retransmit_timer(c);
1418                 } else {
1419                         // No, we don't want your packets, send a RST back
1420                         len = 1;
1421                         goto reset;
1422                 }
1423
1424                 return 0;
1425         }
1426
1427         debug(c, "state %s\n", strstate[c->state]);
1428
1429         // In case this is for a CLOSED connection, ignore the packet.
1430         // TODO: make it so incoming packets can never match a CLOSED connection.
1431
1432         if(c->state == CLOSED) {
1433                 debug(c, "got packet for closed connection\n");
1434                 goto reset;
1435         }
1436
1437         // It is for an existing connection.
1438
1439         // 1. Drop invalid packets.
1440
1441         // 1a. Drop packets that should not happen in our current state.
1442
1443         switch(c->state) {
1444         case SYN_SENT:
1445         case SYN_RECEIVED:
1446         case ESTABLISHED:
1447         case FIN_WAIT_1:
1448         case FIN_WAIT_2:
1449         case CLOSE_WAIT:
1450         case CLOSING:
1451         case LAST_ACK:
1452         case TIME_WAIT:
1453                 break;
1454
1455         default:
1456 #ifdef UTCP_DEBUG
1457                 abort();
1458 #endif
1459                 break;
1460         }
1461
1462         // 1b. Discard data that is not in our receive window.
1463
1464         if(is_reliable(c)) {
1465                 bool acceptable;
1466
1467                 if(c->state == SYN_SENT) {
1468                         acceptable = true;
1469                 } else if(len == 0) {
1470                         acceptable = seqdiff(hdr.seq, c->rcv.nxt) >= 0;
1471                 } else {
1472                         int32_t rcv_offset = seqdiff(hdr.seq, c->rcv.nxt);
1473
1474                         // cut already accepted front overlapping
1475                         if(rcv_offset < 0) {
1476                                 acceptable = len > (size_t) - rcv_offset;
1477
1478                                 if(acceptable) {
1479                                         ptr -= rcv_offset;
1480                                         len += rcv_offset;
1481                                         hdr.seq -= rcv_offset;
1482                                 }
1483                         } else {
1484                                 acceptable = seqdiff(hdr.seq, c->rcv.nxt) >= 0 && seqdiff(hdr.seq, c->rcv.nxt) + len <= c->rcvbuf.maxsize;
1485                         }
1486                 }
1487
1488                 if(!acceptable) {
1489                         debug(c, "packet not acceptable, %u <= %u + %lu < %u\n", c->rcv.nxt, hdr.seq, (unsigned long)len, c->rcv.nxt + c->rcvbuf.maxsize);
1490
1491                         // Ignore unacceptable RST packets.
1492                         if(hdr.ctl & RST) {
1493                                 return 0;
1494                         }
1495
1496                         // Otherwise, continue processing.
1497                         len = 0;
1498                 }
1499         } else {
1500 #if UTCP_DEBUG
1501                 int32_t rcv_offset = seqdiff(hdr.seq, c->rcv.nxt);
1502
1503                 if(rcv_offset) {
1504                         debug(c, "packet out of order, offset %u bytes", rcv_offset);
1505                 }
1506
1507 #endif
1508         }
1509
1510         c->snd.wnd = hdr.wnd; // TODO: move below
1511
1512         // 1c. Drop packets with an invalid ACK.
1513         // ackno should not roll back, and it should also not be bigger than what we ever could have sent
1514         // (= snd.una + c->sndbuf.used).
1515
1516         if(!is_reliable(c)) {
1517                 if(hdr.ack != c->snd.last && c->state >= ESTABLISHED) {
1518                         hdr.ack = c->snd.una;
1519                 }
1520         }
1521
1522         // 2. Handle RST packets
1523
1524         if(hdr.ctl & RST) {
1525                 switch(c->state) {
1526                 case SYN_SENT:
1527                         if(!(hdr.ctl & ACK)) {
1528                                 return 0;
1529                         }
1530
1531                         // The peer has refused our connection.
1532                         set_state(c, CLOSED);
1533                         errno = ECONNREFUSED;
1534
1535                         if(c->recv) {
1536                                 c->recv(c, NULL, 0);
1537                         }
1538
1539                         if(c->poll && !c->reapable) {
1540                                 c->poll(c, 0);
1541                         }
1542
1543                         return 0;
1544
1545                 case SYN_RECEIVED:
1546                         if(hdr.ctl & ACK) {
1547                                 return 0;
1548                         }
1549
1550                         // We haven't told the application about this connection yet. Silently delete.
1551                         free_connection(c);
1552                         return 0;
1553
1554                 case ESTABLISHED:
1555                 case FIN_WAIT_1:
1556                 case FIN_WAIT_2:
1557                 case CLOSE_WAIT:
1558                         if(hdr.ctl & ACK) {
1559                                 return 0;
1560                         }
1561
1562                         // The peer has aborted our connection.
1563                         set_state(c, CLOSED);
1564                         errno = ECONNRESET;
1565
1566                         if(c->recv) {
1567                                 c->recv(c, NULL, 0);
1568                         }
1569
1570                         if(c->poll && !c->reapable) {
1571                                 c->poll(c, 0);
1572                         }
1573
1574                         return 0;
1575
1576                 case CLOSING:
1577                 case LAST_ACK:
1578                 case TIME_WAIT:
1579                         if(hdr.ctl & ACK) {
1580                                 return 0;
1581                         }
1582
1583                         // As far as the application is concerned, the connection has already been closed.
1584                         // If it has called utcp_close() already, we can immediately free this connection.
1585                         if(c->reapable) {
1586                                 free_connection(c);
1587                                 return 0;
1588                         }
1589
1590                         // Otherwise, immediately move to the CLOSED state.
1591                         set_state(c, CLOSED);
1592                         return 0;
1593
1594                 default:
1595 #ifdef UTCP_DEBUG
1596                         abort();
1597 #endif
1598                         break;
1599                 }
1600         }
1601
1602         uint32_t advanced;
1603
1604         if(!(hdr.ctl & ACK)) {
1605                 advanced = 0;
1606                 goto skip_ack;
1607         }
1608
1609         // 3. Advance snd.una
1610
1611         if(seqdiff(hdr.ack, c->snd.last) > 0 || seqdiff(hdr.ack, c->snd.una) < 0) {
1612                 debug(c, "packet ack seqno out of range, %u <= %u < %u\n", c->snd.una, hdr.ack, c->snd.una + c->sndbuf.used);
1613                 goto reset;
1614         }
1615
1616         advanced = seqdiff(hdr.ack, c->snd.una);
1617
1618         if(advanced) {
1619                 // RTT measurement
1620                 if(c->rtt_start.tv_sec) {
1621                         if(c->rtt_seq == hdr.ack) {
1622                                 struct timespec now;
1623                                 clock_gettime(UTCP_CLOCK, &now);
1624                                 int32_t diff = timespec_diff_usec(&now, &c->rtt_start);
1625                                 update_rtt(c, diff);
1626                                 c->rtt_start.tv_sec = 0;
1627                         } else if(c->rtt_seq < hdr.ack) {
1628                                 debug(c, "cancelling RTT measurement: %u < %u\n", c->rtt_seq, hdr.ack);
1629                                 c->rtt_start.tv_sec = 0;
1630                         }
1631                 }
1632
1633                 int32_t data_acked = advanced;
1634
1635                 switch(c->state) {
1636                 case SYN_SENT:
1637                 case SYN_RECEIVED:
1638                         data_acked--;
1639                         break;
1640
1641                 // TODO: handle FIN as well.
1642                 default:
1643                         break;
1644                 }
1645
1646                 assert(data_acked >= 0);
1647
1648 #ifndef NDEBUG
1649                 int32_t bufused = seqdiff(c->snd.last, c->snd.una);
1650                 assert(data_acked <= bufused);
1651 #endif
1652
1653                 if(data_acked) {
1654                         buffer_discard(&c->sndbuf, data_acked);
1655
1656                         if(is_reliable(c)) {
1657                                 c->do_poll = true;
1658                         }
1659                 }
1660
1661                 // Also advance snd.nxt if possible
1662                 if(seqdiff(c->snd.nxt, hdr.ack) < 0) {
1663                         c->snd.nxt = hdr.ack;
1664                 }
1665
1666                 c->snd.una = hdr.ack;
1667
1668                 if(c->dupack) {
1669                         if(c->dupack >= 3) {
1670                                 debug(c, "fast recovery ended\n");
1671                                 c->snd.cwnd = c->snd.ssthresh;
1672                         }
1673
1674                         c->dupack = 0;
1675                 }
1676
1677                 // Increase the congestion window according to RFC 5681
1678                 if(c->snd.cwnd < c->snd.ssthresh) {
1679                         c->snd.cwnd += min(advanced, utcp->mss); // eq. 2
1680                 } else {
1681                         c->snd.cwnd += max(1, (utcp->mss * utcp->mss) / c->snd.cwnd); // eq. 3
1682                 }
1683
1684                 if(c->snd.cwnd > c->sndbuf.maxsize) {
1685                         c->snd.cwnd = c->sndbuf.maxsize;
1686                 }
1687
1688                 debug_cwnd(c);
1689
1690                 // Check if we have sent a FIN that is now ACKed.
1691                 switch(c->state) {
1692                 case FIN_WAIT_1:
1693                         if(c->snd.una == c->snd.last) {
1694                                 set_state(c, FIN_WAIT_2);
1695                         }
1696
1697                         break;
1698
1699                 case CLOSING:
1700                         if(c->snd.una == c->snd.last) {
1701                                 clock_gettime(UTCP_CLOCK, &c->conn_timeout);
1702                                 c->conn_timeout.tv_sec += utcp->timeout;
1703                                 set_state(c, TIME_WAIT);
1704                         }
1705
1706                         break;
1707
1708                 default:
1709                         break;
1710                 }
1711         } else {
1712                 if(!len && is_reliable(c) && c->snd.una != c->snd.last) {
1713                         c->dupack++;
1714                         debug(c, "duplicate ACK %d\n", c->dupack);
1715
1716                         if(c->dupack == 3) {
1717                                 // RFC 5681 fast recovery
1718                                 debug(c, "fast recovery started\n", c->dupack);
1719                                 uint32_t flightsize = seqdiff(c->snd.nxt, c->snd.una);
1720                                 c->snd.ssthresh = max(flightsize / 2, utcp->mss * 2); // eq. 4
1721                                 c->snd.cwnd = min(c->snd.ssthresh + 3 * utcp->mss, c->sndbuf.maxsize);
1722
1723                                 if(c->snd.cwnd > c->sndbuf.maxsize) {
1724                                         c->snd.cwnd = c->sndbuf.maxsize;
1725                                 }
1726
1727                                 debug_cwnd(c);
1728
1729                                 fast_retransmit(c);
1730                         } else if(c->dupack > 3) {
1731                                 c->snd.cwnd += utcp->mss;
1732
1733                                 if(c->snd.cwnd > c->sndbuf.maxsize) {
1734                                         c->snd.cwnd = c->sndbuf.maxsize;
1735                                 }
1736
1737                                 debug_cwnd(c);
1738                         }
1739
1740                         // We got an ACK which indicates the other side did get one of our packets.
1741                         // Reset the retransmission timer to avoid going to slow start,
1742                         // but don't touch the connection timeout.
1743                         start_retransmit_timer(c);
1744                 }
1745         }
1746
1747         // 4. Update timers
1748
1749         if(advanced) {
1750                 if(c->snd.una == c->snd.last) {
1751                         stop_retransmit_timer(c);
1752                         timespec_clear(&c->conn_timeout);
1753                 } else if(is_reliable(c)) {
1754                         start_retransmit_timer(c);
1755                         clock_gettime(UTCP_CLOCK, &c->conn_timeout);
1756                         c->conn_timeout.tv_sec += utcp->timeout;
1757                 }
1758         }
1759
1760 skip_ack:
1761         // 5. Process SYN stuff
1762
1763         if(hdr.ctl & SYN) {
1764                 switch(c->state) {
1765                 case SYN_SENT:
1766
1767                         // This is a SYNACK. It should always have ACKed the SYN.
1768                         if(!advanced) {
1769                                 goto reset;
1770                         }
1771
1772                         c->rcv.irs = hdr.seq;
1773                         c->rcv.nxt = hdr.seq + 1;
1774
1775                         if(c->shut_wr) {
1776                                 c->snd.last++;
1777                                 set_state(c, FIN_WAIT_1);
1778                         } else {
1779                                 c->do_poll = true;
1780                                 set_state(c, ESTABLISHED);
1781                         }
1782
1783                         break;
1784
1785                 case SYN_RECEIVED:
1786                         // This is a retransmit of a SYN, send back the SYNACK.
1787                         goto synack;
1788
1789                 case ESTABLISHED:
1790                 case FIN_WAIT_1:
1791                 case FIN_WAIT_2:
1792                 case CLOSE_WAIT:
1793                 case CLOSING:
1794                 case LAST_ACK:
1795                 case TIME_WAIT:
1796                         // This could be a retransmission. Ignore the SYN flag, but send an ACK back.
1797                         break;
1798
1799                 default:
1800 #ifdef UTCP_DEBUG
1801                         abort();
1802 #endif
1803                         return 0;
1804                 }
1805         }
1806
1807         // 6. Process new data
1808
1809         if(c->state == SYN_RECEIVED) {
1810                 // This is the ACK after the SYNACK. It should always have ACKed the SYNACK.
1811                 if(!advanced) {
1812                         goto reset;
1813                 }
1814
1815                 // Are we still LISTENing?
1816                 if(utcp->accept) {
1817                         utcp->accept(c, c->src);
1818                 }
1819
1820                 if(c->state != ESTABLISHED) {
1821                         set_state(c, CLOSED);
1822                         c->reapable = true;
1823                         goto reset;
1824                 }
1825         }
1826
1827         if(len) {
1828                 switch(c->state) {
1829                 case SYN_SENT:
1830                 case SYN_RECEIVED:
1831                         // This should never happen.
1832 #ifdef UTCP_DEBUG
1833                         abort();
1834 #endif
1835                         return 0;
1836
1837                 case ESTABLISHED:
1838                         break;
1839
1840                 case FIN_WAIT_1:
1841                 case FIN_WAIT_2:
1842                         if(c->reapable) {
1843                                 // We already closed the connection and are not interested in more data.
1844                                 goto reset;
1845                         }
1846
1847                         break;
1848
1849                 case CLOSE_WAIT:
1850                 case CLOSING:
1851                 case LAST_ACK:
1852                 case TIME_WAIT:
1853                         // Ehm no, We should never receive more data after a FIN.
1854                         goto reset;
1855
1856                 default:
1857 #ifdef UTCP_DEBUG
1858                         abort();
1859 #endif
1860                         return 0;
1861                 }
1862
1863                 handle_incoming_data(c, &hdr, ptr, len);
1864         }
1865
1866         // 7. Process FIN stuff
1867
1868         if((hdr.ctl & FIN) && (!is_reliable(c) || hdr.seq + len == c->rcv.nxt)) {
1869                 switch(c->state) {
1870                 case SYN_SENT:
1871                 case SYN_RECEIVED:
1872                         // This should never happen.
1873 #ifdef UTCP_DEBUG
1874                         abort();
1875 #endif
1876                         break;
1877
1878                 case ESTABLISHED:
1879                         set_state(c, CLOSE_WAIT);
1880                         break;
1881
1882                 case FIN_WAIT_1:
1883                         set_state(c, CLOSING);
1884                         break;
1885
1886                 case FIN_WAIT_2:
1887                         clock_gettime(UTCP_CLOCK, &c->conn_timeout);
1888                         c->conn_timeout.tv_sec += utcp->timeout;
1889                         set_state(c, TIME_WAIT);
1890                         break;
1891
1892                 case CLOSE_WAIT:
1893                 case CLOSING:
1894                 case LAST_ACK:
1895                 case TIME_WAIT:
1896                         // Ehm, no. We should never receive a second FIN.
1897                         goto reset;
1898
1899                 default:
1900 #ifdef UTCP_DEBUG
1901                         abort();
1902 #endif
1903                         break;
1904                 }
1905
1906                 // FIN counts as one sequence number
1907                 c->rcv.nxt++;
1908                 len++;
1909
1910                 // Inform the application that the peer closed its end of the connection.
1911                 if(c->recv) {
1912                         errno = 0;
1913                         c->recv(c, NULL, 0);
1914                 }
1915         }
1916
1917         // Now we send something back if:
1918         // - we received data, so we have to send back an ACK
1919         //   -> sendatleastone = true
1920         // - or we got an ack, so we should maybe send a bit more data
1921         //   -> sendatleastone = false
1922
1923         if(is_reliable(c) || hdr.ctl & SYN || hdr.ctl & FIN) {
1924                 ack(c, has_data);
1925         }
1926
1927         return 0;
1928
1929 reset:
1930         swap_ports(&hdr);
1931         hdr.wnd = 0;
1932         hdr.aux = 0;
1933
1934         if(hdr.ctl & ACK) {
1935                 hdr.seq = hdr.ack;
1936                 hdr.ctl = RST;
1937         } else {
1938                 hdr.ack = hdr.seq + len;
1939                 hdr.seq = 0;
1940                 hdr.ctl = RST | ACK;
1941         }
1942
1943         print_packet(c, "send", &hdr, sizeof(hdr));
1944         utcp->send(utcp, &hdr, sizeof(hdr));
1945         return 0;
1946
1947 }
1948
1949 int utcp_shutdown(struct utcp_connection *c, int dir) {
1950         debug(c, "shutdown %d at %u\n", dir, c ? c->snd.last : 0);
1951
1952         if(!c) {
1953                 errno = EFAULT;
1954                 return -1;
1955         }
1956
1957         if(c->reapable) {
1958                 debug(c, "shutdown() called on closed connection\n");
1959                 errno = EBADF;
1960                 return -1;
1961         }
1962
1963         if(!(dir == UTCP_SHUT_RD || dir == UTCP_SHUT_WR || dir == UTCP_SHUT_RDWR)) {
1964                 errno = EINVAL;
1965                 return -1;
1966         }
1967
1968         // TCP does not have a provision for stopping incoming packets.
1969         // The best we can do is to just ignore them.
1970         if(dir == UTCP_SHUT_RD || dir == UTCP_SHUT_RDWR) {
1971                 c->recv = NULL;
1972         }
1973
1974         // The rest of the code deals with shutting down writes.
1975         if(dir == UTCP_SHUT_RD) {
1976                 return 0;
1977         }
1978
1979         // Only process shutting down writes once.
1980         if(c->shut_wr) {
1981                 return 0;
1982         }
1983
1984         c->shut_wr = true;
1985
1986         switch(c->state) {
1987         case CLOSED:
1988         case LISTEN:
1989                 errno = ENOTCONN;
1990                 return -1;
1991
1992         case SYN_SENT:
1993                 return 0;
1994
1995         case SYN_RECEIVED:
1996         case ESTABLISHED:
1997                 set_state(c, FIN_WAIT_1);
1998                 break;
1999
2000         case FIN_WAIT_1:
2001         case FIN_WAIT_2:
2002                 return 0;
2003
2004         case CLOSE_WAIT:
2005                 set_state(c, CLOSING);
2006                 break;
2007
2008         case CLOSING:
2009         case LAST_ACK:
2010         case TIME_WAIT:
2011                 return 0;
2012         }
2013
2014         c->snd.last++;
2015
2016         ack(c, false);
2017
2018         if(!timespec_isset(&c->rtrx_timeout)) {
2019                 start_retransmit_timer(c);
2020         }
2021
2022         return 0;
2023 }
2024
2025 static bool reset_connection(struct utcp_connection *c) {
2026         if(!c) {
2027                 errno = EFAULT;
2028                 return false;
2029         }
2030
2031         if(c->reapable) {
2032                 debug(c, "abort() called on closed connection\n");
2033                 errno = EBADF;
2034                 return false;
2035         }
2036
2037         c->recv = NULL;
2038         c->poll = NULL;
2039
2040         switch(c->state) {
2041         case CLOSED:
2042                 return true;
2043
2044         case LISTEN:
2045         case SYN_SENT:
2046         case CLOSING:
2047         case LAST_ACK:
2048         case TIME_WAIT:
2049                 set_state(c, CLOSED);
2050                 return true;
2051
2052         case SYN_RECEIVED:
2053         case ESTABLISHED:
2054         case FIN_WAIT_1:
2055         case FIN_WAIT_2:
2056         case CLOSE_WAIT:
2057                 set_state(c, CLOSED);
2058                 break;
2059         }
2060
2061         // Send RST
2062
2063         struct hdr hdr;
2064
2065         hdr.src = c->src;
2066         hdr.dst = c->dst;
2067         hdr.seq = c->snd.nxt;
2068         hdr.ack = c->rcv.nxt;
2069         hdr.wnd = 0;
2070         hdr.ctl = RST;
2071
2072         print_packet(c, "send", &hdr, sizeof(hdr));
2073         c->utcp->send(c->utcp, &hdr, sizeof(hdr));
2074         return true;
2075 }
2076
2077 static void set_reapable(struct utcp_connection *c) {
2078         if(c->sndbuf.external) {
2079                 set_buffer_storage(&c->sndbuf, NULL, DEFAULT_MTU);
2080         }
2081
2082         if(c->rcvbuf.external) {
2083                 set_buffer_storage(&c->rcvbuf, NULL, DEFAULT_MTU);
2084         }
2085
2086         c->recv = NULL;
2087         c->poll = NULL;
2088         c->reapable = true;
2089 }
2090
2091 // Closes all the opened connections
2092 void utcp_abort_all_connections(struct utcp *utcp) {
2093         if(!utcp) {
2094                 errno = EINVAL;
2095                 return;
2096         }
2097
2098         for(int i = 0; i < utcp->nconnections; i++) {
2099                 struct utcp_connection *c = utcp->connections[i];
2100
2101                 if(c->reapable || c->state == CLOSED) {
2102                         continue;
2103                 }
2104
2105                 utcp_recv_t old_recv = c->recv;
2106                 utcp_poll_t old_poll = c->poll;
2107
2108                 utcp_abort(c);
2109
2110                 if(old_recv) {
2111                         errno = 0;
2112                         old_recv(c, NULL, 0);
2113                 }
2114
2115                 if(old_poll && !c->reapable) {
2116                         errno = 0;
2117                         old_poll(c, 0);
2118                 }
2119         }
2120
2121         return;
2122 }
2123
2124 int utcp_close(struct utcp_connection *c) {
2125         if(utcp_shutdown(c, SHUT_RDWR) && errno != ENOTCONN) {
2126                 return -1;
2127         }
2128
2129         set_reapable(c);
2130         return 0;
2131 }
2132
2133 int utcp_abort(struct utcp_connection *c) {
2134         if(!reset_connection(c)) {
2135                 return -1;
2136         }
2137
2138         set_reapable(c);
2139         return 0;
2140 }
2141
2142 /* Handle timeouts.
2143  * One call to this function will loop through all connections,
2144  * checking if something needs to be resent or not.
2145  * The return value is the time to the next timeout in milliseconds,
2146  * or maybe a negative value if the timeout is infinite.
2147  */
2148 struct timespec utcp_timeout(struct utcp *utcp) {
2149         struct timespec now;
2150         clock_gettime(UTCP_CLOCK, &now);
2151         struct timespec next = {now.tv_sec + 3600, now.tv_nsec};
2152
2153         for(int i = 0; i < utcp->nconnections; i++) {
2154                 struct utcp_connection *c = utcp->connections[i];
2155
2156                 if(!c) {
2157                         continue;
2158                 }
2159
2160                 // delete connections that have been utcp_close()d.
2161                 if(c->state == CLOSED) {
2162                         if(c->reapable) {
2163                                 debug(c, "reaping\n");
2164                                 free_connection(c);
2165                                 i--;
2166                         }
2167
2168                         continue;
2169                 }
2170
2171                 if(timespec_isset(&c->conn_timeout) && timespec_lt(&c->conn_timeout, &now)) {
2172                         errno = ETIMEDOUT;
2173                         c->state = CLOSED;
2174
2175                         if(c->recv) {
2176                                 c->recv(c, NULL, 0);
2177                         }
2178
2179                         if(c->poll && !c->reapable) {
2180                                 c->poll(c, 0);
2181                         }
2182
2183                         continue;
2184                 }
2185
2186                 if(timespec_isset(&c->rtrx_timeout) && timespec_lt(&c->rtrx_timeout, &now)) {
2187                         debug(c, "retransmitting after timeout\n");
2188                         retransmit(c);
2189                 }
2190
2191                 if(c->poll) {
2192                         if((c->state == ESTABLISHED || c->state == CLOSE_WAIT) && c->do_poll) {
2193                                 c->do_poll = false;
2194                                 uint32_t len = buffer_free(&c->sndbuf);
2195
2196                                 if(len) {
2197                                         c->poll(c, len);
2198                                 }
2199                         } else if(c->state == CLOSED) {
2200                                 c->poll(c, 0);
2201                         }
2202                 }
2203
2204                 if(timespec_isset(&c->conn_timeout) && timespec_lt(&c->conn_timeout, &next)) {
2205                         next = c->conn_timeout;
2206                 }
2207
2208                 if(timespec_isset(&c->rtrx_timeout) && timespec_lt(&c->rtrx_timeout, &next)) {
2209                         next = c->rtrx_timeout;
2210                 }
2211         }
2212
2213         struct timespec diff;
2214
2215         timespec_sub(&next, &now, &diff);
2216
2217         return diff;
2218 }
2219
2220 bool utcp_is_active(struct utcp *utcp) {
2221         if(!utcp) {
2222                 return false;
2223         }
2224
2225         for(int i = 0; i < utcp->nconnections; i++)
2226                 if(utcp->connections[i]->state != CLOSED && utcp->connections[i]->state != TIME_WAIT) {
2227                         return true;
2228                 }
2229
2230         return false;
2231 }
2232
2233 struct utcp *utcp_init(utcp_accept_t accept, utcp_listen_t listen, utcp_send_t send, void *priv) {
2234         if(!send) {
2235                 errno = EFAULT;
2236                 return NULL;
2237         }
2238
2239         struct utcp *utcp = calloc(1, sizeof(*utcp));
2240
2241         if(!utcp) {
2242                 return NULL;
2243         }
2244
2245         utcp_set_mtu(utcp, DEFAULT_MTU);
2246
2247         if(!utcp->pkt) {
2248                 free(utcp);
2249                 return NULL;
2250         }
2251
2252         if(!CLOCK_GRANULARITY) {
2253                 struct timespec res;
2254                 clock_getres(UTCP_CLOCK, &res);
2255                 CLOCK_GRANULARITY = res.tv_sec * USEC_PER_SEC + res.tv_nsec / 1000;
2256         }
2257
2258         utcp->accept = accept;
2259         utcp->listen = listen;
2260         utcp->send = send;
2261         utcp->priv = priv;
2262         utcp->timeout = DEFAULT_USER_TIMEOUT; // sec
2263
2264         return utcp;
2265 }
2266
2267 void utcp_exit(struct utcp *utcp) {
2268         if(!utcp) {
2269                 return;
2270         }
2271
2272         for(int i = 0; i < utcp->nconnections; i++) {
2273                 struct utcp_connection *c = utcp->connections[i];
2274
2275                 if(!c->reapable) {
2276                         if(c->recv) {
2277                                 c->recv(c, NULL, 0);
2278                         }
2279
2280                         if(c->poll && !c->reapable) {
2281                                 c->poll(c, 0);
2282                         }
2283                 }
2284
2285                 buffer_exit(&c->rcvbuf);
2286                 buffer_exit(&c->sndbuf);
2287                 free(c);
2288         }
2289
2290         free(utcp->connections);
2291         free(utcp->pkt);
2292         free(utcp);
2293 }
2294
2295 uint16_t utcp_get_mtu(struct utcp *utcp) {
2296         return utcp ? utcp->mtu : 0;
2297 }
2298
2299 uint16_t utcp_get_mss(struct utcp *utcp) {
2300         return utcp ? utcp->mss : 0;
2301 }
2302
2303 void utcp_set_mtu(struct utcp *utcp, uint16_t mtu) {
2304         if(!utcp) {
2305                 return;
2306         }
2307
2308         if(mtu <= sizeof(struct hdr)) {
2309                 return;
2310         }
2311
2312         if(mtu > utcp->mtu) {
2313                 char *new = realloc(utcp->pkt, mtu + sizeof(struct hdr));
2314
2315                 if(!new) {
2316                         return;
2317                 }
2318
2319                 utcp->pkt = new;
2320         }
2321
2322         utcp->mtu = mtu;
2323         utcp->mss = mtu - sizeof(struct hdr);
2324 }
2325
2326 void utcp_reset_timers(struct utcp *utcp) {
2327         if(!utcp) {
2328                 return;
2329         }
2330
2331         struct timespec now, then;
2332
2333         clock_gettime(UTCP_CLOCK, &now);
2334
2335         then = now;
2336
2337         then.tv_sec += utcp->timeout;
2338
2339         for(int i = 0; i < utcp->nconnections; i++) {
2340                 struct utcp_connection *c = utcp->connections[i];
2341
2342                 if(c->reapable) {
2343                         continue;
2344                 }
2345
2346                 if(timespec_isset(&c->rtrx_timeout)) {
2347                         c->rtrx_timeout = now;
2348                 }
2349
2350                 if(timespec_isset(&c->conn_timeout)) {
2351                         c->conn_timeout = then;
2352                 }
2353
2354                 c->rtt_start.tv_sec = 0;
2355
2356                 if(c->rto > START_RTO) {
2357                         c->rto = START_RTO;
2358                 }
2359         }
2360 }
2361
2362 int utcp_get_user_timeout(struct utcp *u) {
2363         return u ? u->timeout : 0;
2364 }
2365
2366 void utcp_set_user_timeout(struct utcp *u, int timeout) {
2367         if(u) {
2368                 u->timeout = timeout;
2369         }
2370 }
2371
2372 size_t utcp_get_sndbuf(struct utcp_connection *c) {
2373         return c ? c->sndbuf.maxsize : 0;
2374 }
2375
2376 size_t utcp_get_sndbuf_free(struct utcp_connection *c) {
2377         if(!c) {
2378                 return 0;
2379         }
2380
2381         switch(c->state) {
2382         case SYN_SENT:
2383         case SYN_RECEIVED:
2384         case ESTABLISHED:
2385         case CLOSE_WAIT:
2386                 return buffer_free(&c->sndbuf);
2387
2388         default:
2389                 return 0;
2390         }
2391 }
2392
2393 void utcp_set_sndbuf(struct utcp_connection *c, void *data, size_t size) {
2394         if(!c) {
2395                 return;
2396         }
2397
2398         set_buffer_storage(&c->sndbuf, data, size);
2399
2400         c->do_poll = is_reliable(c) && buffer_free(&c->sndbuf);
2401 }
2402
2403 size_t utcp_get_rcvbuf(struct utcp_connection *c) {
2404         return c ? c->rcvbuf.maxsize : 0;
2405 }
2406
2407 size_t utcp_get_rcvbuf_free(struct utcp_connection *c) {
2408         if(c && (c->state == ESTABLISHED || c->state == CLOSE_WAIT)) {
2409                 return buffer_free(&c->rcvbuf);
2410         } else {
2411                 return 0;
2412         }
2413 }
2414
2415 void utcp_set_rcvbuf(struct utcp_connection *c, void *data, size_t size) {
2416         if(!c) {
2417                 return;
2418         }
2419
2420         set_buffer_storage(&c->rcvbuf, data, size);
2421 }
2422
2423 size_t utcp_get_sendq(struct utcp_connection *c) {
2424         return c->sndbuf.used;
2425 }
2426
2427 size_t utcp_get_recvq(struct utcp_connection *c) {
2428         return c->rcvbuf.used;
2429 }
2430
2431 bool utcp_get_nodelay(struct utcp_connection *c) {
2432         return c ? c->nodelay : false;
2433 }
2434
2435 void utcp_set_nodelay(struct utcp_connection *c, bool nodelay) {
2436         if(c) {
2437                 c->nodelay = nodelay;
2438         }
2439 }
2440
2441 bool utcp_get_keepalive(struct utcp_connection *c) {
2442         return c ? c->keepalive : false;
2443 }
2444
2445 void utcp_set_keepalive(struct utcp_connection *c, bool keepalive) {
2446         if(c) {
2447                 c->keepalive = keepalive;
2448         }
2449 }
2450
2451 size_t utcp_get_outq(struct utcp_connection *c) {
2452         return c ? seqdiff(c->snd.nxt, c->snd.una) : 0;
2453 }
2454
2455 void utcp_set_recv_cb(struct utcp_connection *c, utcp_recv_t recv) {
2456         if(c) {
2457                 c->recv = recv;
2458         }
2459 }
2460
2461 void utcp_set_poll_cb(struct utcp_connection *c, utcp_poll_t poll) {
2462         if(c) {
2463                 c->poll = poll;
2464                 c->do_poll = is_reliable(c) && buffer_free(&c->sndbuf);
2465         }
2466 }
2467
2468 void utcp_set_accept_cb(struct utcp *utcp, utcp_accept_t accept, utcp_listen_t listen) {
2469         if(utcp) {
2470                 utcp->accept = accept;
2471                 utcp->listen = listen;
2472         }
2473 }
2474
2475 void utcp_expect_data(struct utcp_connection *c, bool expect) {
2476         if(!c || c->reapable) {
2477                 return;
2478         }
2479
2480         if(!(c->state == ESTABLISHED || c->state == FIN_WAIT_1 || c->state == FIN_WAIT_2)) {
2481                 return;
2482         }
2483
2484         if(expect) {
2485                 // If we expect data, start the connection timer.
2486                 if(!timespec_isset(&c->conn_timeout)) {
2487                         clock_gettime(UTCP_CLOCK, &c->conn_timeout);
2488                         c->conn_timeout.tv_sec += c->utcp->timeout;
2489                 }
2490         } else {
2491                 // If we want to cancel expecting data, only clear the timer when there is no unACKed data.
2492                 if(c->snd.una == c->snd.last) {
2493                         timespec_clear(&c->conn_timeout);
2494                 }
2495         }
2496 }
2497
2498 void utcp_set_flags(struct utcp_connection *c, uint32_t flags) {
2499         c->flags &= ~UTCP_CHANGEABLE_FLAGS;
2500         c->flags |= flags & UTCP_CHANGEABLE_FLAGS;
2501 }
2502
2503 void utcp_offline(struct utcp *utcp, bool offline) {
2504         struct timespec now;
2505         clock_gettime(UTCP_CLOCK, &now);
2506
2507         for(int i = 0; i < utcp->nconnections; i++) {
2508                 struct utcp_connection *c = utcp->connections[i];
2509
2510                 if(c->reapable) {
2511                         continue;
2512                 }
2513
2514                 utcp_expect_data(c, offline);
2515
2516                 if(!offline) {
2517                         if(timespec_isset(&c->rtrx_timeout)) {
2518                                 c->rtrx_timeout = now;
2519                         }
2520
2521                         utcp->connections[i]->rtt_start.tv_sec = 0;
2522
2523                         if(c->rto > START_RTO) {
2524                                 c->rto = START_RTO;
2525                         }
2526                 }
2527         }
2528 }
2529
2530 void utcp_set_retransmit_cb(struct utcp *utcp, utcp_retransmit_t cb) {
2531         utcp->retransmit = cb;
2532 }
2533
2534 void utcp_set_clock_granularity(long granularity) {
2535         CLOCK_GRANULARITY = granularity;
2536 }