2
0

quic_txp.c 112 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155
  1. /*
  2. * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the Apache License 2.0 (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. #include "internal/quic_txp.h"
  10. #include "internal/quic_fifd.h"
  11. #include "internal/quic_stream_map.h"
  12. #include "internal/quic_error.h"
  13. #include "internal/common.h"
  14. #include <openssl/err.h>
  15. #define MIN_CRYPTO_HDR_SIZE 3
  16. #define MIN_FRAME_SIZE_HANDSHAKE_DONE 1
  17. #define MIN_FRAME_SIZE_MAX_DATA 2
  18. #define MIN_FRAME_SIZE_ACK 5
  19. #define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1)
  20. #define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */
  21. #define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2
  22. #define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2
  23. /*
  24. * Packet Archetypes
  25. * =================
  26. */
  27. /* Generate normal packets containing most frame types, subject to EL. */
  28. #define TX_PACKETISER_ARCHETYPE_NORMAL 0
  29. /*
  30. * A probe packet is different in that:
  31. * - It bypasses CC, but *is* counted as in flight for purposes of CC;
  32. * - It must be ACK-eliciting.
  33. */
  34. #define TX_PACKETISER_ARCHETYPE_PROBE 1
  35. /*
  36. * An ACK-only packet is different in that:
  37. * - It bypasses CC, and is considered a 'non-inflight' packet;
  38. * - It may not contain anything other than an ACK frame, not even padding.
  39. */
  40. #define TX_PACKETISER_ARCHETYPE_ACK_ONLY 2
  41. #define TX_PACKETISER_ARCHETYPE_NUM 3
  42. struct ossl_quic_tx_packetiser_st {
  43. OSSL_QUIC_TX_PACKETISER_ARGS args;
  44. /*
  45. * Opaque initial token blob provided by caller. TXP frees using the
  46. * callback when it is no longer needed.
  47. */
  48. const unsigned char *initial_token;
  49. size_t initial_token_len;
  50. ossl_quic_initial_token_free_fn *initial_token_free_cb;
  51. void *initial_token_free_cb_arg;
  52. /* Subcomponents of the TXP that we own. */
  53. QUIC_FIFD fifd; /* QUIC Frame-in-Flight Dispatcher */
  54. /* Internal state. */
  55. uint64_t next_pn[QUIC_PN_SPACE_NUM]; /* Next PN to use in given PN space. */
  56. OSSL_TIME last_tx_time; /* Last time a packet was generated, or 0. */
  57. /* Internal state - frame (re)generation flags. */
  58. unsigned int want_handshake_done : 1;
  59. unsigned int want_max_data : 1;
  60. unsigned int want_max_streams_bidi : 1;
  61. unsigned int want_max_streams_uni : 1;
  62. /* Internal state - frame (re)generation flags - per PN space. */
  63. unsigned int want_ack : QUIC_PN_SPACE_NUM;
  64. unsigned int force_ack_eliciting : QUIC_PN_SPACE_NUM;
  65. /*
  66. * Internal state - connection close terminal state.
  67. * Once this is set, it is not unset unlike other want_ flags - we keep
  68. * sending it in every packet.
  69. */
  70. unsigned int want_conn_close : 1;
  71. /* Has the handshake been completed? */
  72. unsigned int handshake_complete : 1;
  73. OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame;
  74. /*
  75. * Counts of the number of bytes received and sent while in the closing
  76. * state.
  77. */
  78. uint64_t closing_bytes_recv;
  79. uint64_t closing_bytes_xmit;
  80. /* Internal state - packet assembly. */
  81. struct txp_el {
  82. unsigned char *scratch; /* scratch buffer for packet assembly */
  83. size_t scratch_len; /* number of bytes allocated for scratch */
  84. OSSL_QTX_IOVEC *iovec; /* scratch iovec array for use with QTX */
  85. size_t alloc_iovec; /* size of iovec array */
  86. } el[QUIC_ENC_LEVEL_NUM];
  87. /* Message callback related arguments */
  88. ossl_msg_cb msg_callback;
  89. void *msg_callback_arg;
  90. SSL *msg_callback_ssl;
  91. /* Callbacks. */
  92. void (*ack_tx_cb)(const OSSL_QUIC_FRAME_ACK *ack,
  93. uint32_t pn_space,
  94. void *arg);
  95. void *ack_tx_cb_arg;
  96. };
  97. /*
  98. * The TX helper records state used while generating frames into packets. It
  99. * enables serialization into the packet to be done "transactionally" where
  100. * serialization of a frame can be rolled back if it fails midway (e.g. if it
  101. * does not fit).
  102. */
  103. struct tx_helper {
  104. OSSL_QUIC_TX_PACKETISER *txp;
  105. /*
  106. * The Maximum Packet Payload Length in bytes. This is the amount of
  107. * space we have to generate frames into.
  108. */
  109. size_t max_ppl;
  110. /*
  111. * Number of bytes we have generated so far.
  112. */
  113. size_t bytes_appended;
  114. /*
  115. * Number of scratch bytes in txp->scratch we have used so far. Some iovecs
  116. * will reference this scratch buffer. When we need to use more of it (e.g.
  117. * when we need to put frame headers somewhere), we append to the scratch
  118. * buffer, resizing if necessary, and increase this accordingly.
  119. */
  120. size_t scratch_bytes;
  121. /*
  122. * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare
  123. * until reserve_allowed is set to 1. Currently this is always at most 1, as
  124. * a PING frame takes up one byte and this mechanism is only used to ensure
  125. * we can encode a PING frame if we have been asked to ensure a packet is
  126. * ACK-eliciting and we are unusure if we are going to add any other
  127. * ACK-eliciting frames before we reach our MaxPPL budget.
  128. */
  129. size_t reserve;
  130. /*
  131. * Number of iovecs we have currently appended. This is the number of
  132. * entries valid in txp->iovec.
  133. */
  134. size_t num_iovec;
  135. /* The EL this TX helper is being used for. */
  136. uint32_t enc_level;
  137. /*
  138. * Whether we are allowed to make use of the reserve bytes in our MaxPPL
  139. * budget. This is used to ensure we have room to append a PING frame later
  140. * if we need to. Once we know we will not need to append a PING frame, this
  141. * is set to 1.
  142. */
  143. unsigned int reserve_allowed : 1;
  144. /*
  145. * Set to 1 if we have appended a STREAM frame with an implicit length. If
  146. * this happens we should never append another frame after that frame as it
  147. * cannot be validly encoded. This is just a safety check.
  148. */
  149. unsigned int done_implicit : 1;
  150. struct {
  151. /*
  152. * The fields in this structure are valid if active is set, which means
  153. * that a serialization transaction is currently in progress.
  154. */
  155. unsigned char *data;
  156. WPACKET wpkt;
  157. unsigned int active : 1;
  158. } txn;
  159. };
  160. static void tx_helper_rollback(struct tx_helper *h);
  161. static int txp_el_ensure_iovec(struct txp_el *el, size_t num);
  162. /* Initialises the TX helper. */
  163. static int tx_helper_init(struct tx_helper *h, OSSL_QUIC_TX_PACKETISER *txp,
  164. uint32_t enc_level, size_t max_ppl, size_t reserve)
  165. {
  166. if (reserve > max_ppl)
  167. return 0;
  168. h->txp = txp;
  169. h->enc_level = enc_level;
  170. h->max_ppl = max_ppl;
  171. h->reserve = reserve;
  172. h->num_iovec = 0;
  173. h->bytes_appended = 0;
  174. h->scratch_bytes = 0;
  175. h->reserve_allowed = 0;
  176. h->done_implicit = 0;
  177. h->txn.data = NULL;
  178. h->txn.active = 0;
  179. if (max_ppl > h->txp->el[enc_level].scratch_len) {
  180. unsigned char *scratch;
  181. scratch = OPENSSL_realloc(h->txp->el[enc_level].scratch, max_ppl);
  182. if (scratch == NULL)
  183. return 0;
  184. h->txp->el[enc_level].scratch = scratch;
  185. h->txp->el[enc_level].scratch_len = max_ppl;
  186. }
  187. return 1;
  188. }
  189. static void tx_helper_cleanup(struct tx_helper *h)
  190. {
  191. if (h->txn.active)
  192. tx_helper_rollback(h);
  193. h->txp = NULL;
  194. }
  195. static void tx_helper_unrestrict(struct tx_helper *h)
  196. {
  197. h->reserve_allowed = 1;
  198. }
  199. /*
  200. * Append an extent of memory to the iovec list. The memory must remain
  201. * allocated until we finish generating the packet and call the QTX.
  202. *
  203. * In general, the buffers passed to this function will be from one of two
  204. * ranges:
  205. *
  206. * - Application data contained in stream buffers managed elsewhere
  207. * in the QUIC stack; or
  208. *
  209. * - Control frame data appended into txp->scratch using tx_helper_begin and
  210. * tx_helper_commit.
  211. *
  212. */
  213. static int tx_helper_append_iovec(struct tx_helper *h,
  214. const unsigned char *buf,
  215. size_t buf_len)
  216. {
  217. struct txp_el *el = &h->txp->el[h->enc_level];
  218. if (buf_len == 0)
  219. return 1;
  220. if (!ossl_assert(!h->done_implicit))
  221. return 0;
  222. if (!txp_el_ensure_iovec(el, h->num_iovec + 1))
  223. return 0;
  224. el->iovec[h->num_iovec].buf = buf;
  225. el->iovec[h->num_iovec].buf_len = buf_len;
  226. ++h->num_iovec;
  227. h->bytes_appended += buf_len;
  228. return 1;
  229. }
  230. /*
  231. * How many more bytes of space do we have left in our plaintext packet payload?
  232. */
  233. static size_t tx_helper_get_space_left(struct tx_helper *h)
  234. {
  235. return h->max_ppl
  236. - (h->reserve_allowed ? 0 : h->reserve) - h->bytes_appended;
  237. }
  238. /*
  239. * Begin a control frame serialization transaction. This allows the
  240. * serialization of the control frame to be backed out if it turns out it won't
  241. * fit. Write the control frame to the returned WPACKET. Ensure you always
  242. * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns
  243. * NULL on failure.
  244. */
  245. static WPACKET *tx_helper_begin(struct tx_helper *h)
  246. {
  247. size_t space_left, len;
  248. unsigned char *data;
  249. struct txp_el *el = &h->txp->el[h->enc_level];
  250. if (!ossl_assert(!h->txn.active))
  251. return NULL;
  252. if (!ossl_assert(!h->done_implicit))
  253. return NULL;
  254. data = (unsigned char *)el->scratch + h->scratch_bytes;
  255. len = el->scratch_len - h->scratch_bytes;
  256. space_left = tx_helper_get_space_left(h);
  257. if (!ossl_assert(space_left <= len))
  258. return NULL;
  259. if (!WPACKET_init_static_len(&h->txn.wpkt, data, len, 0))
  260. return NULL;
  261. if (!WPACKET_set_max_size(&h->txn.wpkt, space_left)) {
  262. WPACKET_cleanup(&h->txn.wpkt);
  263. return NULL;
  264. }
  265. h->txn.data = data;
  266. h->txn.active = 1;
  267. return &h->txn.wpkt;
  268. }
  269. static void tx_helper_end(struct tx_helper *h, int success)
  270. {
  271. if (success)
  272. WPACKET_finish(&h->txn.wpkt);
  273. else
  274. WPACKET_cleanup(&h->txn.wpkt);
  275. h->txn.active = 0;
  276. h->txn.data = NULL;
  277. }
  278. /* Abort a control frame serialization transaction. */
  279. static void tx_helper_rollback(struct tx_helper *h)
  280. {
  281. if (!h->txn.active)
  282. return;
  283. tx_helper_end(h, 0);
  284. }
  285. /* Commit a control frame. */
  286. static int tx_helper_commit(struct tx_helper *h)
  287. {
  288. size_t l = 0;
  289. if (!h->txn.active)
  290. return 0;
  291. if (!WPACKET_get_total_written(&h->txn.wpkt, &l)) {
  292. tx_helper_end(h, 0);
  293. return 0;
  294. }
  295. if (!tx_helper_append_iovec(h, h->txn.data, l)) {
  296. tx_helper_end(h, 0);
  297. return 0;
  298. }
  299. if (h->txp->msg_callback != NULL && l > 0) {
  300. uint64_t ftype;
  301. int ctype = SSL3_RT_QUIC_FRAME_FULL;
  302. PACKET pkt;
  303. if (!PACKET_buf_init(&pkt, h->txn.data, l)
  304. || !ossl_quic_wire_peek_frame_header(&pkt, &ftype, NULL)) {
  305. tx_helper_end(h, 0);
  306. return 0;
  307. }
  308. if (ftype == OSSL_QUIC_FRAME_TYPE_PADDING)
  309. ctype = SSL3_RT_QUIC_FRAME_PADDING;
  310. else if (OSSL_QUIC_FRAME_TYPE_IS_STREAM(ftype)
  311. || ftype == OSSL_QUIC_FRAME_TYPE_CRYPTO)
  312. ctype = SSL3_RT_QUIC_FRAME_HEADER;
  313. h->txp->msg_callback(1, OSSL_QUIC1_VERSION, ctype, h->txn.data, l,
  314. h->txp->msg_callback_ssl,
  315. h->txp->msg_callback_arg);
  316. }
  317. h->scratch_bytes += l;
  318. tx_helper_end(h, 1);
  319. return 1;
  320. }
  321. struct archetype_data {
  322. unsigned int allow_ack : 1;
  323. unsigned int allow_ping : 1;
  324. unsigned int allow_crypto : 1;
  325. unsigned int allow_handshake_done : 1;
  326. unsigned int allow_path_challenge : 1;
  327. unsigned int allow_path_response : 1;
  328. unsigned int allow_new_conn_id : 1;
  329. unsigned int allow_retire_conn_id : 1;
  330. unsigned int allow_stream_rel : 1;
  331. unsigned int allow_conn_fc : 1;
  332. unsigned int allow_conn_close : 1;
  333. unsigned int allow_cfq_other : 1;
  334. unsigned int allow_new_token : 1;
  335. unsigned int allow_force_ack_eliciting : 1;
  336. unsigned int allow_padding : 1;
  337. unsigned int require_ack_eliciting : 1;
  338. unsigned int bypass_cc : 1;
  339. };
  340. struct txp_pkt_geom {
  341. size_t cmpl, cmppl, hwm, pkt_overhead;
  342. uint32_t archetype;
  343. struct archetype_data adata;
  344. };
  345. struct txp_pkt {
  346. struct tx_helper h;
  347. int h_valid;
  348. QUIC_TXPIM_PKT *tpkt;
  349. QUIC_STREAM *stream_head;
  350. QUIC_PKT_HDR phdr;
  351. struct txp_pkt_geom geom;
  352. int force_pad;
  353. };
  354. static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
  355. void *arg);
  356. static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
  357. QUIC_TXPIM_PKT *pkt, void *arg);
  358. static void on_confirm_notify(uint64_t frame_type, uint64_t stream_id,
  359. QUIC_TXPIM_PKT *pkt, void *arg);
  360. static void on_sstream_updated(uint64_t stream_id, void *arg);
  361. static int sstream_is_pending(QUIC_SSTREAM *sstream);
  362. static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER *txp,
  363. uint32_t enc_level,
  364. uint32_t archetype,
  365. uint64_t cc_limit,
  366. uint32_t *conn_close_enc_level);
  367. static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp);
  368. static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
  369. size_t pl,
  370. uint32_t enc_level,
  371. size_t hdr_len,
  372. size_t *r);
  373. static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp);
  374. static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp,
  375. struct txp_pkt *pkt,
  376. int chosen_for_conn_close);
  377. static int txp_pkt_init(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp,
  378. uint32_t enc_level, uint32_t archetype,
  379. size_t running_total);
  380. static void txp_pkt_cleanup(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp);
  381. static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt *pkt,
  382. OSSL_QUIC_TX_PACKETISER *txp);
  383. static int txp_pkt_append_padding(struct txp_pkt *pkt,
  384. OSSL_QUIC_TX_PACKETISER *txp, size_t num_bytes);
  385. static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER *txp, struct txp_pkt *pkt,
  386. uint32_t archetype, int *txpim_pkt_reffed);
  387. static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER *txp,
  388. uint64_t cc_limit);
  389. OSSL_QUIC_TX_PACKETISER *ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS *args)
  390. {
  391. OSSL_QUIC_TX_PACKETISER *txp;
  392. if (args == NULL
  393. || args->qtx == NULL
  394. || args->txpim == NULL
  395. || args->cfq == NULL
  396. || args->ackm == NULL
  397. || args->qsm == NULL
  398. || args->conn_txfc == NULL
  399. || args->conn_rxfc == NULL
  400. || args->max_streams_bidi_rxfc == NULL
  401. || args->max_streams_uni_rxfc == NULL) {
  402. ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
  403. return NULL;
  404. }
  405. txp = OPENSSL_zalloc(sizeof(*txp));
  406. if (txp == NULL)
  407. return NULL;
  408. txp->args = *args;
  409. txp->last_tx_time = ossl_time_zero();
  410. if (!ossl_quic_fifd_init(&txp->fifd,
  411. txp->args.cfq, txp->args.ackm, txp->args.txpim,
  412. get_sstream_by_id, txp,
  413. on_regen_notify, txp,
  414. on_confirm_notify, txp,
  415. on_sstream_updated, txp,
  416. args->get_qlog_cb,
  417. args->get_qlog_cb_arg)) {
  418. OPENSSL_free(txp);
  419. return NULL;
  420. }
  421. return txp;
  422. }
  423. void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER *txp)
  424. {
  425. uint32_t enc_level;
  426. if (txp == NULL)
  427. return;
  428. ossl_quic_tx_packetiser_set_initial_token(txp, NULL, 0, NULL, NULL);
  429. ossl_quic_fifd_cleanup(&txp->fifd);
  430. OPENSSL_free(txp->conn_close_frame.reason);
  431. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  432. enc_level < QUIC_ENC_LEVEL_NUM;
  433. ++enc_level) {
  434. OPENSSL_free(txp->el[enc_level].iovec);
  435. OPENSSL_free(txp->el[enc_level].scratch);
  436. }
  437. OPENSSL_free(txp);
  438. }
  439. /*
  440. * Determine if an Initial packet token length is reasonable based on the
  441. * current MDPL, returning 1 if it is OK.
  442. *
  443. * The real PMTU to the peer could differ from our (pessimistic) understanding
  444. * of the PMTU, therefore it is possible we could receive an Initial token from
  445. * a server in a Retry packet which is bigger than the MDPL. In this case it is
  446. * impossible for us ever to make forward progress and we need to error out
  447. * and fail the connection attempt.
  448. *
  449. * The specific boundary condition is complex: for example, after the size of
  450. * the Initial token, there are the Initial packet header overheads and then
  451. * encryption/AEAD tag overheads. After that, the minimum room for frame data in
  452. * order to guarantee forward progress must be guaranteed. For example, a crypto
  453. * stream needs to always be able to serialize at least one byte in a CRYPTO
  454. * frame in order to make forward progress. Because the offset field of a CRYPTO
  455. * frame uses a variable-length integer, the number of bytes needed to ensure
  456. * this also varies.
  457. *
  458. * Rather than trying to get this boundary condition check actually right,
  459. * require a reasonable amount of slack to avoid pathological behaviours. (After
  460. * all, transmitting a CRYPTO stream one byte at a time is probably not
  461. * desirable anyway.)
  462. *
  463. * We choose 160 bytes as the required margin, which is double the rough
  464. * estimation of the minimum we would require to guarantee forward progress
  465. * under worst case packet overheads.
  466. */
  467. #define TXP_REQUIRED_TOKEN_MARGIN 160
  468. static int txp_check_token_len(size_t token_len, size_t mdpl)
  469. {
  470. if (token_len == 0)
  471. return 1;
  472. if (token_len >= mdpl)
  473. return 0;
  474. if (TXP_REQUIRED_TOKEN_MARGIN >= mdpl)
  475. /* (should not be possible because MDPL must be at least 1200) */
  476. return 0;
  477. if (token_len > mdpl - TXP_REQUIRED_TOKEN_MARGIN)
  478. return 0;
  479. return 1;
  480. }
  481. int ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER *txp,
  482. const unsigned char *token,
  483. size_t token_len,
  484. ossl_quic_initial_token_free_fn *free_cb,
  485. void *free_cb_arg)
  486. {
  487. if (!txp_check_token_len(token_len, txp_get_mdpl(txp)))
  488. return 0;
  489. if (txp->initial_token != NULL && txp->initial_token_free_cb != NULL)
  490. txp->initial_token_free_cb(txp->initial_token, txp->initial_token_len,
  491. txp->initial_token_free_cb_arg);
  492. txp->initial_token = token;
  493. txp->initial_token_len = token_len;
  494. txp->initial_token_free_cb = free_cb;
  495. txp->initial_token_free_cb_arg = free_cb_arg;
  496. return 1;
  497. }
  498. int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER *txp,
  499. const QUIC_CONN_ID *dcid)
  500. {
  501. if (dcid == NULL) {
  502. ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
  503. return 0;
  504. }
  505. txp->args.cur_dcid = *dcid;
  506. return 1;
  507. }
  508. int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER *txp,
  509. const QUIC_CONN_ID *scid)
  510. {
  511. if (scid == NULL) {
  512. ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
  513. return 0;
  514. }
  515. txp->args.cur_scid = *scid;
  516. return 1;
  517. }
  518. /* Change the destination L4 address the TXP uses to send datagrams. */
  519. int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER *txp,
  520. const BIO_ADDR *peer)
  521. {
  522. if (peer == NULL) {
  523. BIO_ADDR_clear(&txp->args.peer);
  524. return 1;
  525. }
  526. txp->args.peer = *peer;
  527. return 1;
  528. }
  529. void ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER *txp,
  530. void (*cb)(const OSSL_QUIC_FRAME_ACK *ack,
  531. uint32_t pn_space,
  532. void *arg),
  533. void *cb_arg)
  534. {
  535. txp->ack_tx_cb = cb;
  536. txp->ack_tx_cb_arg = cb_arg;
  537. }
  538. void ossl_quic_tx_packetiser_set_qlog_cb(OSSL_QUIC_TX_PACKETISER *txp,
  539. QLOG *(*get_qlog_cb)(void *arg),
  540. void *get_qlog_cb_arg)
  541. {
  542. ossl_quic_fifd_set_qlog_cb(&txp->fifd, get_qlog_cb, get_qlog_cb_arg);
  543. }
  544. int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER *txp,
  545. uint32_t enc_level)
  546. {
  547. if (enc_level >= QUIC_ENC_LEVEL_NUM) {
  548. ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT);
  549. return 0;
  550. }
  551. if (enc_level != QUIC_ENC_LEVEL_0RTT)
  552. txp->args.crypto[ossl_quic_enc_level_to_pn_space(enc_level)] = NULL;
  553. return 1;
  554. }
  555. void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER *txp)
  556. {
  557. txp->handshake_complete = 1;
  558. }
  559. void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER *txp)
  560. {
  561. txp->want_handshake_done = 1;
  562. }
  563. void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER *txp,
  564. uint32_t pn_space)
  565. {
  566. txp->force_ack_eliciting |= (1UL << pn_space);
  567. }
  568. void ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER *txp,
  569. uint32_t pn_space)
  570. {
  571. txp->want_ack |= (1UL << pn_space);
  572. }
  573. #define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */
  574. #define TXP_ERR_SUCCESS 1 /* Success */
  575. #define TXP_ERR_SPACE 2 /* Not enough room for another packet */
  576. #define TXP_ERR_INPUT 3 /* Invalid/malformed input */
  577. /*
  578. * Generates a datagram by polling the various ELs to determine if they want to
  579. * generate any frames, and generating a datagram which coalesces packets for
  580. * any ELs which do.
  581. */
  582. int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER *txp,
  583. QUIC_TXP_STATUS *status)
  584. {
  585. /*
  586. * Called to generate one or more datagrams, each containing one or more
  587. * packets.
  588. *
  589. * There are some tricky things to note here:
  590. *
  591. * - The TXP is only concerned with generating encrypted packets;
  592. * other packets use a different path.
  593. *
  594. * - Any datagram containing an Initial packet must have a payload length
  595. * (DPL) of at least 1200 bytes. This padding need not necessarily be
  596. * found in the Initial packet.
  597. *
  598. * - It is desirable to be able to coalesce an Initial packet
  599. * with a Handshake packet. Since, before generating the Handshake
  600. * packet, we do not know how long it will be, we cannot know the
  601. * correct amount of padding to ensure a DPL of at least 1200 bytes.
  602. * Thus this padding must added to the Handshake packet (or whatever
  603. * packet is the last in the datagram).
  604. *
  605. * - However, at the time that we generate the Initial packet,
  606. * we do not actually know for sure that we will be followed
  607. * in the datagram by another packet. For example, suppose we have
  608. * some queued data (e.g. crypto stream data for the HANDSHAKE EL)
  609. * it looks like we will want to send on the HANDSHAKE EL.
  610. * We could assume padding will be placed in the Handshake packet
  611. * subsequently and avoid adding any padding to the Initial packet
  612. * (which would leave no room for the Handshake packet in the
  613. * datagram).
  614. *
  615. * However, this is not actually a safe assumption. Suppose that we
  616. * are using a link with a MDPL of 1200 bytes, the minimum allowed by
  617. * QUIC. Suppose that the Initial packet consumes 1195 bytes in total.
  618. * Since it is not possible to fit a Handshake packet in just 5 bytes,
  619. * upon trying to add a Handshake packet after generating the Initial
  620. * packet, we will discover we have no room to fit it! This is not a
  621. * problem in itself as another datagram can be sent subsequently, but
  622. * it is a problem because we were counting to use that packet to hold
  623. * the essential padding. But if we have already finished encrypting
  624. * the Initial packet, we cannot go and add padding to it anymore.
  625. * This leaves us stuck.
  626. *
  627. * Because of this, we have to plan multiple packets simultaneously, such
  628. * that we can start generating a Handshake (or 0-RTT or 1-RTT, or so on)
  629. * packet while still having the option to go back and add padding to the
  630. * Initial packet if it turns out to be needed.
  631. *
  632. * Trying to predict ahead of time (e.g. during Initial packet generation)
  633. * whether we will successfully generate a subsequent packet is fraught with
  634. * error as it relies on a large number of variables:
  635. *
  636. * - Do we have room to fit a packet header? (Consider that due to
  637. * variable-length integer encoding this is highly variable and can even
  638. * depend on payload length due to a variable-length Length field.)
  639. *
  640. * - Can we fit even a single one of the frames we want to put in this
  641. * packet in the packet? (Each frame type has a bespoke encoding. While
  642. * our encodings of some frame types are adaptive based on the available
  643. * room - e.g. STREAM frames - ultimately all frame types have some
  644. * absolute minimum number of bytes to be successfully encoded. For
  645. * example, if after an Initial packet there is enough room to encode
  646. * only one byte of frame data, it is quite likely we can't send any of
  647. * the frames we wanted to send.) While this is not strictly a problem
  648. * because we could just fill the packet with padding frames, this is a
  649. * pointless packet and is wasteful.
  650. *
  651. * Thus we adopt a multi-phase architecture:
  652. *
  653. * 1. Archetype Selection: Determine desired packet archetype.
  654. *
  655. * 2. Packet Staging: Generation of packet information and packet payload
  656. * data (frame data) into staging areas.
  657. *
  658. * 3. Packet Adjustment: Adjustment of staged packets, adding padding to
  659. * the staged packets if needed.
  660. *
  661. * 4. Commit: The packets are sent to the QTX and recorded as having been
  662. * sent to the FIFM.
  663. *
  664. */
  665. int res = 0, rc;
  666. uint32_t archetype, enc_level;
  667. uint32_t conn_close_enc_level = QUIC_ENC_LEVEL_NUM;
  668. struct txp_pkt pkt[QUIC_ENC_LEVEL_NUM];
  669. size_t pkts_done = 0;
  670. uint64_t cc_limit = txp->args.cc_method->get_tx_allowance(txp->args.cc_data);
  671. int need_padding = 0, txpim_pkt_reffed;
  672. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  673. enc_level < QUIC_ENC_LEVEL_NUM;
  674. ++enc_level)
  675. pkt[enc_level].h_valid = 0;
  676. memset(status, 0, sizeof(*status));
  677. /*
  678. * Should not be needed, but a sanity check in case anyone else has been
  679. * using the QTX.
  680. */
  681. ossl_qtx_finish_dgram(txp->args.qtx);
  682. /* 1. Archetype Selection */
  683. archetype = txp_determine_archetype(txp, cc_limit);
  684. /* 2. Packet Staging */
  685. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  686. enc_level < QUIC_ENC_LEVEL_NUM;
  687. ++enc_level) {
  688. size_t running_total = (enc_level > QUIC_ENC_LEVEL_INITIAL)
  689. ? pkt[enc_level - 1].geom.hwm : 0;
  690. pkt[enc_level].geom.hwm = running_total;
  691. if (!txp_should_try_staging(txp, enc_level, archetype, cc_limit,
  692. &conn_close_enc_level))
  693. continue;
  694. if (!txp_pkt_init(&pkt[enc_level], txp, enc_level, archetype,
  695. running_total))
  696. /*
  697. * If this fails this is not a fatal error - it means the geometry
  698. * planning determined there was not enough space for another
  699. * packet. So just proceed with what we've already planned for.
  700. */
  701. break;
  702. rc = txp_generate_for_el(txp, &pkt[enc_level],
  703. conn_close_enc_level == enc_level);
  704. if (rc != TXP_ERR_SUCCESS)
  705. goto out;
  706. if (pkt[enc_level].force_pad)
  707. /*
  708. * txp_generate_for_el emitted a frame which forces packet padding.
  709. */
  710. need_padding = 1;
  711. pkt[enc_level].geom.hwm = running_total
  712. + pkt[enc_level].h.bytes_appended
  713. + pkt[enc_level].geom.pkt_overhead;
  714. }
  715. /* 3. Packet Adjustment */
  716. if (pkt[QUIC_ENC_LEVEL_INITIAL].h_valid
  717. && pkt[QUIC_ENC_LEVEL_INITIAL].h.bytes_appended > 0)
  718. /*
  719. * We have an Initial packet in this datagram, so we need to make sure
  720. * the total size of the datagram is adequate.
  721. */
  722. need_padding = 1;
  723. if (need_padding) {
  724. size_t total_dgram_size = 0;
  725. const size_t min_dpl = QUIC_MIN_INITIAL_DGRAM_LEN;
  726. uint32_t pad_el = QUIC_ENC_LEVEL_NUM;
  727. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  728. enc_level < QUIC_ENC_LEVEL_NUM;
  729. ++enc_level)
  730. if (pkt[enc_level].h_valid && pkt[enc_level].h.bytes_appended > 0) {
  731. if (pad_el == QUIC_ENC_LEVEL_NUM
  732. /*
  733. * We might not be able to add padding, for example if we
  734. * are using the ACK_ONLY archetype.
  735. */
  736. && pkt[enc_level].geom.adata.allow_padding
  737. && !pkt[enc_level].h.done_implicit)
  738. pad_el = enc_level;
  739. txp_pkt_postgen_update_pkt_overhead(&pkt[enc_level], txp);
  740. total_dgram_size += pkt[enc_level].geom.pkt_overhead
  741. + pkt[enc_level].h.bytes_appended;
  742. }
  743. if (pad_el != QUIC_ENC_LEVEL_NUM && total_dgram_size < min_dpl) {
  744. size_t deficit = min_dpl - total_dgram_size;
  745. if (!txp_pkt_append_padding(&pkt[pad_el], txp, deficit))
  746. goto out;
  747. total_dgram_size += deficit;
  748. /*
  749. * Padding frames make a packet ineligible for being a non-inflight
  750. * packet.
  751. */
  752. pkt[pad_el].tpkt->ackm_pkt.is_inflight = 1;
  753. }
  754. /*
  755. * If we have failed to make a datagram of adequate size, for example
  756. * because we have a padding requirement but are using the ACK_ONLY
  757. * archetype (because we are CC limited), which precludes us from
  758. * sending padding, give up on generating the datagram - there is
  759. * nothing we can do.
  760. */
  761. if (total_dgram_size < min_dpl) {
  762. res = 1;
  763. goto out;
  764. }
  765. }
  766. /* 4. Commit */
  767. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  768. enc_level < QUIC_ENC_LEVEL_NUM;
  769. ++enc_level) {
  770. if (!pkt[enc_level].h_valid)
  771. /* Did not attempt to generate a packet for this EL. */
  772. continue;
  773. if (pkt[enc_level].h.bytes_appended == 0)
  774. /* Nothing was generated for this EL, so skip. */
  775. continue;
  776. rc = txp_pkt_commit(txp, &pkt[enc_level], archetype,
  777. &txpim_pkt_reffed);
  778. if (rc) {
  779. status->sent_ack_eliciting
  780. = status->sent_ack_eliciting
  781. || pkt[enc_level].tpkt->ackm_pkt.is_ack_eliciting;
  782. if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE)
  783. status->sent_handshake
  784. = (pkt[enc_level].h_valid
  785. && pkt[enc_level].h.bytes_appended > 0);
  786. }
  787. if (txpim_pkt_reffed)
  788. pkt[enc_level].tpkt = NULL; /* don't free */
  789. if (!rc)
  790. goto out;
  791. ++pkts_done;
  792. }
  793. /* Flush & Cleanup */
  794. res = 1;
  795. out:
  796. ossl_qtx_finish_dgram(txp->args.qtx);
  797. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  798. enc_level < QUIC_ENC_LEVEL_NUM;
  799. ++enc_level)
  800. txp_pkt_cleanup(&pkt[enc_level], txp);
  801. status->sent_pkt = pkts_done;
  802. return res;
  803. }
  804. static const struct archetype_data archetypes[QUIC_ENC_LEVEL_NUM][TX_PACKETISER_ARCHETYPE_NUM] = {
  805. /* EL 0(INITIAL) */
  806. {
  807. /* EL 0(INITIAL) - Archetype 0(NORMAL) */
  808. {
  809. /*allow_ack =*/ 1,
  810. /*allow_ping =*/ 1,
  811. /*allow_crypto =*/ 1,
  812. /*allow_handshake_done =*/ 0,
  813. /*allow_path_challenge =*/ 0,
  814. /*allow_path_response =*/ 0,
  815. /*allow_new_conn_id =*/ 0,
  816. /*allow_retire_conn_id =*/ 0,
  817. /*allow_stream_rel =*/ 0,
  818. /*allow_conn_fc =*/ 0,
  819. /*allow_conn_close =*/ 1,
  820. /*allow_cfq_other =*/ 0,
  821. /*allow_new_token =*/ 0,
  822. /*allow_force_ack_eliciting =*/ 1,
  823. /*allow_padding =*/ 1,
  824. /*require_ack_eliciting =*/ 0,
  825. /*bypass_cc =*/ 0,
  826. },
  827. /* EL 0(INITIAL) - Archetype 1(PROBE) */
  828. {
  829. /*allow_ack =*/ 1,
  830. /*allow_ping =*/ 1,
  831. /*allow_crypto =*/ 1,
  832. /*allow_handshake_done =*/ 0,
  833. /*allow_path_challenge =*/ 0,
  834. /*allow_path_response =*/ 0,
  835. /*allow_new_conn_id =*/ 0,
  836. /*allow_retire_conn_id =*/ 0,
  837. /*allow_stream_rel =*/ 0,
  838. /*allow_conn_fc =*/ 0,
  839. /*allow_conn_close =*/ 1,
  840. /*allow_cfq_other =*/ 0,
  841. /*allow_new_token =*/ 0,
  842. /*allow_force_ack_eliciting =*/ 1,
  843. /*allow_padding =*/ 1,
  844. /*require_ack_eliciting =*/ 1,
  845. /*bypass_cc =*/ 1,
  846. },
  847. /* EL 0(INITIAL) - Archetype 2(ACK_ONLY) */
  848. {
  849. /*allow_ack =*/ 1,
  850. /*allow_ping =*/ 0,
  851. /*allow_crypto =*/ 0,
  852. /*allow_handshake_done =*/ 0,
  853. /*allow_path_challenge =*/ 0,
  854. /*allow_path_response =*/ 0,
  855. /*allow_new_conn_id =*/ 0,
  856. /*allow_retire_conn_id =*/ 0,
  857. /*allow_stream_rel =*/ 0,
  858. /*allow_conn_fc =*/ 0,
  859. /*allow_conn_close =*/ 0,
  860. /*allow_cfq_other =*/ 0,
  861. /*allow_new_token =*/ 0,
  862. /*allow_force_ack_eliciting =*/ 1,
  863. /*allow_padding =*/ 0,
  864. /*require_ack_eliciting =*/ 0,
  865. /*bypass_cc =*/ 1,
  866. },
  867. },
  868. /* EL 1(HANDSHAKE) */
  869. {
  870. /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */
  871. {
  872. /*allow_ack =*/ 1,
  873. /*allow_ping =*/ 1,
  874. /*allow_crypto =*/ 1,
  875. /*allow_handshake_done =*/ 0,
  876. /*allow_path_challenge =*/ 0,
  877. /*allow_path_response =*/ 0,
  878. /*allow_new_conn_id =*/ 0,
  879. /*allow_retire_conn_id =*/ 0,
  880. /*allow_stream_rel =*/ 0,
  881. /*allow_conn_fc =*/ 0,
  882. /*allow_conn_close =*/ 1,
  883. /*allow_cfq_other =*/ 0,
  884. /*allow_new_token =*/ 0,
  885. /*allow_force_ack_eliciting =*/ 1,
  886. /*allow_padding =*/ 1,
  887. /*require_ack_eliciting =*/ 0,
  888. /*bypass_cc =*/ 0,
  889. },
  890. /* EL 1(HANDSHAKE) - Archetype 1(PROBE) */
  891. {
  892. /*allow_ack =*/ 1,
  893. /*allow_ping =*/ 1,
  894. /*allow_crypto =*/ 1,
  895. /*allow_handshake_done =*/ 0,
  896. /*allow_path_challenge =*/ 0,
  897. /*allow_path_response =*/ 0,
  898. /*allow_new_conn_id =*/ 0,
  899. /*allow_retire_conn_id =*/ 0,
  900. /*allow_stream_rel =*/ 0,
  901. /*allow_conn_fc =*/ 0,
  902. /*allow_conn_close =*/ 1,
  903. /*allow_cfq_other =*/ 0,
  904. /*allow_new_token =*/ 0,
  905. /*allow_force_ack_eliciting =*/ 1,
  906. /*allow_padding =*/ 1,
  907. /*require_ack_eliciting =*/ 1,
  908. /*bypass_cc =*/ 1,
  909. },
  910. /* EL 1(HANDSHAKE) - Archetype 2(ACK_ONLY) */
  911. {
  912. /*allow_ack =*/ 1,
  913. /*allow_ping =*/ 0,
  914. /*allow_crypto =*/ 0,
  915. /*allow_handshake_done =*/ 0,
  916. /*allow_path_challenge =*/ 0,
  917. /*allow_path_response =*/ 0,
  918. /*allow_new_conn_id =*/ 0,
  919. /*allow_retire_conn_id =*/ 0,
  920. /*allow_stream_rel =*/ 0,
  921. /*allow_conn_fc =*/ 0,
  922. /*allow_conn_close =*/ 0,
  923. /*allow_cfq_other =*/ 0,
  924. /*allow_new_token =*/ 0,
  925. /*allow_force_ack_eliciting =*/ 1,
  926. /*allow_padding =*/ 0,
  927. /*require_ack_eliciting =*/ 0,
  928. /*bypass_cc =*/ 1,
  929. },
  930. },
  931. /* EL 2(0RTT) */
  932. {
  933. /* EL 2(0RTT) - Archetype 0(NORMAL) */
  934. {
  935. /*allow_ack =*/ 0,
  936. /*allow_ping =*/ 1,
  937. /*allow_crypto =*/ 0,
  938. /*allow_handshake_done =*/ 0,
  939. /*allow_path_challenge =*/ 0,
  940. /*allow_path_response =*/ 0,
  941. /*allow_new_conn_id =*/ 1,
  942. /*allow_retire_conn_id =*/ 1,
  943. /*allow_stream_rel =*/ 1,
  944. /*allow_conn_fc =*/ 1,
  945. /*allow_conn_close =*/ 1,
  946. /*allow_cfq_other =*/ 0,
  947. /*allow_new_token =*/ 0,
  948. /*allow_force_ack_eliciting =*/ 0,
  949. /*allow_padding =*/ 1,
  950. /*require_ack_eliciting =*/ 0,
  951. /*bypass_cc =*/ 0,
  952. },
  953. /* EL 2(0RTT) - Archetype 1(PROBE) */
  954. {
  955. /*allow_ack =*/ 0,
  956. /*allow_ping =*/ 1,
  957. /*allow_crypto =*/ 0,
  958. /*allow_handshake_done =*/ 0,
  959. /*allow_path_challenge =*/ 0,
  960. /*allow_path_response =*/ 0,
  961. /*allow_new_conn_id =*/ 1,
  962. /*allow_retire_conn_id =*/ 1,
  963. /*allow_stream_rel =*/ 1,
  964. /*allow_conn_fc =*/ 1,
  965. /*allow_conn_close =*/ 1,
  966. /*allow_cfq_other =*/ 0,
  967. /*allow_new_token =*/ 0,
  968. /*allow_force_ack_eliciting =*/ 0,
  969. /*allow_padding =*/ 1,
  970. /*require_ack_eliciting =*/ 1,
  971. /*bypass_cc =*/ 1,
  972. },
  973. /* EL 2(0RTT) - Archetype 2(ACK_ONLY) */
  974. {
  975. /*allow_ack =*/ 0,
  976. /*allow_ping =*/ 0,
  977. /*allow_crypto =*/ 0,
  978. /*allow_handshake_done =*/ 0,
  979. /*allow_path_challenge =*/ 0,
  980. /*allow_path_response =*/ 0,
  981. /*allow_new_conn_id =*/ 0,
  982. /*allow_retire_conn_id =*/ 0,
  983. /*allow_stream_rel =*/ 0,
  984. /*allow_conn_fc =*/ 0,
  985. /*allow_conn_close =*/ 0,
  986. /*allow_cfq_other =*/ 0,
  987. /*allow_new_token =*/ 0,
  988. /*allow_force_ack_eliciting =*/ 0,
  989. /*allow_padding =*/ 0,
  990. /*require_ack_eliciting =*/ 0,
  991. /*bypass_cc =*/ 1,
  992. },
  993. },
  994. /* EL 3(1RTT) */
  995. {
  996. /* EL 3(1RTT) - Archetype 0(NORMAL) */
  997. {
  998. /*allow_ack =*/ 1,
  999. /*allow_ping =*/ 1,
  1000. /*allow_crypto =*/ 1,
  1001. /*allow_handshake_done =*/ 1,
  1002. /*allow_path_challenge =*/ 0,
  1003. /*allow_path_response =*/ 1,
  1004. /*allow_new_conn_id =*/ 1,
  1005. /*allow_retire_conn_id =*/ 1,
  1006. /*allow_stream_rel =*/ 1,
  1007. /*allow_conn_fc =*/ 1,
  1008. /*allow_conn_close =*/ 1,
  1009. /*allow_cfq_other =*/ 1,
  1010. /*allow_new_token =*/ 1,
  1011. /*allow_force_ack_eliciting =*/ 1,
  1012. /*allow_padding =*/ 1,
  1013. /*require_ack_eliciting =*/ 0,
  1014. /*bypass_cc =*/ 0,
  1015. },
  1016. /* EL 3(1RTT) - Archetype 1(PROBE) */
  1017. {
  1018. /*allow_ack =*/ 1,
  1019. /*allow_ping =*/ 1,
  1020. /*allow_crypto =*/ 1,
  1021. /*allow_handshake_done =*/ 1,
  1022. /*allow_path_challenge =*/ 0,
  1023. /*allow_path_response =*/ 1,
  1024. /*allow_new_conn_id =*/ 1,
  1025. /*allow_retire_conn_id =*/ 1,
  1026. /*allow_stream_rel =*/ 1,
  1027. /*allow_conn_fc =*/ 1,
  1028. /*allow_conn_close =*/ 1,
  1029. /*allow_cfq_other =*/ 1,
  1030. /*allow_new_token =*/ 1,
  1031. /*allow_force_ack_eliciting =*/ 1,
  1032. /*allow_padding =*/ 1,
  1033. /*require_ack_eliciting =*/ 1,
  1034. /*bypass_cc =*/ 1,
  1035. },
  1036. /* EL 3(1RTT) - Archetype 2(ACK_ONLY) */
  1037. {
  1038. /*allow_ack =*/ 1,
  1039. /*allow_ping =*/ 0,
  1040. /*allow_crypto =*/ 0,
  1041. /*allow_handshake_done =*/ 0,
  1042. /*allow_path_challenge =*/ 0,
  1043. /*allow_path_response =*/ 0,
  1044. /*allow_new_conn_id =*/ 0,
  1045. /*allow_retire_conn_id =*/ 0,
  1046. /*allow_stream_rel =*/ 0,
  1047. /*allow_conn_fc =*/ 0,
  1048. /*allow_conn_close =*/ 0,
  1049. /*allow_cfq_other =*/ 0,
  1050. /*allow_new_token =*/ 0,
  1051. /*allow_force_ack_eliciting =*/ 1,
  1052. /*allow_padding =*/ 0,
  1053. /*require_ack_eliciting =*/ 0,
  1054. /*bypass_cc =*/ 1,
  1055. }
  1056. }
  1057. };
  1058. static int txp_get_archetype_data(uint32_t enc_level,
  1059. uint32_t archetype,
  1060. struct archetype_data *a)
  1061. {
  1062. if (enc_level >= QUIC_ENC_LEVEL_NUM
  1063. || archetype >= TX_PACKETISER_ARCHETYPE_NUM)
  1064. return 0;
  1065. /* No need to avoid copying this as it should not exceed one int in size. */
  1066. *a = archetypes[enc_level][archetype];
  1067. return 1;
  1068. }
  1069. static int txp_determine_geometry(OSSL_QUIC_TX_PACKETISER *txp,
  1070. uint32_t archetype,
  1071. uint32_t enc_level,
  1072. size_t running_total,
  1073. QUIC_PKT_HDR *phdr,
  1074. struct txp_pkt_geom *geom)
  1075. {
  1076. size_t mdpl, cmpl, hdr_len;
  1077. /* Get information about packet archetype. */
  1078. if (!txp_get_archetype_data(enc_level, archetype, &geom->adata))
  1079. return 0;
  1080. /* Assemble packet header. */
  1081. phdr->type = ossl_quic_enc_level_to_pkt_type(enc_level);
  1082. phdr->spin_bit = 0;
  1083. phdr->pn_len = txp_determine_pn_len(txp);
  1084. phdr->partial = 0;
  1085. phdr->fixed = 1;
  1086. phdr->reserved = 0;
  1087. phdr->version = QUIC_VERSION_1;
  1088. phdr->dst_conn_id = txp->args.cur_dcid;
  1089. phdr->src_conn_id = txp->args.cur_scid;
  1090. /*
  1091. * We need to know the length of the payload to get an accurate header
  1092. * length for non-1RTT packets, because the Length field found in
  1093. * Initial/Handshake/0-RTT packets uses a variable-length encoding. However,
  1094. * we don't have a good idea of the length of our payload, because the
  1095. * length of the payload depends on the room in the datagram after fitting
  1096. * the header, which depends on the size of the header.
  1097. *
  1098. * In general, it does not matter if a packet is slightly shorter (because
  1099. * e.g. we predicted use of a 2-byte length field, but ended up only needing
  1100. * a 1-byte length field). However this does matter for Initial packets
  1101. * which must be at least 1200 bytes, which is also the assumed default MTU;
  1102. * therefore in many cases Initial packets will be padded to 1200 bytes,
  1103. * which means if we overestimated the header size, we will be short by a
  1104. * few bytes and the server will ignore the packet for being too short. In
  1105. * this case, however, such packets always *will* be padded to meet 1200
  1106. * bytes, which requires a 2-byte length field, so we don't actually need to
  1107. * worry about this. Thus we estimate the header length assuming a 2-byte
  1108. * length field here, which should in practice work well in all cases.
  1109. */
  1110. phdr->len = OSSL_QUIC_VLINT_2B_MAX - phdr->pn_len;
  1111. if (enc_level == QUIC_ENC_LEVEL_INITIAL) {
  1112. phdr->token = txp->initial_token;
  1113. phdr->token_len = txp->initial_token_len;
  1114. } else {
  1115. phdr->token = NULL;
  1116. phdr->token_len = 0;
  1117. }
  1118. hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(phdr->dst_conn_id.id_len,
  1119. phdr);
  1120. if (hdr_len == 0)
  1121. return 0;
  1122. /* MDPL: Maximum datagram payload length. */
  1123. mdpl = txp_get_mdpl(txp);
  1124. /*
  1125. * CMPL: Maximum encoded packet size we can put into this datagram given any
  1126. * previous packets coalesced into it.
  1127. */
  1128. if (running_total > mdpl)
  1129. /* Should not be possible, but if it happens: */
  1130. cmpl = 0;
  1131. else
  1132. cmpl = mdpl - running_total;
  1133. /* CMPPL: Maximum amount we can put into the current packet payload */
  1134. if (!txp_determine_ppl_from_pl(txp, cmpl, enc_level, hdr_len, &geom->cmppl))
  1135. return 0;
  1136. geom->cmpl = cmpl;
  1137. geom->pkt_overhead = cmpl - geom->cmppl;
  1138. geom->archetype = archetype;
  1139. return 1;
  1140. }
  1141. static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER *txp,
  1142. uint64_t cc_limit)
  1143. {
  1144. OSSL_ACKM_PROBE_INFO *probe_info
  1145. = ossl_ackm_get0_probe_request(txp->args.ackm);
  1146. uint32_t pn_space;
  1147. /*
  1148. * If ACKM has requested probe generation (e.g. due to PTO), we generate a
  1149. * Probe-archetype packet. Actually, we determine archetype on a
  1150. * per-datagram basis, so if any EL wants a probe, do a pass in which
  1151. * we try and generate a probe (if needed) for all ELs.
  1152. */
  1153. if (probe_info->anti_deadlock_initial > 0
  1154. || probe_info->anti_deadlock_handshake > 0)
  1155. return TX_PACKETISER_ARCHETYPE_PROBE;
  1156. for (pn_space = QUIC_PN_SPACE_INITIAL;
  1157. pn_space < QUIC_PN_SPACE_NUM;
  1158. ++pn_space)
  1159. if (probe_info->pto[pn_space] > 0)
  1160. return TX_PACKETISER_ARCHETYPE_PROBE;
  1161. /*
  1162. * If we are out of CC budget, we cannot send a normal packet,
  1163. * but we can do an ACK-only packet (potentially, if we
  1164. * want to send an ACK).
  1165. */
  1166. if (cc_limit == 0)
  1167. return TX_PACKETISER_ARCHETYPE_ACK_ONLY;
  1168. /* All other packets. */
  1169. return TX_PACKETISER_ARCHETYPE_NORMAL;
  1170. }
  1171. static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER *txp,
  1172. uint32_t enc_level,
  1173. uint32_t archetype,
  1174. uint64_t cc_limit,
  1175. uint32_t *conn_close_enc_level)
  1176. {
  1177. struct archetype_data a;
  1178. uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  1179. QUIC_CFQ_ITEM *cfq_item;
  1180. if (!ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level))
  1181. return 0;
  1182. if (!txp_get_archetype_data(enc_level, archetype, &a))
  1183. return 0;
  1184. if (!a.bypass_cc && cc_limit == 0)
  1185. /* CC not allowing us to send. */
  1186. return 0;
  1187. /*
  1188. * We can produce CONNECTION_CLOSE frames on any EL in principle, which
  1189. * means we need to choose which EL we would prefer to use. After a
  1190. * connection is fully established we have only one provisioned EL and this
  1191. * is a non-issue. Where multiple ELs are provisioned, it is possible the
  1192. * peer does not have the keys for the EL yet, which suggests in general it
  1193. * is preferable to use the lowest EL which is still provisioned.
  1194. *
  1195. * However (RFC 9000 s. 10.2.3 & 12.5) we are also required to not send
  1196. * application CONNECTION_CLOSE frames in non-1-RTT ELs, so as to not
  1197. * potentially leak application data on a connection which has yet to be
  1198. * authenticated. Thus when we have an application CONNECTION_CLOSE frame
  1199. * queued and need to send it on a non-1-RTT EL, we have to convert it
  1200. * into a transport CONNECTION_CLOSE frame which contains no application
  1201. * data. Since this loses information, it suggests we should use the 1-RTT
  1202. * EL to avoid this if possible, even if a lower EL is also available.
  1203. *
  1204. * At the same time, just because we have the 1-RTT EL provisioned locally
  1205. * does not necessarily mean the peer does, for example if a handshake
  1206. * CRYPTO frame has been lost. It is fairly important that CONNECTION_CLOSE
  1207. * is signalled in a way we know our peer can decrypt, as we stop processing
  1208. * connection retransmission logic for real after connection close and
  1209. * simply 'blindly' retransmit the same CONNECTION_CLOSE frame.
  1210. *
  1211. * This is not a major concern for clients, since if a client has a 1-RTT EL
  1212. * provisioned the server is guaranteed to also have a 1-RTT EL provisioned.
  1213. *
  1214. * TODO(QUIC SERVER): Revisit this when server support is added.
  1215. */
  1216. if (*conn_close_enc_level > enc_level
  1217. && *conn_close_enc_level != QUIC_ENC_LEVEL_1RTT)
  1218. *conn_close_enc_level = enc_level;
  1219. /* Do we need to send a PTO probe? */
  1220. if (a.allow_force_ack_eliciting) {
  1221. OSSL_ACKM_PROBE_INFO *probe_info
  1222. = ossl_ackm_get0_probe_request(txp->args.ackm);
  1223. if ((enc_level == QUIC_ENC_LEVEL_INITIAL
  1224. && probe_info->anti_deadlock_initial > 0)
  1225. || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
  1226. && probe_info->anti_deadlock_handshake > 0)
  1227. || probe_info->pto[pn_space] > 0)
  1228. return 1;
  1229. }
  1230. /* Does the crypto stream for this EL want to produce anything? */
  1231. if (a.allow_crypto && sstream_is_pending(txp->args.crypto[pn_space]))
  1232. return 1;
  1233. /* Does the ACKM for this PN space want to produce anything? */
  1234. if (a.allow_ack && (ossl_ackm_is_ack_desired(txp->args.ackm, pn_space)
  1235. || (txp->want_ack & (1UL << pn_space)) != 0))
  1236. return 1;
  1237. /* Do we need to force emission of an ACK-eliciting packet? */
  1238. if (a.allow_force_ack_eliciting
  1239. && (txp->force_ack_eliciting & (1UL << pn_space)) != 0)
  1240. return 1;
  1241. /* Does the connection-level RXFC want to produce a frame? */
  1242. if (a.allow_conn_fc && (txp->want_max_data
  1243. || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0)))
  1244. return 1;
  1245. /* Do we want to produce a MAX_STREAMS frame? */
  1246. if (a.allow_conn_fc
  1247. && (txp->want_max_streams_bidi
  1248. || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc,
  1249. 0)
  1250. || txp->want_max_streams_uni
  1251. || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc,
  1252. 0)))
  1253. return 1;
  1254. /* Do we want to produce a HANDSHAKE_DONE frame? */
  1255. if (a.allow_handshake_done && txp->want_handshake_done)
  1256. return 1;
  1257. /* Do we want to produce a CONNECTION_CLOSE frame? */
  1258. if (a.allow_conn_close && txp->want_conn_close &&
  1259. *conn_close_enc_level == enc_level)
  1260. /*
  1261. * This is a bit of a special case since CONNECTION_CLOSE can appear in
  1262. * most packet types, and when we decide we want to send it this status
  1263. * isn't tied to a specific EL. So if we want to send it, we send it
  1264. * only on the lowest non-dropped EL.
  1265. */
  1266. return 1;
  1267. /* Does the CFQ have any frames queued for this PN space? */
  1268. if (enc_level != QUIC_ENC_LEVEL_0RTT)
  1269. for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
  1270. cfq_item != NULL;
  1271. cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
  1272. uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
  1273. switch (frame_type) {
  1274. case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
  1275. if (a.allow_new_conn_id)
  1276. return 1;
  1277. break;
  1278. case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
  1279. if (a.allow_retire_conn_id)
  1280. return 1;
  1281. break;
  1282. case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
  1283. if (a.allow_new_token)
  1284. return 1;
  1285. break;
  1286. case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE:
  1287. if (a.allow_path_response)
  1288. return 1;
  1289. break;
  1290. default:
  1291. if (a.allow_cfq_other)
  1292. return 1;
  1293. break;
  1294. }
  1295. }
  1296. if (a.allow_stream_rel && txp->handshake_complete) {
  1297. QUIC_STREAM_ITER it;
  1298. /* If there are any active streams, 0/1-RTT wants to produce a packet.
  1299. * Whether a stream is on the active list is required to be precise
  1300. * (i.e., a stream is never on the active list if we cannot produce a
  1301. * frame for it), and all stream-related frames are governed by
  1302. * a.allow_stream_rel (i.e., if we can send one type of stream-related
  1303. * frame, we can send any of them), so we don't need to inspect
  1304. * individual streams on the active list, just confirm that the active
  1305. * list is non-empty.
  1306. */
  1307. ossl_quic_stream_iter_init(&it, txp->args.qsm, 0);
  1308. if (it.stream != NULL)
  1309. return 1;
  1310. }
  1311. return 0;
  1312. }
  1313. static int sstream_is_pending(QUIC_SSTREAM *sstream)
  1314. {
  1315. OSSL_QUIC_FRAME_STREAM hdr;
  1316. OSSL_QTX_IOVEC iov[2];
  1317. size_t num_iov = OSSL_NELEM(iov);
  1318. return ossl_quic_sstream_get_stream_frame(sstream, 0, &hdr, iov, &num_iov);
  1319. }
  1320. /* Determine how many bytes we should use for the encoded PN. */
  1321. static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp)
  1322. {
  1323. return 4; /* TODO(QUIC FUTURE) */
  1324. }
  1325. /* Determine plaintext packet payload length from payload length. */
  1326. static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
  1327. size_t pl,
  1328. uint32_t enc_level,
  1329. size_t hdr_len,
  1330. size_t *r)
  1331. {
  1332. if (pl < hdr_len)
  1333. return 0;
  1334. pl -= hdr_len;
  1335. if (!ossl_qtx_calculate_plaintext_payload_len(txp->args.qtx, enc_level,
  1336. pl, &pl))
  1337. return 0;
  1338. *r = pl;
  1339. return 1;
  1340. }
  1341. static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp)
  1342. {
  1343. return ossl_qtx_get_mdpl(txp->args.qtx);
  1344. }
  1345. static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
  1346. void *arg)
  1347. {
  1348. OSSL_QUIC_TX_PACKETISER *txp = arg;
  1349. QUIC_STREAM *s;
  1350. if (stream_id == UINT64_MAX)
  1351. return txp->args.crypto[pn_space];
  1352. s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1353. if (s == NULL)
  1354. return NULL;
  1355. return s->sstream;
  1356. }
  1357. static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
  1358. QUIC_TXPIM_PKT *pkt, void *arg)
  1359. {
  1360. OSSL_QUIC_TX_PACKETISER *txp = arg;
  1361. switch (frame_type) {
  1362. case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE:
  1363. txp->want_handshake_done = 1;
  1364. break;
  1365. case OSSL_QUIC_FRAME_TYPE_MAX_DATA:
  1366. txp->want_max_data = 1;
  1367. break;
  1368. case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI:
  1369. txp->want_max_streams_bidi = 1;
  1370. break;
  1371. case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI:
  1372. txp->want_max_streams_uni = 1;
  1373. break;
  1374. case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN:
  1375. txp->want_ack |= (1UL << pkt->ackm_pkt.pkt_space);
  1376. break;
  1377. case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA:
  1378. {
  1379. QUIC_STREAM *s
  1380. = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1381. if (s == NULL)
  1382. return;
  1383. s->want_max_stream_data = 1;
  1384. ossl_quic_stream_map_update_state(txp->args.qsm, s);
  1385. }
  1386. break;
  1387. case OSSL_QUIC_FRAME_TYPE_STOP_SENDING:
  1388. {
  1389. QUIC_STREAM *s
  1390. = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1391. if (s == NULL)
  1392. return;
  1393. ossl_quic_stream_map_schedule_stop_sending(txp->args.qsm, s);
  1394. }
  1395. break;
  1396. case OSSL_QUIC_FRAME_TYPE_RESET_STREAM:
  1397. {
  1398. QUIC_STREAM *s
  1399. = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1400. if (s == NULL)
  1401. return;
  1402. s->want_reset_stream = 1;
  1403. ossl_quic_stream_map_update_state(txp->args.qsm, s);
  1404. }
  1405. break;
  1406. default:
  1407. assert(0);
  1408. break;
  1409. }
  1410. }
  1411. static int txp_need_ping(OSSL_QUIC_TX_PACKETISER *txp,
  1412. uint32_t pn_space,
  1413. const struct archetype_data *adata)
  1414. {
  1415. return adata->allow_ping
  1416. && (adata->require_ack_eliciting
  1417. || (txp->force_ack_eliciting & (1UL << pn_space)) != 0);
  1418. }
  1419. static int txp_pkt_init(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp,
  1420. uint32_t enc_level, uint32_t archetype,
  1421. size_t running_total)
  1422. {
  1423. uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  1424. if (!txp_determine_geometry(txp, archetype, enc_level,
  1425. running_total, &pkt->phdr, &pkt->geom))
  1426. return 0;
  1427. /*
  1428. * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for
  1429. * PING.
  1430. */
  1431. if (!tx_helper_init(&pkt->h, txp, enc_level,
  1432. pkt->geom.cmppl,
  1433. txp_need_ping(txp, pn_space, &pkt->geom.adata) ? 1 : 0))
  1434. return 0;
  1435. pkt->h_valid = 1;
  1436. pkt->tpkt = NULL;
  1437. pkt->stream_head = NULL;
  1438. pkt->force_pad = 0;
  1439. return 1;
  1440. }
  1441. static void txp_pkt_cleanup(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp)
  1442. {
  1443. if (!pkt->h_valid)
  1444. return;
  1445. tx_helper_cleanup(&pkt->h);
  1446. pkt->h_valid = 0;
  1447. if (pkt->tpkt != NULL) {
  1448. ossl_quic_txpim_pkt_release(txp->args.txpim, pkt->tpkt);
  1449. pkt->tpkt = NULL;
  1450. }
  1451. }
  1452. static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt *pkt,
  1453. OSSL_QUIC_TX_PACKETISER *txp)
  1454. {
  1455. /*
  1456. * After we have staged and generated our packets, but before we commit
  1457. * them, it is possible for the estimated packet overhead (packet header +
  1458. * AEAD tag size) to shrink slightly because we generated a short packet
  1459. * whose which can be represented in fewer bytes as a variable-length
  1460. * integer than we were (pessimistically) budgeting for. We need to account
  1461. * for this to ensure that we get our padding calculation exactly right.
  1462. *
  1463. * Update pkt_overhead to be accurate now that we know how much data is
  1464. * going in a packet.
  1465. */
  1466. size_t hdr_len, ciphertext_len;
  1467. if (pkt->h.enc_level == QUIC_ENC_LEVEL_INITIAL)
  1468. /*
  1469. * Don't update overheads for the INITIAL EL - we have not finished
  1470. * appending padding to it and would potentially miscalculate the
  1471. * correct padding if we now update the pkt_overhead field to switch to
  1472. * e.g. a 1-byte length field in the packet header. Since we are padding
  1473. * to QUIC_MIN_INITIAL_DGRAM_LEN which requires a 2-byte length field,
  1474. * this is guaranteed to be moot anyway. See comment in
  1475. * txp_determine_geometry for more information.
  1476. */
  1477. return 1;
  1478. if (!ossl_qtx_calculate_ciphertext_payload_len(txp->args.qtx, pkt->h.enc_level,
  1479. pkt->h.bytes_appended,
  1480. &ciphertext_len))
  1481. return 0;
  1482. pkt->phdr.len = ciphertext_len;
  1483. hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(pkt->phdr.dst_conn_id.id_len,
  1484. &pkt->phdr);
  1485. pkt->geom.pkt_overhead = hdr_len + ciphertext_len - pkt->h.bytes_appended;
  1486. return 1;
  1487. }
  1488. static void on_confirm_notify(uint64_t frame_type, uint64_t stream_id,
  1489. QUIC_TXPIM_PKT *pkt, void *arg)
  1490. {
  1491. OSSL_QUIC_TX_PACKETISER *txp = arg;
  1492. switch (frame_type) {
  1493. case OSSL_QUIC_FRAME_TYPE_STOP_SENDING:
  1494. {
  1495. QUIC_STREAM *s
  1496. = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1497. if (s == NULL)
  1498. return;
  1499. s->acked_stop_sending = 1;
  1500. ossl_quic_stream_map_update_state(txp->args.qsm, s);
  1501. }
  1502. break;
  1503. case OSSL_QUIC_FRAME_TYPE_RESET_STREAM:
  1504. {
  1505. QUIC_STREAM *s
  1506. = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1507. if (s == NULL)
  1508. return;
  1509. /*
  1510. * We must already be in RESET_SENT or RESET_RECVD if we are
  1511. * here, so we don't need to check state here.
  1512. */
  1513. ossl_quic_stream_map_notify_reset_stream_acked(txp->args.qsm, s);
  1514. ossl_quic_stream_map_update_state(txp->args.qsm, s);
  1515. }
  1516. break;
  1517. default:
  1518. assert(0);
  1519. break;
  1520. }
  1521. }
  1522. static int txp_pkt_append_padding(struct txp_pkt *pkt,
  1523. OSSL_QUIC_TX_PACKETISER *txp, size_t num_bytes)
  1524. {
  1525. WPACKET *wpkt;
  1526. if (num_bytes == 0)
  1527. return 1;
  1528. if (!ossl_assert(pkt->h_valid))
  1529. return 0;
  1530. if (!ossl_assert(pkt->tpkt != NULL))
  1531. return 0;
  1532. wpkt = tx_helper_begin(&pkt->h);
  1533. if (wpkt == NULL)
  1534. return 0;
  1535. if (!ossl_quic_wire_encode_padding(wpkt, num_bytes)) {
  1536. tx_helper_rollback(&pkt->h);
  1537. return 0;
  1538. }
  1539. if (!tx_helper_commit(&pkt->h))
  1540. return 0;
  1541. pkt->tpkt->ackm_pkt.num_bytes += num_bytes;
  1542. /* Cannot be non-inflight if we have a PADDING frame */
  1543. pkt->tpkt->ackm_pkt.is_inflight = 1;
  1544. return 1;
  1545. }
  1546. static void on_sstream_updated(uint64_t stream_id, void *arg)
  1547. {
  1548. OSSL_QUIC_TX_PACKETISER *txp = arg;
  1549. QUIC_STREAM *s;
  1550. s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
  1551. if (s == NULL)
  1552. return;
  1553. ossl_quic_stream_map_update_state(txp->args.qsm, s);
  1554. }
  1555. /*
  1556. * Returns 1 if we can send that many bytes in closing state, 0 otherwise.
  1557. * Also maintains the bytes sent state if it returns a success.
  1558. */
  1559. static int try_commit_conn_close(OSSL_QUIC_TX_PACKETISER *txp, size_t n)
  1560. {
  1561. int res;
  1562. /* We can always send the first connection close frame */
  1563. if (txp->closing_bytes_recv == 0)
  1564. return 1;
  1565. /*
  1566. * RFC 9000 s. 10.2.1 Closing Connection State:
  1567. * To avoid being used for an amplification attack, such
  1568. * endpoints MUST limit the cumulative size of packets it sends
  1569. * to three times the cumulative size of the packets that are
  1570. * received and attributed to the connection.
  1571. * and:
  1572. * An endpoint in the closing state MUST either discard packets
  1573. * received from an unvalidated address or limit the cumulative
  1574. * size of packets it sends to an unvalidated address to three
  1575. * times the size of packets it receives from that address.
  1576. */
  1577. res = txp->closing_bytes_xmit + n <= txp->closing_bytes_recv * 3;
  1578. /*
  1579. * Attribute the bytes to the connection, if we are allowed to send them
  1580. * and this isn't the first closing frame.
  1581. */
  1582. if (res && txp->closing_bytes_recv != 0)
  1583. txp->closing_bytes_xmit += n;
  1584. return res;
  1585. }
  1586. void ossl_quic_tx_packetiser_record_received_closing_bytes(
  1587. OSSL_QUIC_TX_PACKETISER *txp, size_t n)
  1588. {
  1589. txp->closing_bytes_recv += n;
  1590. }
  1591. static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER *txp,
  1592. struct txp_pkt *pkt,
  1593. int chosen_for_conn_close,
  1594. int *can_be_non_inflight)
  1595. {
  1596. const uint32_t enc_level = pkt->h.enc_level;
  1597. const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  1598. const struct archetype_data *a = &pkt->geom.adata;
  1599. QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
  1600. struct tx_helper *h = &pkt->h;
  1601. const OSSL_QUIC_FRAME_ACK *ack;
  1602. OSSL_QUIC_FRAME_ACK ack2;
  1603. tpkt->ackm_pkt.largest_acked = QUIC_PN_INVALID;
  1604. /* ACK Frames (Regenerate) */
  1605. if (a->allow_ack
  1606. && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_ACK
  1607. && (((txp->want_ack & (1UL << pn_space)) != 0)
  1608. || ossl_ackm_is_ack_desired(txp->args.ackm, pn_space))
  1609. && (ack = ossl_ackm_get_ack_frame(txp->args.ackm, pn_space)) != NULL) {
  1610. WPACKET *wpkt = tx_helper_begin(h);
  1611. if (wpkt == NULL)
  1612. return 0;
  1613. /* We do not currently support ECN */
  1614. ack2 = *ack;
  1615. ack2.ecn_present = 0;
  1616. if (ossl_quic_wire_encode_frame_ack(wpkt,
  1617. txp->args.ack_delay_exponent,
  1618. &ack2)) {
  1619. if (!tx_helper_commit(h))
  1620. return 0;
  1621. tpkt->had_ack_frame = 1;
  1622. if (ack->num_ack_ranges > 0)
  1623. tpkt->ackm_pkt.largest_acked = ack->ack_ranges[0].end;
  1624. if (txp->ack_tx_cb != NULL)
  1625. txp->ack_tx_cb(&ack2, pn_space, txp->ack_tx_cb_arg);
  1626. } else {
  1627. tx_helper_rollback(h);
  1628. }
  1629. }
  1630. /* CONNECTION_CLOSE Frames (Regenerate) */
  1631. if (a->allow_conn_close && txp->want_conn_close && chosen_for_conn_close) {
  1632. WPACKET *wpkt = tx_helper_begin(h);
  1633. OSSL_QUIC_FRAME_CONN_CLOSE f, *pf = &txp->conn_close_frame;
  1634. size_t l;
  1635. if (wpkt == NULL)
  1636. return 0;
  1637. /*
  1638. * Application CONNECTION_CLOSE frames may only be sent in the
  1639. * Application PN space, as otherwise they may be sent before a
  1640. * connection is authenticated and leak application data. Therefore, if
  1641. * we need to send a CONNECTION_CLOSE frame in another PN space and were
  1642. * given an application CONNECTION_CLOSE frame, convert it into a
  1643. * transport CONNECTION_CLOSE frame, removing any sensitive application
  1644. * data.
  1645. *
  1646. * RFC 9000 s. 10.2.3: "A CONNECTION_CLOSE of type 0x1d MUST be replaced
  1647. * by a CONNECTION_CLOSE of type 0x1c when sending the frame in Initial
  1648. * or Handshake packets. Otherwise, information about the application
  1649. * state might be revealed. Endpoints MUST clear the value of the Reason
  1650. * Phrase field and SHOULD use the APPLICATION_ERROR code when
  1651. * converting to a CONNECTION_CLOSE of type 0x1c."
  1652. */
  1653. if (pn_space != QUIC_PN_SPACE_APP && pf->is_app) {
  1654. pf = &f;
  1655. pf->is_app = 0;
  1656. pf->frame_type = 0;
  1657. pf->error_code = OSSL_QUIC_ERR_APPLICATION_ERROR;
  1658. pf->reason = NULL;
  1659. pf->reason_len = 0;
  1660. }
  1661. if (ossl_quic_wire_encode_frame_conn_close(wpkt, pf)
  1662. && WPACKET_get_total_written(wpkt, &l)
  1663. && try_commit_conn_close(txp, l)) {
  1664. if (!tx_helper_commit(h))
  1665. return 0;
  1666. tpkt->had_conn_close = 1;
  1667. *can_be_non_inflight = 0;
  1668. } else {
  1669. tx_helper_rollback(h);
  1670. }
  1671. }
  1672. return 1;
  1673. }
  1674. static int try_len(size_t space_left, size_t orig_len,
  1675. size_t base_hdr_len, size_t lenbytes,
  1676. uint64_t maxn, size_t *hdr_len, size_t *payload_len)
  1677. {
  1678. size_t n;
  1679. size_t maxn_ = maxn > SIZE_MAX ? SIZE_MAX : (size_t)maxn;
  1680. *hdr_len = base_hdr_len + lenbytes;
  1681. if (orig_len == 0 && space_left >= *hdr_len) {
  1682. *payload_len = 0;
  1683. return 1;
  1684. }
  1685. n = orig_len;
  1686. if (n > maxn_)
  1687. n = maxn_;
  1688. if (n + *hdr_len > space_left)
  1689. n = (space_left >= *hdr_len) ? space_left - *hdr_len : 0;
  1690. *payload_len = n;
  1691. return n > 0;
  1692. }
  1693. static int determine_len(size_t space_left, size_t orig_len,
  1694. size_t base_hdr_len,
  1695. uint64_t *hlen, uint64_t *len)
  1696. {
  1697. int ok = 0;
  1698. size_t chosen_payload_len = 0;
  1699. size_t chosen_hdr_len = 0;
  1700. size_t payload_len[4], hdr_len[4];
  1701. int i, valid[4] = {0};
  1702. valid[0] = try_len(space_left, orig_len, base_hdr_len,
  1703. 1, OSSL_QUIC_VLINT_1B_MAX,
  1704. &hdr_len[0], &payload_len[0]);
  1705. valid[1] = try_len(space_left, orig_len, base_hdr_len,
  1706. 2, OSSL_QUIC_VLINT_2B_MAX,
  1707. &hdr_len[1], &payload_len[1]);
  1708. valid[2] = try_len(space_left, orig_len, base_hdr_len,
  1709. 4, OSSL_QUIC_VLINT_4B_MAX,
  1710. &hdr_len[2], &payload_len[2]);
  1711. valid[3] = try_len(space_left, orig_len, base_hdr_len,
  1712. 8, OSSL_QUIC_VLINT_8B_MAX,
  1713. &hdr_len[3], &payload_len[3]);
  1714. for (i = OSSL_NELEM(valid) - 1; i >= 0; --i)
  1715. if (valid[i] && payload_len[i] >= chosen_payload_len) {
  1716. chosen_payload_len = payload_len[i];
  1717. chosen_hdr_len = hdr_len[i];
  1718. ok = 1;
  1719. }
  1720. *hlen = chosen_hdr_len;
  1721. *len = chosen_payload_len;
  1722. return ok;
  1723. }
  1724. /*
  1725. * Given a CRYPTO frame header with accurate chdr->len and a budget
  1726. * (space_left), try to find the optimal value of chdr->len to fill as much of
  1727. * the budget as possible. This is slightly hairy because larger values of
  1728. * chdr->len cause larger encoded sizes of the length field of the frame, which
  1729. * in turn mean less space available for payload data. We check all possible
  1730. * encodings and choose the optimal encoding.
  1731. */
  1732. static int determine_crypto_len(struct tx_helper *h,
  1733. OSSL_QUIC_FRAME_CRYPTO *chdr,
  1734. size_t space_left,
  1735. uint64_t *hlen,
  1736. uint64_t *len)
  1737. {
  1738. size_t orig_len;
  1739. size_t base_hdr_len; /* CRYPTO header length without length field */
  1740. if (chdr->len > SIZE_MAX)
  1741. return 0;
  1742. orig_len = (size_t)chdr->len;
  1743. chdr->len = 0;
  1744. base_hdr_len = ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr);
  1745. chdr->len = orig_len;
  1746. if (base_hdr_len == 0)
  1747. return 0;
  1748. --base_hdr_len;
  1749. return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
  1750. }
  1751. static int determine_stream_len(struct tx_helper *h,
  1752. OSSL_QUIC_FRAME_STREAM *shdr,
  1753. size_t space_left,
  1754. uint64_t *hlen,
  1755. uint64_t *len)
  1756. {
  1757. size_t orig_len;
  1758. size_t base_hdr_len; /* STREAM header length without length field */
  1759. if (shdr->len > SIZE_MAX)
  1760. return 0;
  1761. orig_len = (size_t)shdr->len;
  1762. shdr->len = 0;
  1763. base_hdr_len = ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr);
  1764. shdr->len = orig_len;
  1765. if (base_hdr_len == 0)
  1766. return 0;
  1767. if (shdr->has_explicit_len)
  1768. --base_hdr_len;
  1769. return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
  1770. }
  1771. static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER *txp,
  1772. struct txp_pkt *pkt,
  1773. int *have_ack_eliciting)
  1774. {
  1775. const uint32_t enc_level = pkt->h.enc_level;
  1776. const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  1777. QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
  1778. struct tx_helper *h = &pkt->h;
  1779. size_t num_stream_iovec;
  1780. OSSL_QUIC_FRAME_STREAM shdr = {0};
  1781. OSSL_QUIC_FRAME_CRYPTO chdr = {0};
  1782. OSSL_QTX_IOVEC iov[2];
  1783. uint64_t hdr_bytes;
  1784. WPACKET *wpkt;
  1785. QUIC_TXPIM_CHUNK chunk = {0};
  1786. size_t i, space_left;
  1787. for (i = 0;; ++i) {
  1788. space_left = tx_helper_get_space_left(h);
  1789. if (space_left < MIN_FRAME_SIZE_CRYPTO)
  1790. return 1; /* no point trying */
  1791. /* Do we have any CRYPTO data waiting? */
  1792. num_stream_iovec = OSSL_NELEM(iov);
  1793. if (!ossl_quic_sstream_get_stream_frame(txp->args.crypto[pn_space],
  1794. i, &shdr, iov,
  1795. &num_stream_iovec))
  1796. return 1; /* nothing to do */
  1797. /* Convert STREAM frame header to CRYPTO frame header */
  1798. chdr.offset = shdr.offset;
  1799. chdr.len = shdr.len;
  1800. if (chdr.len == 0)
  1801. return 1; /* nothing to do */
  1802. /* Find best fit (header length, payload length) combination. */
  1803. if (!determine_crypto_len(h, &chdr, space_left, &hdr_bytes,
  1804. &chdr.len))
  1805. return 1; /* can't fit anything */
  1806. /*
  1807. * Truncate IOVs to match our chosen length.
  1808. *
  1809. * The length cannot be more than SIZE_MAX because this length comes
  1810. * from our send stream buffer.
  1811. */
  1812. ossl_quic_sstream_adjust_iov((size_t)chdr.len, iov, num_stream_iovec);
  1813. /*
  1814. * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
  1815. * the stream data.)
  1816. */
  1817. if (!txp_el_ensure_iovec(&txp->el[enc_level], h->num_iovec + 3))
  1818. return 0; /* alloc error */
  1819. /* Encode the header. */
  1820. wpkt = tx_helper_begin(h);
  1821. if (wpkt == NULL)
  1822. return 0; /* alloc error */
  1823. if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt, &chdr)) {
  1824. tx_helper_rollback(h);
  1825. return 1; /* can't fit */
  1826. }
  1827. if (!tx_helper_commit(h))
  1828. return 0; /* alloc error */
  1829. /* Add payload iovecs to the helper (infallible). */
  1830. for (i = 0; i < num_stream_iovec; ++i)
  1831. tx_helper_append_iovec(h, iov[i].buf, iov[i].buf_len);
  1832. *have_ack_eliciting = 1;
  1833. tx_helper_unrestrict(h); /* no longer need PING */
  1834. /* Log chunk to TXPIM. */
  1835. chunk.stream_id = UINT64_MAX; /* crypto stream */
  1836. chunk.start = chdr.offset;
  1837. chunk.end = chdr.offset + chdr.len - 1;
  1838. chunk.has_fin = 0; /* Crypto stream never ends */
  1839. if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
  1840. return 0; /* alloc error */
  1841. }
  1842. }
  1843. struct chunk_info {
  1844. OSSL_QUIC_FRAME_STREAM shdr;
  1845. uint64_t orig_len;
  1846. OSSL_QTX_IOVEC iov[2];
  1847. size_t num_stream_iovec;
  1848. int valid;
  1849. };
  1850. static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER *txp,
  1851. struct tx_helper *h,
  1852. QUIC_SSTREAM *sstream,
  1853. QUIC_TXFC *stream_txfc,
  1854. size_t skip,
  1855. struct chunk_info *chunk,
  1856. uint64_t consumed)
  1857. {
  1858. uint64_t fc_credit, fc_swm, fc_limit;
  1859. chunk->num_stream_iovec = OSSL_NELEM(chunk->iov);
  1860. chunk->valid = ossl_quic_sstream_get_stream_frame(sstream, skip,
  1861. &chunk->shdr,
  1862. chunk->iov,
  1863. &chunk->num_stream_iovec);
  1864. if (!chunk->valid)
  1865. return 1;
  1866. if (!ossl_assert(chunk->shdr.len > 0 || chunk->shdr.is_fin))
  1867. /* Should only have 0-length chunk if FIN */
  1868. return 0;
  1869. chunk->orig_len = chunk->shdr.len;
  1870. /* Clamp according to connection and stream-level TXFC. */
  1871. fc_credit = ossl_quic_txfc_get_credit(stream_txfc, consumed);
  1872. fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
  1873. fc_limit = fc_swm + fc_credit;
  1874. if (chunk->shdr.len > 0 && chunk->shdr.offset + chunk->shdr.len > fc_limit) {
  1875. chunk->shdr.len = (fc_limit <= chunk->shdr.offset)
  1876. ? 0 : fc_limit - chunk->shdr.offset;
  1877. chunk->shdr.is_fin = 0;
  1878. }
  1879. if (chunk->shdr.len == 0 && !chunk->shdr.is_fin) {
  1880. /*
  1881. * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending
  1882. * order of offset we don't need to check any later chunks, so stop
  1883. * iterating here.
  1884. */
  1885. chunk->valid = 0;
  1886. return 1;
  1887. }
  1888. return 1;
  1889. }
  1890. /*
  1891. * Returns 0 on fatal error (e.g. allocation failure), 1 on success.
  1892. * *packet_full is set to 1 if there is no longer enough room for another STREAM
  1893. * frame.
  1894. */
  1895. static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER *txp,
  1896. struct txp_pkt *pkt,
  1897. uint64_t id,
  1898. QUIC_SSTREAM *sstream,
  1899. QUIC_TXFC *stream_txfc,
  1900. QUIC_STREAM *next_stream,
  1901. int *have_ack_eliciting,
  1902. int *packet_full,
  1903. uint64_t *new_credit_consumed,
  1904. uint64_t conn_consumed)
  1905. {
  1906. int rc = 0;
  1907. struct chunk_info chunks[2] = {0};
  1908. const uint32_t enc_level = pkt->h.enc_level;
  1909. QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
  1910. struct tx_helper *h = &pkt->h;
  1911. OSSL_QUIC_FRAME_STREAM *shdr;
  1912. WPACKET *wpkt;
  1913. QUIC_TXPIM_CHUNK chunk;
  1914. size_t i, j, space_left;
  1915. int can_fill_payload, use_explicit_len;
  1916. int could_have_following_chunk;
  1917. uint64_t orig_len;
  1918. uint64_t hdr_len_implicit, payload_len_implicit;
  1919. uint64_t hdr_len_explicit, payload_len_explicit;
  1920. uint64_t fc_swm, fc_new_hwm;
  1921. fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
  1922. fc_new_hwm = fc_swm;
  1923. /*
  1924. * Load the first two chunks if any offered by the send stream. We retrieve
  1925. * the next chunk in advance so we can determine if we need to send any more
  1926. * chunks from the same stream after this one, which is needed when
  1927. * determining when we can use an implicit length in a STREAM frame.
  1928. */
  1929. for (i = 0; i < 2; ++i) {
  1930. if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i, &chunks[i],
  1931. conn_consumed))
  1932. goto err;
  1933. if (i == 0 && !chunks[i].valid) {
  1934. /* No chunks, nothing to do. */
  1935. rc = 1;
  1936. goto err;
  1937. }
  1938. }
  1939. for (i = 0;; ++i) {
  1940. space_left = tx_helper_get_space_left(h);
  1941. if (!chunks[i % 2].valid) {
  1942. /* Out of chunks; we're done. */
  1943. rc = 1;
  1944. goto err;
  1945. }
  1946. if (space_left < MIN_FRAME_SIZE_STREAM) {
  1947. *packet_full = 1;
  1948. rc = 1;
  1949. goto err;
  1950. }
  1951. if (!ossl_assert(!h->done_implicit))
  1952. /*
  1953. * Logic below should have ensured we didn't append an
  1954. * implicit-length unless we filled the packet or didn't have
  1955. * another stream to handle, so this should not be possible.
  1956. */
  1957. goto err;
  1958. shdr = &chunks[i % 2].shdr;
  1959. orig_len = chunks[i % 2].orig_len;
  1960. if (i > 0)
  1961. /* Load next chunk for lookahead. */
  1962. if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i + 1,
  1963. &chunks[(i + 1) % 2], conn_consumed))
  1964. goto err;
  1965. /*
  1966. * Find best fit (header length, payload length) combination for if we
  1967. * use an implicit length.
  1968. */
  1969. shdr->has_explicit_len = 0;
  1970. hdr_len_implicit = payload_len_implicit = 0;
  1971. if (!determine_stream_len(h, shdr, space_left,
  1972. &hdr_len_implicit, &payload_len_implicit)) {
  1973. *packet_full = 1;
  1974. rc = 1;
  1975. goto err; /* can't fit anything */
  1976. }
  1977. /*
  1978. * If there is a next stream, we don't use the implicit length so we can
  1979. * add more STREAM frames after this one, unless there is enough data
  1980. * for this STREAM frame to fill the packet.
  1981. */
  1982. can_fill_payload = (hdr_len_implicit + payload_len_implicit
  1983. >= space_left);
  1984. /*
  1985. * Is there is a stream after this one, or another chunk pending
  1986. * transmission in this stream?
  1987. */
  1988. could_have_following_chunk
  1989. = (next_stream != NULL || chunks[(i + 1) % 2].valid);
  1990. /* Choose between explicit or implicit length representations. */
  1991. use_explicit_len = !((can_fill_payload || !could_have_following_chunk)
  1992. && !pkt->force_pad);
  1993. if (use_explicit_len) {
  1994. /*
  1995. * Find best fit (header length, payload length) combination for if
  1996. * we use an explicit length.
  1997. */
  1998. shdr->has_explicit_len = 1;
  1999. hdr_len_explicit = payload_len_explicit = 0;
  2000. if (!determine_stream_len(h, shdr, space_left,
  2001. &hdr_len_explicit, &payload_len_explicit)) {
  2002. *packet_full = 1;
  2003. rc = 1;
  2004. goto err; /* can't fit anything */
  2005. }
  2006. shdr->len = payload_len_explicit;
  2007. } else {
  2008. *packet_full = 1;
  2009. shdr->has_explicit_len = 0;
  2010. shdr->len = payload_len_implicit;
  2011. }
  2012. /* If this is a FIN, don't keep filling the packet with more FINs. */
  2013. if (shdr->is_fin)
  2014. chunks[(i + 1) % 2].valid = 0;
  2015. /*
  2016. * We are now committed to our length (shdr->len can't change).
  2017. * If we truncated the chunk, clear the FIN bit.
  2018. */
  2019. if (shdr->len < orig_len)
  2020. shdr->is_fin = 0;
  2021. /* Truncate IOVs to match our chosen length. */
  2022. ossl_quic_sstream_adjust_iov((size_t)shdr->len, chunks[i % 2].iov,
  2023. chunks[i % 2].num_stream_iovec);
  2024. /*
  2025. * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
  2026. * the stream data.)
  2027. */
  2028. if (!txp_el_ensure_iovec(&txp->el[enc_level], h->num_iovec + 3))
  2029. goto err; /* alloc error */
  2030. /* Encode the header. */
  2031. wpkt = tx_helper_begin(h);
  2032. if (wpkt == NULL)
  2033. goto err; /* alloc error */
  2034. shdr->stream_id = id;
  2035. if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt, shdr))) {
  2036. /* (Should not be possible.) */
  2037. tx_helper_rollback(h);
  2038. *packet_full = 1;
  2039. rc = 1;
  2040. goto err; /* can't fit */
  2041. }
  2042. if (!tx_helper_commit(h))
  2043. goto err; /* alloc error */
  2044. /* Add payload iovecs to the helper (infallible). */
  2045. for (j = 0; j < chunks[i % 2].num_stream_iovec; ++j)
  2046. tx_helper_append_iovec(h, chunks[i % 2].iov[j].buf,
  2047. chunks[i % 2].iov[j].buf_len);
  2048. *have_ack_eliciting = 1;
  2049. tx_helper_unrestrict(h); /* no longer need PING */
  2050. if (!shdr->has_explicit_len)
  2051. h->done_implicit = 1;
  2052. /* Log new TXFC credit which was consumed. */
  2053. if (shdr->len > 0 && shdr->offset + shdr->len > fc_new_hwm)
  2054. fc_new_hwm = shdr->offset + shdr->len;
  2055. /* Log chunk to TXPIM. */
  2056. chunk.stream_id = shdr->stream_id;
  2057. chunk.start = shdr->offset;
  2058. chunk.end = shdr->offset + shdr->len - 1;
  2059. chunk.has_fin = shdr->is_fin;
  2060. chunk.has_stop_sending = 0;
  2061. chunk.has_reset_stream = 0;
  2062. if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
  2063. goto err; /* alloc error */
  2064. if (shdr->len < orig_len) {
  2065. /*
  2066. * If we did not serialize all of this chunk we definitely do not
  2067. * want to try the next chunk
  2068. */
  2069. rc = 1;
  2070. goto err;
  2071. }
  2072. }
  2073. err:
  2074. *new_credit_consumed = fc_new_hwm - fc_swm;
  2075. return rc;
  2076. }
  2077. static void txp_enlink_tmp(QUIC_STREAM **tmp_head, QUIC_STREAM *stream)
  2078. {
  2079. stream->txp_next = *tmp_head;
  2080. *tmp_head = stream;
  2081. }
  2082. static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER *txp,
  2083. struct txp_pkt *pkt,
  2084. int *have_ack_eliciting,
  2085. QUIC_STREAM **tmp_head)
  2086. {
  2087. QUIC_STREAM_ITER it;
  2088. WPACKET *wpkt;
  2089. uint64_t cwm;
  2090. QUIC_STREAM *stream, *snext;
  2091. struct tx_helper *h = &pkt->h;
  2092. uint64_t conn_consumed = 0;
  2093. for (ossl_quic_stream_iter_init(&it, txp->args.qsm, 1);
  2094. it.stream != NULL;) {
  2095. stream = it.stream;
  2096. ossl_quic_stream_iter_next(&it);
  2097. snext = it.stream;
  2098. stream->txp_sent_fc = 0;
  2099. stream->txp_sent_stop_sending = 0;
  2100. stream->txp_sent_reset_stream = 0;
  2101. stream->txp_blocked = 0;
  2102. stream->txp_txfc_new_credit_consumed = 0;
  2103. /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */
  2104. if (stream->want_stop_sending) {
  2105. OSSL_QUIC_FRAME_STOP_SENDING f;
  2106. wpkt = tx_helper_begin(h);
  2107. if (wpkt == NULL)
  2108. return 0; /* alloc error */
  2109. f.stream_id = stream->id;
  2110. f.app_error_code = stream->stop_sending_aec;
  2111. if (!ossl_quic_wire_encode_frame_stop_sending(wpkt, &f)) {
  2112. tx_helper_rollback(h); /* can't fit */
  2113. txp_enlink_tmp(tmp_head, stream);
  2114. break;
  2115. }
  2116. if (!tx_helper_commit(h))
  2117. return 0; /* alloc error */
  2118. *have_ack_eliciting = 1;
  2119. tx_helper_unrestrict(h); /* no longer need PING */
  2120. stream->txp_sent_stop_sending = 1;
  2121. }
  2122. if (stream->want_reset_stream) {
  2123. OSSL_QUIC_FRAME_RESET_STREAM f;
  2124. if (!ossl_assert(stream->send_state == QUIC_SSTREAM_STATE_RESET_SENT))
  2125. return 0;
  2126. wpkt = tx_helper_begin(h);
  2127. if (wpkt == NULL)
  2128. return 0; /* alloc error */
  2129. f.stream_id = stream->id;
  2130. f.app_error_code = stream->reset_stream_aec;
  2131. if (!ossl_quic_stream_send_get_final_size(stream, &f.final_size))
  2132. return 0; /* should not be possible */
  2133. if (!ossl_quic_wire_encode_frame_reset_stream(wpkt, &f)) {
  2134. tx_helper_rollback(h); /* can't fit */
  2135. txp_enlink_tmp(tmp_head, stream);
  2136. break;
  2137. }
  2138. if (!tx_helper_commit(h))
  2139. return 0; /* alloc error */
  2140. *have_ack_eliciting = 1;
  2141. tx_helper_unrestrict(h); /* no longer need PING */
  2142. stream->txp_sent_reset_stream = 1;
  2143. /*
  2144. * The final size of the stream as indicated by RESET_STREAM is used
  2145. * to ensure a consistent view of flow control state by both
  2146. * parties; if we happen to send a RESET_STREAM that consumes more
  2147. * flow control credit, make sure we account for that.
  2148. */
  2149. if (!ossl_assert(f.final_size <= ossl_quic_txfc_get_swm(&stream->txfc)))
  2150. return 0;
  2151. stream->txp_txfc_new_credit_consumed
  2152. = f.final_size - ossl_quic_txfc_get_swm(&stream->txfc);
  2153. }
  2154. /*
  2155. * Stream Flow Control Frames (MAX_STREAM_DATA)
  2156. *
  2157. * RFC 9000 s. 13.3: "An endpoint SHOULD stop sending MAX_STREAM_DATA
  2158. * frames when the receiving part of the stream enters a "Size Known" or
  2159. * "Reset Recvd" state." -- In practice, RECV is the only state
  2160. * in which it makes sense to generate more MAX_STREAM_DATA frames.
  2161. */
  2162. if (stream->recv_state == QUIC_RSTREAM_STATE_RECV
  2163. && (stream->want_max_stream_data
  2164. || ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 0))) {
  2165. wpkt = tx_helper_begin(h);
  2166. if (wpkt == NULL)
  2167. return 0; /* alloc error */
  2168. cwm = ossl_quic_rxfc_get_cwm(&stream->rxfc);
  2169. if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt, stream->id,
  2170. cwm)) {
  2171. tx_helper_rollback(h); /* can't fit */
  2172. txp_enlink_tmp(tmp_head, stream);
  2173. break;
  2174. }
  2175. if (!tx_helper_commit(h))
  2176. return 0; /* alloc error */
  2177. *have_ack_eliciting = 1;
  2178. tx_helper_unrestrict(h); /* no longer need PING */
  2179. stream->txp_sent_fc = 1;
  2180. }
  2181. /*
  2182. * Stream Data Frames (STREAM)
  2183. *
  2184. * RFC 9000 s. 3.3: A sender MUST NOT send a STREAM [...] frame for a
  2185. * stream in the "Reset Sent" state [or any terminal state]. We don't
  2186. * send any more STREAM frames if we are sending, have sent, or are
  2187. * planning to send, RESET_STREAM. The other terminal state is Data
  2188. * Recvd, but txp_generate_stream_frames() is guaranteed to generate
  2189. * nothing in this case.
  2190. */
  2191. if (ossl_quic_stream_has_send_buffer(stream)
  2192. && !ossl_quic_stream_send_is_reset(stream)) {
  2193. int packet_full = 0;
  2194. if (!ossl_assert(!stream->want_reset_stream))
  2195. return 0;
  2196. if (!txp_generate_stream_frames(txp, pkt,
  2197. stream->id, stream->sstream,
  2198. &stream->txfc,
  2199. snext,
  2200. have_ack_eliciting,
  2201. &packet_full,
  2202. &stream->txp_txfc_new_credit_consumed,
  2203. conn_consumed)) {
  2204. /* Fatal error (allocation, etc.) */
  2205. txp_enlink_tmp(tmp_head, stream);
  2206. return 0;
  2207. }
  2208. conn_consumed += stream->txp_txfc_new_credit_consumed;
  2209. if (packet_full) {
  2210. txp_enlink_tmp(tmp_head, stream);
  2211. break;
  2212. }
  2213. }
  2214. txp_enlink_tmp(tmp_head, stream);
  2215. }
  2216. return 1;
  2217. }
  2218. static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp,
  2219. struct txp_pkt *pkt,
  2220. int chosen_for_conn_close)
  2221. {
  2222. int rc = TXP_ERR_SUCCESS;
  2223. const uint32_t enc_level = pkt->h.enc_level;
  2224. const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  2225. int have_ack_eliciting = 0, done_pre_token = 0;
  2226. const struct archetype_data a = pkt->geom.adata;
  2227. /*
  2228. * Cleared if we encode any non-ACK-eliciting frame type which rules out the
  2229. * packet being a non-inflight frame. This means any non-ACK ACK-eliciting
  2230. * frame, even PADDING frames. ACK eliciting frames always cause a packet to
  2231. * become ineligible for non-inflight treatment so it is not necessary to
  2232. * clear this in cases where have_ack_eliciting is set, as it is ignored in
  2233. * that case.
  2234. */
  2235. int can_be_non_inflight = 1;
  2236. QUIC_CFQ_ITEM *cfq_item;
  2237. QUIC_TXPIM_PKT *tpkt = NULL;
  2238. struct tx_helper *h = &pkt->h;
  2239. /* Maximum PN reached? */
  2240. if (!ossl_quic_pn_valid(txp->next_pn[pn_space]))
  2241. goto fatal_err;
  2242. if (!ossl_assert(pkt->tpkt == NULL))
  2243. goto fatal_err;
  2244. if ((pkt->tpkt = tpkt = ossl_quic_txpim_pkt_alloc(txp->args.txpim)) == NULL)
  2245. goto fatal_err;
  2246. /*
  2247. * Frame Serialization
  2248. * ===================
  2249. *
  2250. * We now serialize frames into the packet in descending order of priority.
  2251. */
  2252. /* HANDSHAKE_DONE (Regenerate) */
  2253. if (a.allow_handshake_done && txp->want_handshake_done
  2254. && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_HANDSHAKE_DONE) {
  2255. WPACKET *wpkt = tx_helper_begin(h);
  2256. if (wpkt == NULL)
  2257. goto fatal_err;
  2258. if (ossl_quic_wire_encode_frame_handshake_done(wpkt)) {
  2259. tpkt->had_handshake_done_frame = 1;
  2260. have_ack_eliciting = 1;
  2261. if (!tx_helper_commit(h))
  2262. goto fatal_err;
  2263. tx_helper_unrestrict(h); /* no longer need PING */
  2264. } else {
  2265. tx_helper_rollback(h);
  2266. }
  2267. }
  2268. /* MAX_DATA (Regenerate) */
  2269. if (a.allow_conn_fc
  2270. && (txp->want_max_data
  2271. || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0))
  2272. && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_DATA) {
  2273. WPACKET *wpkt = tx_helper_begin(h);
  2274. uint64_t cwm = ossl_quic_rxfc_get_cwm(txp->args.conn_rxfc);
  2275. if (wpkt == NULL)
  2276. goto fatal_err;
  2277. if (ossl_quic_wire_encode_frame_max_data(wpkt, cwm)) {
  2278. tpkt->had_max_data_frame = 1;
  2279. have_ack_eliciting = 1;
  2280. if (!tx_helper_commit(h))
  2281. goto fatal_err;
  2282. tx_helper_unrestrict(h); /* no longer need PING */
  2283. } else {
  2284. tx_helper_rollback(h);
  2285. }
  2286. }
  2287. /* MAX_STREAMS_BIDI (Regenerate) */
  2288. if (a.allow_conn_fc
  2289. && (txp->want_max_streams_bidi
  2290. || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 0))
  2291. && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI) {
  2292. WPACKET *wpkt = tx_helper_begin(h);
  2293. uint64_t max_streams
  2294. = ossl_quic_rxfc_get_cwm(txp->args.max_streams_bidi_rxfc);
  2295. if (wpkt == NULL)
  2296. goto fatal_err;
  2297. if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/0,
  2298. max_streams)) {
  2299. tpkt->had_max_streams_bidi_frame = 1;
  2300. have_ack_eliciting = 1;
  2301. if (!tx_helper_commit(h))
  2302. goto fatal_err;
  2303. tx_helper_unrestrict(h); /* no longer need PING */
  2304. } else {
  2305. tx_helper_rollback(h);
  2306. }
  2307. }
  2308. /* MAX_STREAMS_UNI (Regenerate) */
  2309. if (a.allow_conn_fc
  2310. && (txp->want_max_streams_uni
  2311. || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 0))
  2312. && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI) {
  2313. WPACKET *wpkt = tx_helper_begin(h);
  2314. uint64_t max_streams
  2315. = ossl_quic_rxfc_get_cwm(txp->args.max_streams_uni_rxfc);
  2316. if (wpkt == NULL)
  2317. goto fatal_err;
  2318. if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/1,
  2319. max_streams)) {
  2320. tpkt->had_max_streams_uni_frame = 1;
  2321. have_ack_eliciting = 1;
  2322. if (!tx_helper_commit(h))
  2323. goto fatal_err;
  2324. tx_helper_unrestrict(h); /* no longer need PING */
  2325. } else {
  2326. tx_helper_rollback(h);
  2327. }
  2328. }
  2329. /* GCR Frames */
  2330. for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
  2331. cfq_item != NULL;
  2332. cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
  2333. uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
  2334. const unsigned char *encoded = ossl_quic_cfq_item_get_encoded(cfq_item);
  2335. size_t encoded_len = ossl_quic_cfq_item_get_encoded_len(cfq_item);
  2336. switch (frame_type) {
  2337. case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
  2338. if (!a.allow_new_conn_id)
  2339. continue;
  2340. break;
  2341. case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
  2342. if (!a.allow_retire_conn_id)
  2343. continue;
  2344. break;
  2345. case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
  2346. if (!a.allow_new_token)
  2347. continue;
  2348. /*
  2349. * NEW_TOKEN frames are handled via GCR, but some
  2350. * Regenerate-strategy frames should come before them (namely
  2351. * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If
  2352. * we find a NEW_TOKEN frame, do these now. If there are no
  2353. * NEW_TOKEN frames in the GCR queue we will handle these below.
  2354. */
  2355. if (!done_pre_token)
  2356. if (txp_generate_pre_token(txp, pkt,
  2357. chosen_for_conn_close,
  2358. &can_be_non_inflight))
  2359. done_pre_token = 1;
  2360. break;
  2361. case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE:
  2362. if (!a.allow_path_response)
  2363. continue;
  2364. /*
  2365. * RFC 9000 s. 8.2.2: An endpoint MUST expand datagrams that
  2366. * contain a PATH_RESPONSE frame to at least the smallest
  2367. * allowed maximum datagram size of 1200 bytes.
  2368. */
  2369. pkt->force_pad = 1;
  2370. break;
  2371. default:
  2372. if (!a.allow_cfq_other)
  2373. continue;
  2374. break;
  2375. }
  2376. /*
  2377. * If the frame is too big, don't try to schedule any more GCR frames in
  2378. * this packet rather than sending subsequent ones out of order.
  2379. */
  2380. if (encoded_len > tx_helper_get_space_left(h))
  2381. break;
  2382. if (!tx_helper_append_iovec(h, encoded, encoded_len))
  2383. goto fatal_err;
  2384. ossl_quic_txpim_pkt_add_cfq_item(tpkt, cfq_item);
  2385. if (ossl_quic_frame_type_is_ack_eliciting(frame_type)) {
  2386. have_ack_eliciting = 1;
  2387. tx_helper_unrestrict(h); /* no longer need PING */
  2388. }
  2389. }
  2390. /*
  2391. * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or
  2392. * PATH_RESPONSE (as desired) before, do so now.
  2393. */
  2394. if (!done_pre_token)
  2395. if (txp_generate_pre_token(txp, pkt,
  2396. chosen_for_conn_close,
  2397. &can_be_non_inflight))
  2398. done_pre_token = 1;
  2399. /* CRYPTO Frames */
  2400. if (a.allow_crypto)
  2401. if (!txp_generate_crypto_frames(txp, pkt, &have_ack_eliciting))
  2402. goto fatal_err;
  2403. /* Stream-specific frames */
  2404. if (a.allow_stream_rel && txp->handshake_complete)
  2405. if (!txp_generate_stream_related(txp, pkt,
  2406. &have_ack_eliciting,
  2407. &pkt->stream_head))
  2408. goto fatal_err;
  2409. /* PING */
  2410. tx_helper_unrestrict(h);
  2411. if (!have_ack_eliciting && txp_need_ping(txp, pn_space, &a)) {
  2412. WPACKET *wpkt;
  2413. assert(h->reserve > 0);
  2414. wpkt = tx_helper_begin(h);
  2415. if (wpkt == NULL)
  2416. goto fatal_err;
  2417. if (!ossl_quic_wire_encode_frame_ping(wpkt)
  2418. || !tx_helper_commit(h))
  2419. /*
  2420. * We treat a request to be ACK-eliciting as a requirement, so this
  2421. * is an error.
  2422. */
  2423. goto fatal_err;
  2424. have_ack_eliciting = 1;
  2425. }
  2426. /* PADDING is added by ossl_quic_tx_packetiser_generate(). */
  2427. /*
  2428. * ACKM Data
  2429. * =========
  2430. */
  2431. if (have_ack_eliciting)
  2432. can_be_non_inflight = 0;
  2433. /* ACKM Data */
  2434. tpkt->ackm_pkt.num_bytes = h->bytes_appended + pkt->geom.pkt_overhead;
  2435. tpkt->ackm_pkt.pkt_num = txp->next_pn[pn_space];
  2436. /* largest_acked is set in txp_generate_pre_token */
  2437. tpkt->ackm_pkt.pkt_space = pn_space;
  2438. tpkt->ackm_pkt.is_inflight = !can_be_non_inflight;
  2439. tpkt->ackm_pkt.is_ack_eliciting = have_ack_eliciting;
  2440. tpkt->ackm_pkt.is_pto_probe = 0;
  2441. tpkt->ackm_pkt.is_mtu_probe = 0;
  2442. tpkt->ackm_pkt.time = txp->args.now(txp->args.now_arg);
  2443. tpkt->pkt_type = pkt->phdr.type;
  2444. /* Done. */
  2445. return rc;
  2446. fatal_err:
  2447. /*
  2448. * Handler for fatal errors, i.e. errors causing us to abort the entire
  2449. * packet rather than just one frame. Examples of such errors include
  2450. * allocation errors.
  2451. */
  2452. if (tpkt != NULL) {
  2453. ossl_quic_txpim_pkt_release(txp->args.txpim, tpkt);
  2454. pkt->tpkt = NULL;
  2455. }
  2456. return TXP_ERR_INTERNAL;
  2457. }
  2458. /*
  2459. * Commits and queues a packet for transmission. There is no backing out after
  2460. * this.
  2461. *
  2462. * This:
  2463. *
  2464. * - Sends the packet to the QTX for encryption and transmission;
  2465. *
  2466. * - Records the packet as having been transmitted in FIFM. ACKM is informed,
  2467. * etc. and the TXPIM record is filed.
  2468. *
  2469. * - Informs various subsystems of frames that were sent and clears frame
  2470. * wanted flags so that we do not generate the same frames again.
  2471. *
  2472. * Assumptions:
  2473. *
  2474. * - pkt is a txp_pkt for the correct EL;
  2475. *
  2476. * - pkt->tpkt is valid;
  2477. *
  2478. * - pkt->tpkt->ackm_pkt has been fully filled in;
  2479. *
  2480. * - Stream chunk records have been appended to pkt->tpkt for STREAM and
  2481. * CRYPTO frames, but not for RESET_STREAM or STOP_SENDING frames;
  2482. *
  2483. * - The chosen stream list for the packet can be fully walked from
  2484. * pkt->stream_head using stream->txp_next;
  2485. *
  2486. * - pkt->has_ack_eliciting is set correctly.
  2487. *
  2488. */
  2489. static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER *txp,
  2490. struct txp_pkt *pkt,
  2491. uint32_t archetype,
  2492. int *txpim_pkt_reffed)
  2493. {
  2494. int rc = 1;
  2495. uint32_t enc_level = pkt->h.enc_level;
  2496. uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  2497. QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
  2498. QUIC_STREAM *stream;
  2499. OSSL_QTX_PKT txpkt;
  2500. struct archetype_data a;
  2501. *txpim_pkt_reffed = 0;
  2502. /* Cannot send a packet with an empty payload. */
  2503. if (pkt->h.bytes_appended == 0)
  2504. return 0;
  2505. if (!txp_get_archetype_data(enc_level, archetype, &a))
  2506. return 0;
  2507. /* Packet Information for QTX */
  2508. txpkt.hdr = &pkt->phdr;
  2509. txpkt.iovec = txp->el[enc_level].iovec;
  2510. txpkt.num_iovec = pkt->h.num_iovec;
  2511. txpkt.local = NULL;
  2512. txpkt.peer = BIO_ADDR_family(&txp->args.peer) == AF_UNSPEC
  2513. ? NULL : &txp->args.peer;
  2514. txpkt.pn = txp->next_pn[pn_space];
  2515. txpkt.flags = OSSL_QTX_PKT_FLAG_COALESCE; /* always try to coalesce */
  2516. /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */
  2517. for (stream = pkt->stream_head; stream != NULL; stream = stream->txp_next)
  2518. if (stream->txp_sent_stop_sending || stream->txp_sent_reset_stream) {
  2519. /* Log STOP_SENDING/RESET_STREAM chunk to TXPIM. */
  2520. QUIC_TXPIM_CHUNK chunk;
  2521. chunk.stream_id = stream->id;
  2522. chunk.start = UINT64_MAX;
  2523. chunk.end = 0;
  2524. chunk.has_fin = 0;
  2525. chunk.has_stop_sending = stream->txp_sent_stop_sending;
  2526. chunk.has_reset_stream = stream->txp_sent_reset_stream;
  2527. if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
  2528. return 0; /* alloc error */
  2529. }
  2530. /* Dispatch to FIFD. */
  2531. if (!ossl_quic_fifd_pkt_commit(&txp->fifd, tpkt))
  2532. return 0;
  2533. /*
  2534. * Transmission and Post-Packet Generation Bookkeeping
  2535. * ===================================================
  2536. *
  2537. * No backing out anymore - at this point the ACKM has recorded the packet
  2538. * as having been sent, so we need to increment our next PN counter, or
  2539. * the ACKM will complain when we try to record a duplicate packet with
  2540. * the same PN later. At this point actually sending the packet may still
  2541. * fail. In this unlikely event it will simply be handled as though it
  2542. * were a lost packet.
  2543. */
  2544. ++txp->next_pn[pn_space];
  2545. *txpim_pkt_reffed = 1;
  2546. /* Send the packet. */
  2547. if (!ossl_qtx_write_pkt(txp->args.qtx, &txpkt))
  2548. return 0;
  2549. /*
  2550. * Record FC and stream abort frames as sent; deactivate streams which no
  2551. * longer have anything to do.
  2552. */
  2553. for (stream = pkt->stream_head; stream != NULL; stream = stream->txp_next) {
  2554. if (stream->txp_sent_fc) {
  2555. stream->want_max_stream_data = 0;
  2556. ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 1);
  2557. }
  2558. if (stream->txp_sent_stop_sending)
  2559. stream->want_stop_sending = 0;
  2560. if (stream->txp_sent_reset_stream)
  2561. stream->want_reset_stream = 0;
  2562. if (stream->txp_txfc_new_credit_consumed > 0) {
  2563. if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream->txfc,
  2564. stream->txp_txfc_new_credit_consumed)))
  2565. /*
  2566. * Should not be possible, but we should continue with our
  2567. * bookkeeping as we have already committed the packet to the
  2568. * FIFD. Just change the value we return.
  2569. */
  2570. rc = 0;
  2571. stream->txp_txfc_new_credit_consumed = 0;
  2572. }
  2573. /*
  2574. * If we no longer need to generate any flow control (MAX_STREAM_DATA),
  2575. * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because
  2576. * the stream is drained of data or TXFC-blocked), we can mark the
  2577. * stream as inactive.
  2578. */
  2579. ossl_quic_stream_map_update_state(txp->args.qsm, stream);
  2580. if (ossl_quic_stream_has_send_buffer(stream)
  2581. && !ossl_quic_sstream_has_pending(stream->sstream)
  2582. && ossl_quic_sstream_get_final_size(stream->sstream, NULL))
  2583. /*
  2584. * Transition to DATA_SENT if stream has a final size and we have
  2585. * sent all data.
  2586. */
  2587. ossl_quic_stream_map_notify_all_data_sent(txp->args.qsm, stream);
  2588. }
  2589. /* We have now sent the packet, so update state accordingly. */
  2590. if (tpkt->ackm_pkt.is_ack_eliciting)
  2591. txp->force_ack_eliciting &= ~(1UL << pn_space);
  2592. if (tpkt->had_handshake_done_frame)
  2593. txp->want_handshake_done = 0;
  2594. if (tpkt->had_max_data_frame) {
  2595. txp->want_max_data = 0;
  2596. ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 1);
  2597. }
  2598. if (tpkt->had_max_streams_bidi_frame) {
  2599. txp->want_max_streams_bidi = 0;
  2600. ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 1);
  2601. }
  2602. if (tpkt->had_max_streams_uni_frame) {
  2603. txp->want_max_streams_uni = 0;
  2604. ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 1);
  2605. }
  2606. if (tpkt->had_ack_frame)
  2607. txp->want_ack &= ~(1UL << pn_space);
  2608. if (tpkt->had_conn_close)
  2609. txp->want_conn_close = 0;
  2610. /*
  2611. * Decrement probe request counts if we have sent a packet that meets
  2612. * the requirement of a probe, namely being ACK-eliciting.
  2613. */
  2614. if (tpkt->ackm_pkt.is_ack_eliciting) {
  2615. OSSL_ACKM_PROBE_INFO *probe_info
  2616. = ossl_ackm_get0_probe_request(txp->args.ackm);
  2617. if (enc_level == QUIC_ENC_LEVEL_INITIAL
  2618. && probe_info->anti_deadlock_initial > 0)
  2619. --probe_info->anti_deadlock_initial;
  2620. if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
  2621. && probe_info->anti_deadlock_handshake > 0)
  2622. --probe_info->anti_deadlock_handshake;
  2623. if (a.allow_force_ack_eliciting /* (i.e., not for 0-RTT) */
  2624. && probe_info->pto[pn_space] > 0)
  2625. --probe_info->pto[pn_space];
  2626. }
  2627. return rc;
  2628. }
  2629. /* Ensure the iovec array is at least num elements long. */
  2630. static int txp_el_ensure_iovec(struct txp_el *el, size_t num)
  2631. {
  2632. OSSL_QTX_IOVEC *iovec;
  2633. if (el->alloc_iovec >= num)
  2634. return 1;
  2635. num = el->alloc_iovec != 0 ? el->alloc_iovec * 2 : 8;
  2636. iovec = OPENSSL_realloc(el->iovec, sizeof(OSSL_QTX_IOVEC) * num);
  2637. if (iovec == NULL)
  2638. return 0;
  2639. el->iovec = iovec;
  2640. el->alloc_iovec = num;
  2641. return 1;
  2642. }
  2643. int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER *txp,
  2644. const OSSL_QUIC_FRAME_CONN_CLOSE *f)
  2645. {
  2646. char *reason = NULL;
  2647. size_t reason_len = f->reason_len;
  2648. size_t max_reason_len = txp_get_mdpl(txp) / 2;
  2649. if (txp->want_conn_close)
  2650. return 0;
  2651. /*
  2652. * Arbitrarily limit the length of the reason length string to half of the
  2653. * MDPL.
  2654. */
  2655. if (reason_len > max_reason_len)
  2656. reason_len = max_reason_len;
  2657. if (reason_len > 0) {
  2658. reason = OPENSSL_memdup(f->reason, reason_len);
  2659. if (reason == NULL)
  2660. return 0;
  2661. }
  2662. txp->conn_close_frame = *f;
  2663. txp->conn_close_frame.reason = reason;
  2664. txp->conn_close_frame.reason_len = reason_len;
  2665. txp->want_conn_close = 1;
  2666. return 1;
  2667. }
  2668. void ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER *txp,
  2669. ossl_msg_cb msg_callback,
  2670. SSL *msg_callback_ssl)
  2671. {
  2672. txp->msg_callback = msg_callback;
  2673. txp->msg_callback_ssl = msg_callback_ssl;
  2674. }
  2675. void ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER *txp,
  2676. void *msg_callback_arg)
  2677. {
  2678. txp->msg_callback_arg = msg_callback_arg;
  2679. }
  2680. QUIC_PN ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER *txp,
  2681. uint32_t pn_space)
  2682. {
  2683. if (pn_space >= QUIC_PN_SPACE_NUM)
  2684. return UINT64_MAX;
  2685. return txp->next_pn[pn_space];
  2686. }
  2687. OSSL_TIME ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER *txp)
  2688. {
  2689. /*
  2690. * TXP-specific deadline computations which rely on TXP innards. This is in
  2691. * turn relied on by the QUIC_CHANNEL code to determine the channel event
  2692. * handling deadline.
  2693. */
  2694. OSSL_TIME deadline = ossl_time_infinite();
  2695. uint32_t enc_level, pn_space;
  2696. /*
  2697. * ACK generation is not CC-gated - packets containing only ACKs are allowed
  2698. * to bypass CC. We want to generate ACK frames even if we are currently
  2699. * restricted by CC so the peer knows we have received data. The generate
  2700. * call will take care of selecting the correct packet archetype.
  2701. */
  2702. for (enc_level = QUIC_ENC_LEVEL_INITIAL;
  2703. enc_level < QUIC_ENC_LEVEL_NUM;
  2704. ++enc_level)
  2705. if (ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level)) {
  2706. pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
  2707. deadline = ossl_time_min(deadline,
  2708. ossl_ackm_get_ack_deadline(txp->args.ackm, pn_space));
  2709. }
  2710. /* When will CC let us send more? */
  2711. if (txp->args.cc_method->get_tx_allowance(txp->args.cc_data) == 0)
  2712. deadline = ossl_time_min(deadline,
  2713. txp->args.cc_method->get_wakeup_deadline(txp->args.cc_data));
  2714. return deadline;
  2715. }