lex.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380
  1. #include <u.h>
  2. #include <libc.h>
  3. #include <draw.h>
  4. #include <ctype.h>
  5. #include <html.h>
  6. #include "impl.h"
  7. typedef struct TokenSource TokenSource;
  8. struct TokenSource
  9. {
  10. int i; // index of next byte to use
  11. uchar* data; // all the data
  12. int edata; // data[0:edata] is valid
  13. int chset; // one of US_Ascii, etc.
  14. int mtype; // TextHtml or TextPlain
  15. };
  16. enum {
  17. EOF = -2,
  18. EOB = -1
  19. };
  20. #define ISNAMCHAR(c) ((c)<256 && (isalpha(c) || isdigit(c) || (c) == '-' || (c) == '.'))
  21. #define SMALLBUFSIZE 240
  22. #define BIGBUFSIZE 2000
  23. // HTML 4.0 tag names.
  24. // Keep sorted, and in correspondence with enum in iparse.h.
  25. Rune* tagnames[] = {
  26. L" ",
  27. L"!",
  28. L"a",
  29. L"abbr",
  30. L"acronym",
  31. L"address",
  32. L"applet",
  33. L"area",
  34. L"b",
  35. L"base",
  36. L"basefont",
  37. L"bdo",
  38. L"big",
  39. L"blink",
  40. L"blockquote",
  41. L"body",
  42. L"bq",
  43. L"br",
  44. L"button",
  45. L"caption",
  46. L"center",
  47. L"cite",
  48. L"code",
  49. L"col",
  50. L"colgroup",
  51. L"dd",
  52. L"del",
  53. L"dfn",
  54. L"dir",
  55. L"div",
  56. L"dl",
  57. L"dt",
  58. L"em",
  59. L"fieldset",
  60. L"font",
  61. L"form",
  62. L"frame",
  63. L"frameset",
  64. L"h1",
  65. L"h2",
  66. L"h3",
  67. L"h4",
  68. L"h5",
  69. L"h6",
  70. L"head",
  71. L"hr",
  72. L"html",
  73. L"i",
  74. L"iframe",
  75. L"img",
  76. L"input",
  77. L"ins",
  78. L"isindex",
  79. L"kbd",
  80. L"label",
  81. L"legend",
  82. L"li",
  83. L"link",
  84. L"map",
  85. L"menu",
  86. L"meta",
  87. L"nobr",
  88. L"noframes",
  89. L"noscript",
  90. L"object",
  91. L"ol",
  92. L"optgroup",
  93. L"option",
  94. L"p",
  95. L"param",
  96. L"pre",
  97. L"q",
  98. L"s",
  99. L"samp",
  100. L"script",
  101. L"select",
  102. L"small",
  103. L"span",
  104. L"strike",
  105. L"strong",
  106. L"style",
  107. L"sub",
  108. L"sup",
  109. L"table",
  110. L"tbody",
  111. L"td",
  112. L"textarea",
  113. L"tfoot",
  114. L"th",
  115. L"thead",
  116. L"title",
  117. L"tr",
  118. L"tt",
  119. L"u",
  120. L"ul",
  121. L"var"
  122. };
  123. // HTML 4.0 attribute names.
  124. // Keep sorted, and in correspondence with enum in i.h.
  125. Rune* attrnames[] = {
  126. L"abbr",
  127. L"accept-charset",
  128. L"access-key",
  129. L"action",
  130. L"align",
  131. L"alink",
  132. L"alt",
  133. L"archive",
  134. L"axis",
  135. L"background",
  136. L"bgcolor",
  137. L"border",
  138. L"cellpadding",
  139. L"cellspacing",
  140. L"char",
  141. L"charoff",
  142. L"charset",
  143. L"checked",
  144. L"cite",
  145. L"class",
  146. L"classid",
  147. L"clear",
  148. L"code",
  149. L"codebase",
  150. L"codetype",
  151. L"color",
  152. L"cols",
  153. L"colspan",
  154. L"compact",
  155. L"content",
  156. L"coords",
  157. L"data",
  158. L"datetime",
  159. L"declare",
  160. L"defer",
  161. L"dir",
  162. L"disabled",
  163. L"enctype",
  164. L"face",
  165. L"for",
  166. L"frame",
  167. L"frameborder",
  168. L"headers",
  169. L"height",
  170. L"href",
  171. L"hreflang",
  172. L"hspace",
  173. L"http-equiv",
  174. L"id",
  175. L"ismap",
  176. L"label",
  177. L"lang",
  178. L"link",
  179. L"longdesc",
  180. L"marginheight",
  181. L"marginwidth",
  182. L"maxlength",
  183. L"media",
  184. L"method",
  185. L"multiple",
  186. L"name",
  187. L"nohref",
  188. L"noresize",
  189. L"noshade",
  190. L"nowrap",
  191. L"object",
  192. L"onblur",
  193. L"onchange",
  194. L"onclick",
  195. L"ondblclick",
  196. L"onfocus",
  197. L"onkeypress",
  198. L"onkeyup",
  199. L"onload",
  200. L"onmousedown",
  201. L"onmousemove",
  202. L"onmouseout",
  203. L"onmouseover",
  204. L"onmouseup",
  205. L"onreset",
  206. L"onselect",
  207. L"onsubmit",
  208. L"onunload",
  209. L"profile",
  210. L"prompt",
  211. L"readonly",
  212. L"rel",
  213. L"rev",
  214. L"rows",
  215. L"rowspan",
  216. L"rules",
  217. L"scheme",
  218. L"scope",
  219. L"scrolling",
  220. L"selected",
  221. L"shape",
  222. L"size",
  223. L"span",
  224. L"src",
  225. L"standby",
  226. L"start",
  227. L"style",
  228. L"summary",
  229. L"tabindex",
  230. L"target",
  231. L"text",
  232. L"title",
  233. L"type",
  234. L"usemap",
  235. L"valign",
  236. L"value",
  237. L"valuetype",
  238. L"version",
  239. L"vlink",
  240. L"vspace",
  241. L"width"
  242. };
  243. // Character entity to unicode character number map.
  244. // Keep sorted by name.
  245. StringInt chartab[]= {
  246. {L"AElig", 198},
  247. {L"Aacute", 193},
  248. {L"Acirc", 194},
  249. {L"Agrave", 192},
  250. {L"Aring", 197},
  251. {L"Atilde", 195},
  252. {L"Auml", 196},
  253. {L"Ccedil", 199},
  254. {L"ETH", 208},
  255. {L"Eacute", 201},
  256. {L"Ecirc", 202},
  257. {L"Egrave", 200},
  258. {L"Euml", 203},
  259. {L"Iacute", 205},
  260. {L"Icirc", 206},
  261. {L"Igrave", 204},
  262. {L"Iuml", 207},
  263. {L"Ntilde", 209},
  264. {L"Oacute", 211},
  265. {L"Ocirc", 212},
  266. {L"Ograve", 210},
  267. {L"Oslash", 216},
  268. {L"Otilde", 213},
  269. {L"Ouml", 214},
  270. {L"THORN", 222},
  271. {L"Uacute", 218},
  272. {L"Ucirc", 219},
  273. {L"Ugrave", 217},
  274. {L"Uuml", 220},
  275. {L"Yacute", 221},
  276. {L"aacute", 225},
  277. {L"acirc", 226},
  278. {L"acute", 180},
  279. {L"aelig", 230},
  280. {L"agrave", 224},
  281. {L"alpha", 945},
  282. {L"amp", 38},
  283. {L"aring", 229},
  284. {L"atilde", 227},
  285. {L"auml", 228},
  286. {L"beta", 946},
  287. {L"brvbar", 166},
  288. {L"ccedil", 231},
  289. {L"cdots", 8943},
  290. {L"cedil", 184},
  291. {L"cent", 162},
  292. {L"chi", 967},
  293. {L"copy", 169},
  294. {L"curren", 164},
  295. {L"ddots", 8945},
  296. {L"deg", 176},
  297. {L"delta", 948},
  298. {L"divide", 247},
  299. {L"eacute", 233},
  300. {L"ecirc", 234},
  301. {L"egrave", 232},
  302. {L"emdash", 8212}, /* non-standard but commonly used */
  303. {L"emsp", 8195},
  304. {L"endash", 8211}, /* non-standard but commonly used */
  305. {L"ensp", 8194},
  306. {L"epsilon", 949},
  307. {L"eta", 951},
  308. {L"eth", 240},
  309. {L"euml", 235},
  310. {L"frac12", 189},
  311. {L"frac14", 188},
  312. {L"frac34", 190},
  313. {L"gamma", 947},
  314. {L"gt", 62},
  315. {L"iacute", 237},
  316. {L"icirc", 238},
  317. {L"iexcl", 161},
  318. {L"igrave", 236},
  319. {L"iota", 953},
  320. {L"iquest", 191},
  321. {L"iuml", 239},
  322. {L"kappa", 954},
  323. {L"lambda", 955},
  324. {L"laquo", 171},
  325. {L"ldots", 8230},
  326. {L"lt", 60},
  327. {L"macr", 175},
  328. {L"mdash", 8212},
  329. {L"micro", 181},
  330. {L"middot", 183},
  331. {L"mu", 956},
  332. {L"nbsp", 160},
  333. {L"ndash", 8211},
  334. {L"not", 172},
  335. {L"ntilde", 241},
  336. {L"nu", 957},
  337. {L"oacute", 243},
  338. {L"ocirc", 244},
  339. {L"ograve", 242},
  340. {L"omega", 969},
  341. {L"omicron", 959},
  342. {L"ordf", 170},
  343. {L"ordm", 186},
  344. {L"oslash", 248},
  345. {L"otilde", 245},
  346. {L"ouml", 246},
  347. {L"para", 182},
  348. {L"phi", 966},
  349. {L"pi", 960},
  350. {L"plusmn", 177},
  351. {L"pound", 163},
  352. {L"psi", 968},
  353. {L"quad", 8193},
  354. {L"quot", 34},
  355. {L"raquo", 187},
  356. {L"reg", 174},
  357. {L"rho", 961},
  358. {L"sect", 167},
  359. {L"shy", 173},
  360. {L"sigma", 963},
  361. {L"sp", 8194},
  362. {L"sup1", 185},
  363. {L"sup2", 178},
  364. {L"sup3", 179},
  365. {L"szlig", 223},
  366. {L"tau", 964},
  367. {L"theta", 952},
  368. {L"thinsp", 8201},
  369. {L"thorn", 254},
  370. {L"times", 215},
  371. {L"trade", 8482},
  372. {L"uacute", 250},
  373. {L"ucirc", 251},
  374. {L"ugrave", 249},
  375. {L"uml", 168},
  376. {L"upsilon", 965},
  377. {L"uuml", 252},
  378. {L"varepsilon", 8712},
  379. {L"varphi", 981},
  380. {L"varpi", 982},
  381. {L"varrho", 1009},
  382. {L"vdots", 8942},
  383. {L"vsigma", 962},
  384. {L"vtheta", 977},
  385. {L"xi", 958},
  386. {L"yacute", 253},
  387. {L"yen", 165},
  388. {L"yuml", 255},
  389. {L"zeta", 950}
  390. };
  391. #define NCHARTAB (sizeof(chartab)/sizeof(chartab[0]))
  392. // Characters Winstart..Winend are those that Windows
  393. // uses interpolated into the Latin1 set.
  394. // They aren't supposed to appear in HTML, but they do....
  395. enum {
  396. Winstart = 127,
  397. Winend = 159
  398. };
  399. static int winchars[]= { 8226, // 8226 is a bullet
  400. 8226, 8226, 8218, 402, 8222, 8230, 8224, 8225,
  401. 710, 8240, 352, 8249, 338, 8226, 8226, 8226,
  402. 8226, 8216, 8217, 8220, 8221, 8226, 8211, 8212,
  403. 732, 8482, 353, 8250, 339, 8226, 8226, 376};
  404. static StringInt* tagtable; // initialized from tagnames
  405. static StringInt* attrtable; // initialized from attrnames
  406. static void lexinit();
  407. static int getplaindata(TokenSource* ts, Token* a, int* pai);
  408. static int getdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai);
  409. static int getscriptdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai);
  410. static int gettag(TokenSource* ts, int starti, Token* a, int* pai);
  411. static Rune* buftostr(Rune* s, Rune* buf, int j);
  412. static int comment(TokenSource* ts);
  413. static int findstr(TokenSource* ts, Rune* s);
  414. static int ampersand(TokenSource* ts);
  415. static int lowerc(int c);
  416. static int getchar(TokenSource* ts);
  417. static void ungetchar(TokenSource* ts, int c);
  418. static void backup(TokenSource* ts, int savei);
  419. static void freeinsidetoken(Token* t);
  420. static void freeattrs(Attr* ahead);
  421. static Attr* newattr(int attid, Rune* value, Attr* link);
  422. static int Tconv(Fmt* f);
  423. int dbglex = 0;
  424. static int lexinited = 0;
  425. static void
  426. lexinit(void)
  427. {
  428. tagtable = _makestrinttab(tagnames, Numtags);
  429. attrtable = _makestrinttab(attrnames, Numattrs);
  430. fmtinstall('T', Tconv);
  431. lexinited = 1;
  432. }
  433. static TokenSource*
  434. newtokensource(uchar* data, int edata, int chset, int mtype)
  435. {
  436. TokenSource* ans;
  437. assert(chset == US_Ascii || chset == ISO_8859_1 ||
  438. chset == UTF_8 || chset == Unicode);
  439. ans = (TokenSource*)emalloc(sizeof(TokenSource));
  440. ans->i = 0;
  441. ans->data = data;
  442. ans->edata = edata;
  443. ans->chset = chset;
  444. ans->mtype = mtype;
  445. return ans;
  446. }
  447. enum {
  448. ToksChunk = 500
  449. };
  450. // Call this to get the tokens.
  451. // The number of returned tokens is returned in *plen.
  452. Token*
  453. _gettoks(uchar* data, int datalen, int chset, int mtype, int* plen)
  454. {
  455. TokenSource* ts;
  456. Token* a;
  457. int alen;
  458. int ai;
  459. int starti;
  460. int c;
  461. int tag;
  462. if(!lexinited)
  463. lexinit();
  464. ts = newtokensource(data, datalen, chset, mtype);
  465. alen = ToksChunk;
  466. a = (Token*)emalloc(alen * sizeof(Token));
  467. ai = 0;
  468. if(dbglex)
  469. fprint(2, "_gettoks starts, ts.i=%d, ts.edata=%d\n", ts->i, ts->edata);
  470. if(ts->mtype == TextHtml) {
  471. for(;;) {
  472. if(ai == alen) {
  473. a = (Token*)erealloc(a, (alen+ToksChunk)*sizeof(Token));
  474. alen += ToksChunk;
  475. }
  476. starti = ts->i;
  477. c = getchar(ts);
  478. if(c < 0)
  479. break;
  480. if(c == '<') {
  481. tag = gettag(ts, starti, a, &ai);
  482. if(tag == Tscript) {
  483. // special rules for getting Data after....
  484. starti = ts->i;
  485. c = getchar(ts);
  486. tag = getscriptdata(ts, c, starti, a, &ai);
  487. }
  488. }
  489. else
  490. tag = getdata(ts, c, starti, a, &ai);
  491. if(tag == -1)
  492. break;
  493. else if(dbglex > 1 && tag != Comment)
  494. fprint(2, "lex: got token %T\n", &a[ai-1]);
  495. }
  496. }
  497. else {
  498. // plain text (non-html) tokens
  499. for(;;) {
  500. if(ai == alen) {
  501. a = (Token*)erealloc(a, (alen+ToksChunk)*sizeof(Token));
  502. alen += ToksChunk;
  503. }
  504. tag = getplaindata(ts, a, &ai);
  505. if(tag == -1)
  506. break;
  507. if(dbglex > 1)
  508. fprint(2, "lex: got token %T\n", &a[ai]);
  509. }
  510. }
  511. if(dbglex)
  512. fprint(2, "lex: returning %d tokens\n", ai);
  513. *plen = ai;
  514. if(ai == 0)
  515. return nil;
  516. return a;
  517. }
  518. // For case where source isn't HTML.
  519. // Just make data tokens, one per line (or partial line,
  520. // at end of buffer), ignoring non-whitespace control
  521. // characters and dumping \r's.
  522. // If find non-empty token, fill in a[*pai], bump *pai, and return Data.
  523. // Otherwise return -1;
  524. static int
  525. getplaindata(TokenSource* ts, Token* a, int* pai)
  526. {
  527. Rune* s;
  528. int j;
  529. int starti;
  530. int c;
  531. Token* tok;
  532. Rune buf[BIGBUFSIZE];
  533. s = nil;
  534. j = 0;
  535. starti = ts->i;
  536. for(c = getchar(ts); c >= 0; c = getchar(ts)) {
  537. if(c < ' ') {
  538. if(isspace(c)) {
  539. if(c == '\r') {
  540. // ignore it unless no following '\n',
  541. // in which case treat it like '\n'
  542. c = getchar(ts);
  543. if(c != '\n') {
  544. if(c >= 0)
  545. ungetchar(ts, c);
  546. c = '\n';
  547. }
  548. }
  549. }
  550. else
  551. c = 0;
  552. }
  553. if(c != 0) {
  554. buf[j++] = c;
  555. if(j == sizeof(buf)-1) {
  556. s = buftostr(s, buf, j);
  557. j = 0;
  558. }
  559. }
  560. if(c == '\n')
  561. break;
  562. }
  563. s = buftostr(s, buf, j);
  564. if(s == nil)
  565. return -1;
  566. tok = &a[(*pai)++];
  567. tok->tag = Data;
  568. tok->text = s;
  569. tok->attr = nil;
  570. tok->starti = starti;
  571. return Data;
  572. }
  573. // Return concatenation of s and buf[0:j]
  574. static Rune*
  575. buftostr(Rune* s, Rune* buf, int j)
  576. {
  577. buf[j] = 0;
  578. if(s == nil)
  579. s = _Strndup(buf, j);
  580. else
  581. s = _Strdup2(s, buf);
  582. return s;
  583. }
  584. // Gather data up to next start-of-tag or end-of-buffer.
  585. // Translate entity references (&amp;).
  586. // Ignore non-whitespace control characters and get rid of \r's.
  587. // If find non-empty token, fill in a[*pai], bump *pai, and return Data.
  588. // Otherwise return -1;
  589. static int
  590. getdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai)
  591. {
  592. Rune* s;
  593. int j;
  594. int c;
  595. Token* tok;
  596. Rune buf[BIGBUFSIZE];
  597. s = nil;
  598. j = 0;
  599. c = firstc;
  600. while(c >= 0) {
  601. if(c == '&') {
  602. c = ampersand(ts);
  603. if(c < 0)
  604. break;
  605. }
  606. else if(c < ' ') {
  607. if(isspace(c)) {
  608. if(c == '\r') {
  609. // ignore it unless no following '\n',
  610. // in which case treat it like '\n'
  611. c = getchar(ts);
  612. if(c != '\n') {
  613. if(c >= 0)
  614. ungetchar(ts, c);
  615. c = '\n';
  616. }
  617. }
  618. }
  619. else {
  620. if(warn)
  621. fprint(2, "warning: non-whitespace control character %d ignored\n", c);
  622. c = 0;
  623. }
  624. }
  625. else if(c == '<') {
  626. ungetchar(ts, c);
  627. break;
  628. }
  629. if(c != 0) {
  630. buf[j++] = c;
  631. if(j == BIGBUFSIZE-1) {
  632. s = buftostr(s, buf, j);
  633. j = 0;
  634. }
  635. }
  636. c = getchar(ts);
  637. }
  638. s = buftostr(s, buf, j);
  639. if(s == nil)
  640. return -1;
  641. tok = &a[(*pai)++];
  642. tok->tag = Data;
  643. tok->text = s;
  644. tok->attr = nil;
  645. tok->starti = starti;
  646. return Data;
  647. }
  648. // The rules for lexing scripts are different (ugh).
  649. // Gather up everything until see a </SCRIPT>.
  650. static int
  651. getscriptdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai)
  652. {
  653. Rune* s;
  654. int j;
  655. int tstarti;
  656. int savei;
  657. int c;
  658. int tag;
  659. int done;
  660. Token* tok;
  661. Rune buf[BIGBUFSIZE];
  662. s = nil;
  663. j = 0;
  664. tstarti = starti;
  665. c = firstc;
  666. done = 0;
  667. while(c >= 0) {
  668. if(c == '<') {
  669. // other browsers ignore stuff to end of line after <!
  670. savei = ts->i;
  671. c = getchar(ts);
  672. if(c == '!') {
  673. while(c >= 0 && c != '\n' && c != '\r')
  674. c = getchar(ts);
  675. if(c == '\r')
  676. c = getchar(ts);
  677. if(c == '\n')
  678. c = getchar(ts);
  679. }
  680. else if(c >= 0) {
  681. backup(ts, savei);
  682. tag = gettag(ts, tstarti, a, pai);
  683. if(tag == -1)
  684. break;
  685. if(tag != Comment)
  686. (*pai)--;
  687. backup(ts, tstarti);
  688. if(tag == Tscript + RBRA) {
  689. done = 1;
  690. break;
  691. }
  692. // here tag was not </SCRIPT>, so take as regular data
  693. c = getchar(ts);
  694. }
  695. }
  696. if(c < 0)
  697. break;
  698. if(c != 0) {
  699. buf[j++] = c;
  700. if(j == BIGBUFSIZE-1) {
  701. s = buftostr(s, buf, j);
  702. j = 0;
  703. }
  704. }
  705. tstarti = ts->i;
  706. c = getchar(ts);
  707. }
  708. if(done || ts->i == ts->edata) {
  709. s = buftostr(s, buf, j);
  710. tok = &a[(*pai)++];
  711. tok->tag = Data;
  712. tok->text = s;
  713. tok->attr = nil;
  714. tok->starti = starti;
  715. return Data;
  716. }
  717. backup(ts, starti);
  718. return -1;
  719. }
  720. // We've just seen a '<'. Gather up stuff to closing '>' (if buffer
  721. // ends before then, return -1).
  722. // If it's a tag, look up the name, gather the attributes, and return
  723. // the appropriate token.
  724. // Else it's either just plain data or some kind of ignorable stuff:
  725. // return Data or Comment as appropriate.
  726. // If it's not a Comment, put it in a[*pai] and bump *pai.
  727. static int
  728. gettag(TokenSource* ts, int starti, Token* a, int* pai)
  729. {
  730. int rbra;
  731. int ans;
  732. Attr* al;
  733. int nexti;
  734. int c;
  735. int ti;
  736. int afnd;
  737. int attid;
  738. int quote;
  739. Rune* val;
  740. int nv;
  741. int i;
  742. int tag;
  743. Token* tok;
  744. Rune buf[BIGBUFSIZE];
  745. rbra = 0;
  746. nexti = ts->i;
  747. tok = &a[*pai];
  748. tok->tag = Notfound;
  749. tok->text = nil;
  750. tok->attr = nil;
  751. tok->starti = starti;
  752. c = getchar(ts);
  753. if(c == '/') {
  754. rbra = RBRA;
  755. c = getchar(ts);
  756. }
  757. if(c < 0)
  758. goto eob_done;
  759. if(c >= 256 || !isalpha(c)) {
  760. // not a tag
  761. if(c == '!') {
  762. ans = comment(ts);
  763. if(ans != -1)
  764. return ans;
  765. goto eob_done;
  766. }
  767. else {
  768. backup(ts, nexti);
  769. tok->tag = Data;
  770. tok->text = _Strdup(L"<");
  771. (*pai)++;
  772. return Data;
  773. }
  774. }
  775. // c starts a tagname
  776. buf[0] = c;
  777. i = 1;
  778. while(1) {
  779. c = getchar(ts);
  780. if(c < 0)
  781. goto eob_done;
  782. if(!ISNAMCHAR(c))
  783. break;
  784. // if name is bigger than buf it won't be found anyway...
  785. if(i < BIGBUFSIZE)
  786. buf[i++] = c;
  787. }
  788. if(_lookup(tagtable, Numtags, buf, i, &tag))
  789. tok->tag = tag + rbra;
  790. else
  791. tok->text = _Strndup(buf, i); // for warning print, in build
  792. // attribute gathering loop
  793. al = nil;
  794. while(1) {
  795. // look for "ws name" or "ws name ws = ws val" (ws=whitespace)
  796. // skip whitespace
  797. attrloop_continue:
  798. while(c < 256 && isspace(c)) {
  799. c = getchar(ts);
  800. if(c < 0)
  801. goto eob_done;
  802. }
  803. if(c == '>')
  804. goto attrloop_done;
  805. if(c == '<') {
  806. if(warn)
  807. fprint(2, "warning: unclosed tag\n");
  808. ungetchar(ts, c);
  809. goto attrloop_done;
  810. }
  811. if(c >= 256 || !isalpha(c)) {
  812. if(warn)
  813. fprint(2, "warning: expected attribute name\n");
  814. // skipt to next attribute name
  815. while(1) {
  816. c = getchar(ts);
  817. if(c < 0)
  818. goto eob_done;
  819. if(c < 256 && isalpha(c))
  820. goto attrloop_continue;
  821. if(c == '<') {
  822. if(warn)
  823. fprint(2, "warning: unclosed tag\n");
  824. ungetchar(ts, 60);
  825. goto attrloop_done;
  826. }
  827. if(c == '>')
  828. goto attrloop_done;
  829. }
  830. }
  831. // gather attribute name
  832. buf[0] = c;
  833. i = 1;
  834. while(1) {
  835. c = getchar(ts);
  836. if(c < 0)
  837. goto eob_done;
  838. if(!ISNAMCHAR(c))
  839. break;
  840. if(i < BIGBUFSIZE-1)
  841. buf[i++] = c;
  842. }
  843. afnd = _lookup(attrtable, Numattrs, buf, i, &attid);
  844. if(warn && !afnd) {
  845. buf[i] = 0;
  846. fprint(2, "warning: unknown attribute name %S\n", buf);
  847. }
  848. // skip whitespace
  849. while(c < 256 && isspace(c)) {
  850. c = getchar(ts);
  851. if(c < 0)
  852. goto eob_done;
  853. }
  854. if(c != '=') {
  855. if(afnd)
  856. al = newattr(attid, nil, al);
  857. goto attrloop_continue;
  858. }
  859. //# c is '=' here; skip whitespace
  860. while(1) {
  861. c = getchar(ts);
  862. if(c < 0)
  863. goto eob_done;
  864. if(c >= 256 || !isspace(c))
  865. break;
  866. }
  867. quote = 0;
  868. if(c == '\'' || c == '"') {
  869. quote = c;
  870. c = getchar(ts);
  871. if(c < 0)
  872. goto eob_done;
  873. }
  874. val = nil;
  875. nv = 0;
  876. while(1) {
  877. valloop_continue:
  878. if(c < 0)
  879. goto eob_done;
  880. if(c == '>') {
  881. if(quote) {
  882. // c might be part of string (though not good style)
  883. // but if line ends before close quote, assume
  884. // there was an unmatched quote
  885. ti = ts->i;
  886. while(1) {
  887. c = getchar(ts);
  888. if(c < 0)
  889. goto eob_done;
  890. if(c == quote) {
  891. backup(ts, ti);
  892. buf[nv++] = '>';
  893. if(nv == BIGBUFSIZE-1) {
  894. val = buftostr(val, buf, nv);
  895. nv = 0;
  896. }
  897. c = getchar(ts);
  898. goto valloop_continue;
  899. }
  900. if(c == '\n') {
  901. if(warn)
  902. fprint(2, "warning: apparent unmatched quote\n");
  903. backup(ts, ti);
  904. c = '>';
  905. goto valloop_done;
  906. }
  907. }
  908. }
  909. else
  910. goto valloop_done;
  911. }
  912. if(quote) {
  913. if(c == quote) {
  914. c = getchar(ts);
  915. if(c < 0)
  916. goto eob_done;
  917. goto valloop_done;
  918. }
  919. if(c == '\r') {
  920. c = getchar(ts);
  921. goto valloop_continue;
  922. }
  923. if(c == '\t' || c == '\n')
  924. c = ' ';
  925. }
  926. else {
  927. if(c < 256 && isspace(c))
  928. goto valloop_done;
  929. }
  930. if(c == '&') {
  931. c = ampersand(ts);
  932. if(c == -1)
  933. goto eob_done;
  934. }
  935. buf[nv++] = c;
  936. if(nv == BIGBUFSIZE-1) {
  937. val = buftostr(val, buf, nv);
  938. nv = 0;
  939. }
  940. c = getchar(ts);
  941. }
  942. valloop_done:
  943. if(afnd) {
  944. val = buftostr(val, buf, nv);
  945. al = newattr(attid, val, al);
  946. }
  947. }
  948. attrloop_done:
  949. tok->attr = al;
  950. (*pai)++;
  951. return tok->tag;
  952. eob_done:
  953. if(warn)
  954. fprint(2, "warning: incomplete tag at end of page\n");
  955. backup(ts, nexti);
  956. tok->tag = Data;
  957. tok->text = _Strdup(L"<");
  958. return Data;
  959. }
  960. // We've just read a '<!' at position starti,
  961. // so this may be a comment or other ignored section, or it may
  962. // be just a literal string if there is no close before end of file
  963. // (other browsers do that).
  964. // The accepted practice seems to be (note: contrary to SGML spec!):
  965. // If see <!--, look for --> to close, or if none, > to close.
  966. // If see <!(not --), look for > to close.
  967. // If no close before end of file, leave original characters in as literal data.
  968. //
  969. // If we see ignorable stuff, return Comment.
  970. // Else return nil (caller should back up and try again when more data arrives,
  971. // unless at end of file, in which case caller should just make '<' a data token).
  972. static int
  973. comment(TokenSource* ts)
  974. {
  975. int nexti;
  976. int havecomment;
  977. int c;
  978. nexti = ts->i;
  979. havecomment = 0;
  980. c = getchar(ts);
  981. if(c == '-') {
  982. c = getchar(ts);
  983. if(c == '-') {
  984. if(findstr(ts, L"-->"))
  985. havecomment = 1;
  986. else
  987. backup(ts, nexti);
  988. }
  989. }
  990. if(!havecomment) {
  991. if(c == '>')
  992. havecomment = 1;
  993. else if(c >= 0) {
  994. if(findstr(ts, L">"))
  995. havecomment = 1;
  996. }
  997. }
  998. if(havecomment)
  999. return Comment;
  1000. return -1;
  1001. }
  1002. // Look for string s in token source.
  1003. // If found, return 1, with buffer at next char after s,
  1004. // else return 0 (caller should back up).
  1005. static int
  1006. findstr(TokenSource* ts, Rune* s)
  1007. {
  1008. int c0;
  1009. int n;
  1010. int nexti;
  1011. int i;
  1012. int c;
  1013. c0 = s[0];
  1014. n = runestrlen(s);
  1015. while(1) {
  1016. c = getchar(ts);
  1017. if(c < 0)
  1018. break;
  1019. if(c == c0) {
  1020. if(n == 1)
  1021. return 1;
  1022. nexti = ts->i;
  1023. for(i = 1; i < n; i++) {
  1024. c = getchar(ts);
  1025. if(c < 0)
  1026. goto mainloop_done;
  1027. if(c != s[i])
  1028. break;
  1029. }
  1030. if(i == n)
  1031. return 1;
  1032. backup(ts, nexti);
  1033. }
  1034. }
  1035. mainloop_done:
  1036. return 0;
  1037. }
  1038. // We've just read an '&'; look for an entity reference
  1039. // name, and if found, return translated char.
  1040. // if there is a complete entity name but it isn't known,
  1041. // try prefixes (gets around some buggy HTML out there),
  1042. // and if that fails, back up to just past the '&' and return '&'.
  1043. // If the entity can't be completed in the current buffer, back up
  1044. // to the '&' and return -1.
  1045. static int
  1046. ampersand(TokenSource* ts)
  1047. {
  1048. int savei;
  1049. int c;
  1050. int fnd;
  1051. int ans;
  1052. int v;
  1053. int i;
  1054. int k;
  1055. Rune buf[SMALLBUFSIZE];
  1056. savei = ts->i;
  1057. c = getchar(ts);
  1058. fnd = 0;
  1059. ans = -1;
  1060. if(c == '#') {
  1061. c = getchar(ts);
  1062. v = 0;
  1063. while(c >= 0) {
  1064. if(!(c < 256 && isdigit(c)))
  1065. break;
  1066. v = v*10 + c - 48;
  1067. c = getchar(ts);
  1068. }
  1069. if(c >= 0) {
  1070. if(!(c == ';' || c == '\n' || c == '\r'))
  1071. ungetchar(ts, c);
  1072. c = v;
  1073. if(c == 160)
  1074. c = 160;
  1075. if(c >= Winstart && c <= Winend) {
  1076. c = winchars[c - Winstart];
  1077. }
  1078. ans = c;
  1079. fnd = 1;
  1080. }
  1081. }
  1082. else if(c < 256 && isalpha(c)) {
  1083. buf[0] = c;
  1084. k = 1;
  1085. while(1) {
  1086. c = getchar(ts);
  1087. if(c < 0)
  1088. break;
  1089. if(ISNAMCHAR(c)) {
  1090. if(k < SMALLBUFSIZE-1)
  1091. buf[k++] = c;
  1092. }
  1093. else {
  1094. if(!(c == ';' || c == '\n' || c == '\r'))
  1095. ungetchar(ts, c);
  1096. break;
  1097. }
  1098. }
  1099. if(c >= 0) {
  1100. fnd = _lookup(chartab, NCHARTAB, buf, k, &ans);
  1101. if(!fnd) {
  1102. // Try prefixes of s
  1103. if(c == ';' || c == '\n' || c == '\r')
  1104. ungetchar(ts, c);
  1105. i = k;
  1106. while(--k > 0) {
  1107. fnd = _lookup(chartab, NCHARTAB, buf, k, &ans);
  1108. if(fnd) {
  1109. while(i > k) {
  1110. i--;
  1111. ungetchar(ts, buf[i]);
  1112. }
  1113. break;
  1114. }
  1115. }
  1116. }
  1117. }
  1118. }
  1119. if(!fnd) {
  1120. backup(ts, savei);
  1121. ans = '&';
  1122. }
  1123. return ans;
  1124. }
  1125. // Get next char, obeying ts.chset.
  1126. // Returns -1 if no complete character left before current end of data.
  1127. static int
  1128. getchar(TokenSource* ts)
  1129. {
  1130. uchar* buf;
  1131. int c;
  1132. int n;
  1133. int ok;
  1134. Rune r;
  1135. if(ts->i >= ts->edata)
  1136. return -1;
  1137. buf = ts->data;
  1138. c = buf[ts->i];
  1139. switch(ts->chset) {
  1140. case ISO_8859_1:
  1141. if(c >= Winstart && c <= Winend)
  1142. c = winchars[c - Winstart];
  1143. ts->i++;
  1144. break;
  1145. case US_Ascii:
  1146. if(c > 127) {
  1147. if(warn)
  1148. fprint(2, "non-ascii char (%x) when US-ASCII specified\n", c);
  1149. }
  1150. ts->i++;
  1151. break;
  1152. case UTF_8:
  1153. ok = fullrune((char*)(buf+ts->i), ts->edata-ts->i);
  1154. n = chartorune(&r, (char*)(buf+ts->i));
  1155. if(ok) {
  1156. if(warn && c == 0x80)
  1157. fprint(2, "warning: invalid utf-8 sequence (starts with %x)\n", ts->data[ts->i]);
  1158. ts->i += n;
  1159. c = r;
  1160. }
  1161. else {
  1162. // not enough bytes in buf to complete utf-8 char
  1163. ts->i = ts->edata; // mark "all used"
  1164. c = -1;
  1165. }
  1166. break;
  1167. case Unicode:
  1168. if(ts->i < ts->edata - 1) {
  1169. //standards say most-significant byte first
  1170. c = (c << 8)|(buf[ts->i + 1]);
  1171. ts->i += 2;
  1172. }
  1173. else {
  1174. ts->i = ts->edata; // mark "all used"
  1175. c = -1;
  1176. }
  1177. break;
  1178. }
  1179. return c;
  1180. }
  1181. // Assuming c was the last character returned by getchar, set
  1182. // things up so that next getchar will get that same character
  1183. // followed by the current 'next character', etc.
  1184. static void
  1185. ungetchar(TokenSource* ts, int c)
  1186. {
  1187. int n;
  1188. Rune r;
  1189. char a[UTFmax];
  1190. n = 1;
  1191. switch(ts->chset) {
  1192. case UTF_8:
  1193. if(c >= 128) {
  1194. r = c;
  1195. n = runetochar(a, &r);
  1196. }
  1197. break;
  1198. case Unicode:
  1199. n = 2;
  1200. break;
  1201. }
  1202. ts->i -= n;
  1203. }
  1204. // Restore ts so that it is at the state where the index was savei.
  1205. static void
  1206. backup(TokenSource* ts, int savei)
  1207. {
  1208. if(dbglex)
  1209. fprint(2, "lex: backup; i=%d, savei=%d\n", ts->i, savei);
  1210. ts->i = savei;
  1211. }
  1212. // Look for value associated with attribute attid in token t.
  1213. // If there is one, return 1 and put the value in *pans,
  1214. // else return 0.
  1215. // If xfer is true, transfer ownership of the string to the caller
  1216. // (nil it out here); otherwise, caller must duplicate the answer
  1217. // if it needs to save it.
  1218. // OK to have pans==0, in which case this is just looking
  1219. // to see if token is present.
  1220. int
  1221. _tokaval(Token* t, int attid, Rune** pans, int xfer)
  1222. {
  1223. Attr* attr;
  1224. attr = t->attr;
  1225. while(attr != nil) {
  1226. if(attr->attid == attid) {
  1227. if(pans != nil)
  1228. *pans = attr->value;
  1229. if(xfer)
  1230. attr->value = nil;
  1231. return 1;
  1232. }
  1233. attr = attr->next;
  1234. }
  1235. if(pans != nil)
  1236. *pans = nil;
  1237. return 0;
  1238. }
  1239. static int
  1240. Tconv(Fmt *f)
  1241. {
  1242. Token* t;
  1243. int i;
  1244. int tag;
  1245. char* srbra;
  1246. Rune* aname;
  1247. Rune* tname;
  1248. Attr* a;
  1249. char buf[BIGBUFSIZE];
  1250. t = va_arg(f->args, Token*);
  1251. if(t == nil)
  1252. sprint(buf, "<null>");
  1253. else {
  1254. i = 0;
  1255. if(dbglex > 1)
  1256. i = snprint(buf, sizeof(buf), "[%d]", t->starti);
  1257. tag = t->tag;
  1258. if(tag == Data) {
  1259. i += snprint(buf+i, sizeof(buf)-i-1, "'%S'", t->text);
  1260. }
  1261. else {
  1262. srbra = "";
  1263. if(tag >= RBRA) {
  1264. tag -= RBRA;
  1265. srbra = "/";
  1266. }
  1267. tname = tagnames[tag];
  1268. if(tag == Notfound)
  1269. tname = L"?";
  1270. i += snprint(buf+i, sizeof(buf)-i-1, "<%s%S", srbra, tname);
  1271. for(a = t->attr; a != nil; a = a->next) {
  1272. aname = attrnames[a->attid];
  1273. i += snprint(buf+i, sizeof(buf)-i-1, " %S", aname);
  1274. if(a->value != nil)
  1275. i += snprint(buf+i, sizeof(buf)-i-1, "=%S", a->value);
  1276. }
  1277. i += snprint(buf+i, sizeof(buf)-i-1, ">");
  1278. }
  1279. buf[i] = 0;
  1280. }
  1281. return fmtstrcpy(f, buf);
  1282. }
  1283. // Attrs own their constituent strings, but build may eventually
  1284. // transfer some values to its items and nil them out in the Attr.
  1285. static Attr*
  1286. newattr(int attid, Rune* value, Attr* link)
  1287. {
  1288. Attr* ans;
  1289. ans = (Attr*)emalloc(sizeof(Attr));
  1290. ans->attid = attid;
  1291. ans->value = value;
  1292. ans->next = link;
  1293. return ans;
  1294. }
  1295. // Free list of Attrs linked through next field
  1296. static void
  1297. freeattrs(Attr* ahead)
  1298. {
  1299. Attr* a;
  1300. Attr* nexta;
  1301. a = ahead;
  1302. while(a != nil) {
  1303. nexta = a->next;
  1304. free(a->value);
  1305. free(a);
  1306. a = nexta;
  1307. }
  1308. }
  1309. // Free array of Tokens.
  1310. // Allocated space might have room for more than n tokens,
  1311. // but only n of them are initialized.
  1312. // If caller has transferred ownership of constitutent strings
  1313. // or attributes, it must have nil'd out the pointers in the Tokens.
  1314. void
  1315. _freetokens(Token* tarray, int n)
  1316. {
  1317. int i;
  1318. Token* t;
  1319. if(tarray == nil)
  1320. return;
  1321. for(i = 0; i < n; i++) {
  1322. t = &tarray[i];
  1323. free(t->text);
  1324. freeattrs(t->attr);
  1325. }
  1326. free(tarray);
  1327. }