dparse.trivia source code

1 /**
2  * Module to work with trivia tokens (`comment`, `whitespace`,
3  * `specialTokenSequence`) which are attached to tokens near them when source
4  * code gets tokenized.
5  */
6 module dparse.trivia;
7 
8 import std.algorithm;
9 import std.array;
10 import std.range;
11 import std..string;
12 import std.traits;
13 
14 import dparse.lexer;
15 
16 enum CommentType : ubyte
17 {
18     none,
19     docLine,
20     docBlock,
21     normalLine,
22     normalBlock,
23 }
24 
25 CommentType determineCommentType(string comment) pure nothrow @safe
26 {
27     auto bytes = comment.representation;
28     auto index = bytes.startsWith(
29         "//".representation,
30         "/+".representation,
31         "/*".representation
32     );
33     bool isDoc = bytes.length >= 3 && bytes[1] == bytes[2];
34     switch (index)
35     {
36     case 1:
37         // Don't treat "////////...." comments as doc comments
38         isDoc = isDoc && (bytes.length == 3 || bytes[3..$].any!(c => c != '/'));
39         return isDoc ? CommentType.docLine :  CommentType.normalLine;
40     case 2:
41     case 3:
42         return isDoc ? CommentType.docBlock :  CommentType.normalBlock;
43     default:
44         return CommentType.none;
45     }
46 }
47 
48 ///
49 unittest
50 {
51     assert (determineCommentType("/// hello") == CommentType.docLine);
52     assert (determineCommentType("/++ hello") == CommentType.docBlock);
53     assert (determineCommentType("/** hello") == CommentType.docBlock);
54     assert (determineCommentType("// hello") == CommentType.normalLine);
55     assert (determineCommentType("/+ hello") == CommentType.normalBlock);
56     assert (determineCommentType("/* hello") == CommentType.normalBlock);
57     assert (determineCommentType("/ hello") == CommentType.none);
58     assert (determineCommentType("/") == CommentType.none);
59 
60     assert (determineCommentType("////////////////////") == CommentType.normalLine);
61     assert (determineCommentType("///") == CommentType.docLine);
62     assert (determineCommentType("///   ") == CommentType.docLine);
63 }
64 
65 bool isDocComment(CommentType type) @safe nothrow pure
66 {
67     return type == CommentType.docLine || type == CommentType.docBlock;
68 }
69 
70 /**
71  * Removes "decoration" such as leading whitespace, leading + and * characters,
72  * and places the result into the given output range
73  */
74 public void unDecorateComment(T)(string comment, auto ref T outputRange)
75 if (isOutputRange!(T, string))
76 in
77 {
78     assert (comment.length >= 3);
79 }
80 do
81 {
82     import std..string : chompPrefix, KeepTerminator, lineSplitter, stripRight;
83 
84     string leadingChars;
85 
86     enum LineType { none, normal, strange }
87     LineType prevLineType;
88 
89     switch (comment[0 .. 3])
90     {
91     case "///":
92         foreach (line; lineSplitter!(KeepTerminator.yes)(comment))
93         {
94             if (leadingChars.empty)
95             {
96                 size_t k = 3;
97                 while (k < line.length && (line[k] == ' ' || line[k] == '\t'))
98                     k++;
99                 leadingChars = line[0 .. k];
100             }
101             outputRange.put(line.chompPrefix(leadingChars));
102         }
103         break;
104     case "/++":
105     case "/**":
106         alias CL = MultiLineCommentHelper!(ElementEncodingType!(typeof(comment)));
107         CL cl = CL(comment);
108         cl.process(outputRange);
109         break;
110     default:
111         outputRange.put(comment);
112     }
113 }
114 
115 ///
116 unittest
117 {
118     import std.array:array, appender;
119     import std.stdio:stderr;
120     stderr.writeln("Running unittest for unDecorateComment...");
121 
122     string[] inputs = [
123         "/***************\n*******************/",
124         "/***************\n *\n ******************/",
125         "/**\n*/",
126         "/** */",
127         "/***/",
128         "/******/",
129         "/** abcde1 */",
130         "/// abcde2\n/// abcde2",
131         "/**\n * stuff1\n */",
132         "/**\n *\n * stuff2\n */",
133         "/**\n *\n * stuff3\n *\n */",
134         "/**\n *\n * stuff4\n *\n*/",
135         "/**\n *  abcde3\n *    abcde3 \n */",
136         "/**\n * abcde4\n *\n * abcde4\n */",
137         "/**abcde5\n*abcde5\n*/",
138         "/** abcde6\n * abcde6\n*/",
139         "/**\n1\n\n\n\n*/",
140         "/**\r\n1\r\n\r\n\r\n\r\n*/",
141         "/**\na1\n\na2\n\n*/",
142         "/**b1\n*b2\n*b3*/",
143         "/**c1\n    *c2\n    *c3*/",
144         "/**d1\n    *d2\n    *d3\n*/",
145         "///a\fbc\n///def"
146     ];
147     string[] outputs = [
148         "",
149         "",
150         "",
151         "",
152         "",
153         "",
154         "abcde1",
155         "abcde2\nabcde2",
156         "stuff1",
157         "stuff2",
158         "stuff3",
159         "stuff4",
160         "abcde3\n  abcde3",
161         "abcde4\n\nabcde4",
162         "abcde5\nabcde5",
163         "abcde6\nabcde6",
164         "1",
165         "1",
166         "a1\n\na2",
167         "b1\nb2\nb3",
168         "c1\nc2\nc3",
169         "d1\nd2\nd3",
170         "a\fbc\ndef"
171     ];
172 
173     // tests where * and + are not interchangeable
174     string[2][] np =
175     [
176         ["/**\n * d1\n d2\n */", "* d1\nd2"],
177         ["/**\n + d1\n d2\n */", "+ d1\nd2"],
178         ["/**d1\n\n\n*d2\n*/",  "d1\n\n*d2"],
179     ];
180 
181     assert(inputs.length == outputs.length);
182     foreach (pair; zip(inputs, outputs))
183     {
184         foreach (b; [true, false])
185         {
186             auto app = appender!string();
187             unDecorateComment(b ? pair[0] : pair[0].replace("*", "+"), app);
188             assert(pair[1] == app.data, "[[" ~ pair[0] ~ "]] => [[" ~ app.data ~ "]]");
189         }
190     }
191     foreach (pair; np)
192     {
193         auto app = appender!string();
194         unDecorateComment(pair[0], app);
195         assert(pair[1] == app.data, "[[" ~ pair[0] ~ "]] => [[" ~ app.data ~ "]]");
196     }
197     stderr.writeln("Unittest for unDecorateComment passed.");
198 }
199 
200 /** Gives a line per line view on DDOC comments of type `/++` and `/**` which
201  * makes easier to remove the decoration and in an almost 100% nogc way. */
202 private struct MultiLineCommentHelper(CharType : const(char))
203 {
204     // this struct is more used as a 'function with nested functions' would.
205     this() @disable;
206     this(this) @disable;
207     auto opAssign(T)(T t) @disable;
208 
209 private:
210 
211     char[][] lines;
212     // either lines.length or lines.length-1, depending on if last line only closes
213     size_t lastLineInBlockPlusOne;
214     // either '*' or '+'
215     const(char) commentChar;
216     // either 0 or 1, depending on if first line only opens
217     ubyte firstLineInBlock;
218 
219     import std.ascii : isWhite;
220 
221     void stripIndent() @safe @nogc pure nothrow
222     {
223         if (lines.length < 2)
224             return;
225         size_t count;
226         foreach (const j; 0 .. lines[1].length)
227             if (!(lines[1][j]).isWhite)
228         {
229             count = j;
230             break;
231         }
232         if (count < 2)
233             return;
234         foreach (ref line; lines[1 .. $])
235         {
236             foreach (const j; 0 .. line.length)
237             {
238                 if (!(line[j]).isWhite)
239                     break;
240                 if (j == count - 1)
241                 {
242                     line = line[j .. $];
243                     break;
244                 }
245             }
246         }
247     }
248 
249     void processFirstLine() @safe @nogc pure nothrow
250     {
251         assert(lines.length);
252         if (lines[0].length > 3)
253         {
254             foreach (const i; 1..lines[0].length)
255             {
256                 if (lines[0][i] == commentChar)
257                 {
258                     if (i < lines[0].length - 2)
259                         continue;
260                     if (i == lines[0].length - 2 && lines[0][i+1] == '/')
261                     {
262                         lines[0][] = ' ';
263                         break;
264                     }
265                     if (i == lines[0].length - 1)
266                     {
267                         lines[0][] = ' ';
268                         break;
269                     }
270                 }
271                 else
272                 {
273                     lines[0][0..i] = ' ';
274                     break;
275                 }
276             }
277         }
278         lines[0][0..3] = "   ";
279         if (lines.length == 1 &&
280             lines[0][$-2] == commentChar && lines[0][$-1] == '/')
281         {
282             lines[0][$-2..$] = "  ";
283         }
284         foreach (const i; 0..lines[0].length)
285             if (!(lines[0][i].isWhite))
286                 return;
287         firstLineInBlock = 1;
288     }
289 
290     void processLastLine() @safe @nogc pure nothrow
291     {
292         lastLineInBlockPlusOne = lines.length;
293         if (lines.length == 1)
294             return;
295         size_t closeStartIndex = size_t.max;
296         foreach (const i; 0..lines[$-1].length)
297         {
298             if (lines[$-1][i] == commentChar)
299             {
300                 if (closeStartIndex == size_t.max)
301                     closeStartIndex = i;
302                 if (i == lines[$-1].length - 2)
303                 {
304                     // see the FIXME note in unDecorate()
305                     lastLineInBlockPlusOne = closeStartIndex == 0 ? lines.length-1 : lines.length;
306 
307                     lines[$-1][closeStartIndex..$] = ' ';
308                     break;
309                 }
310             }
311             else
312             {
313                 closeStartIndex = size_t.max;
314                 lastLineInBlockPlusOne = lines.length;
315             }
316         }
317     }
318 
319     void unDecorate() @safe @nogc pure nothrow
320     {
321         if (lines.length == 1 || lines.length == 2 && lines[$-1].length == 0)
322             return;
323         bool allDecorated;
324         static immutable char[2][2] pattern = [[' ', '*'],[' ', '+']];
325         const ubyte patternIndex = commentChar == '+';
326         // first line is never decorated
327         const size_t lo = 1;
328         // although very uncommon, the last line can be decorated e.g in `* lastline */`:
329         // the first '*' is a deco if all prev lines are also decorated.
330         // FIXME: `hi` should be set to `lastLineInBlockPlusOne`...
331         const size_t hi = (lines[$-1].length > 1 &&
332             (lines[$-1][0] == commentChar || lines[$-1][0..2] == pattern[patternIndex]))
333             ?  lines.length : lines.length-1;
334         // deco with a leading white
335         foreach (const i; lo .. hi)
336         {
337             if (lines[i].length < 2)
338                 break;
339             else if (lines[i][0..2] != pattern[patternIndex])
340                 break;
341             else if (i == hi-1)
342                 allDecorated = true;
343         }
344         // deco w/o leading white
345         if (!allDecorated)
346             foreach (const i; lo .. hi)
347         {
348             if (lines[i].length == 0)
349                 break;
350             if (lines[i][0] != commentChar)
351                 break;
352             else if (i == hi-1)
353                 allDecorated = true;
354         }
355         if (!allDecorated)
356             return;
357 
358         const size_t indexToChange = (lines[lo][0] == commentChar) ? 0 : 1;
359         foreach (ref line; lines[lo .. hi])
360             line[indexToChange] = ' ';
361     }
362 
363     void stripLeft() @safe @nogc pure nothrow
364     {
365         foreach (const i; 0 .. lines[0].length)
366             if (!(lines[0][i]).isWhite)
367         {
368             lines[0] = lines[0][i..$];
369             break;
370         }
371         if (lines.length == 1)
372             return;
373         while (true)
374         {
375             bool processColumn;
376             foreach (ref line; lines[1 .. lastLineInBlockPlusOne])
377             {
378                 if (line.length == 0)
379                     continue;
380                 if (!(line[0]).isWhite)
381                     return;
382                 processColumn = true;
383             }
384             if (!processColumn)
385                 return;
386             foreach (ref line; lines[1 .. lastLineInBlockPlusOne])
387             {
388                 if (line.length == 0)
389                     continue;
390                 line = line[1..$];
391             }
392         }
393     }
394 
395     void stripRight() @safe @nogc pure nothrow
396     {
397         foreach (ref line; lines[0 .. lines.length])
398         {
399             if (line.length == 0)
400                 continue;
401             if ((line[$-1]).isWhite)
402             {
403                 size_t firstWhite = line.length;
404                 while (firstWhite > 0 && (line[firstWhite-1]).isWhite)
405                     firstWhite--;
406                 line = line[0..firstWhite];
407             }
408         }
409     }
410 
411     void run() @safe @nogc pure nothrow
412     {
413         stripIndent();
414         processFirstLine();
415         processLastLine();
416         unDecorate();
417         stripLeft();
418         stripRight();
419     }
420 
421 public:
422 
423     this(CharType[] text) @safe pure nothrow
424     {
425         assert(text.length >= 3 && text[0] == '/',
426             "MultiLineCommentHelper text must start with a comment in form /++ or /**");
427 
428         commentChar = text[1];
429         size_t startIndex, i;
430         Appender!(char[][]) linesApp;
431         linesApp.reserve(512);
432 
433         void storeLine(size_t endIndexPlusOne)
434         {
435             static if (isMutable!CharType)
436                 linesApp ~= text[startIndex..endIndexPlusOne];
437             else
438                 linesApp ~= text[startIndex..endIndexPlusOne].dup;
439         }
440 
441         // if we go over text length (in \r\n) we already stored the line, so just exit there
442         while (i < text.length)
443         {
444             // check if next char is going to be end of text, store until then & break
445             if (i + 1 == text.length)
446             {
447                 storeLine(text.length);
448                 break;
449             }
450             if (text[i] == '\n')
451             {
452                 storeLine(i);
453                 startIndex = i + 1;
454             }
455             else if (i + 1 < text.length && text[i .. i+2] == "\r\n")
456             {
457                 storeLine(i);
458                 i++;
459                 startIndex = i + 1;
460             }
461             i++;
462         }
463         lines = linesApp.data;
464     }
465 
466     void process(T)(ref T outbuffer)
467     {
468         run();
469         outbuffer.reserve(lines.length * 90);
470         bool prevWritten, empties;
471         foreach (ref line; lines[firstLineInBlock .. lines.length])
472         {
473             if (line.length != 0)
474             {
475                 // close preceeding line
476                 if (prevWritten)
477                     outbuffer ~= "\n";
478                 // insert new empty line
479                 if (prevWritten && empties)
480                     outbuffer ~= "\n";
481 
482                 outbuffer ~= line;
483                 prevWritten = true;
484                 empties = false;
485             }
486             else empties = true;
487         }
488     }
489 }
490 
491 unittest
492 {
493     import std.conv : to;
494 
495     alias SC = MultiLineCommentHelper!(immutable(char));
496 
497     // checks full comment processing on the given string and compares the generated lines
498     void check(string comment, string[] lines, size_t lineNo = __LINE__)
499     {
500         auto sc = SC(comment);
501         sc.run();
502         assert(sc.lines == lines, sc.lines.to!string ~ " != " ~ lines.to!string
503             ~ " (for check on line " ~ lineNo.to!string ~ ")");
504     }
505 
506     // check common cases while typing
507     check("/++", [""]);
508     check("/++\r", [""]);
509     check("/++\n", [""]);
510     check("/++\r\n", [""]);
511     check("/++\r\n+", ["", "+"]);
512     check("/++\r\n+ ok", ["", "ok"]);
513     check("/++\r\n+ ok\r\n+/", ["", "ok", ""]);
514     check("/++/", [""]);
515 }
516 
517 /// Extracts and combines ddoc comments from trivia comments.
518 string extractDdocFromTrivia(Tokens)(Tokens tokens) pure nothrow @safe
519     if (isInputRange!Tokens && is(ElementType!Tokens : Token))
520 {
521     bool hasDoc;
522     auto ret = appender!string;
523     foreach (trivia; tokens)
524     {
525         if (trivia.type == tok!"comment"
526             && trivia.text.determineCommentType.isDocComment)
527         {
528             hasDoc = true;
529             if (!ret.data.empty)
530                 ret.put('\n');
531             unDecorateComment(trivia.text, ret);
532         }
533     }
534 
535     if (ret.data.length)
536         return ret.data;
537     else
538         return hasDoc ? "" : null;
539 }
540 
541 unittest
542 {
543     Token[] tokens = [
544         Token(cast(ubyte) tok!"whitespace", "\n\n", 0, 0, 0),
545         Token(cast(ubyte) tok!"comment", "///", 0, 0, 0),
546         Token(cast(ubyte) tok!"whitespace", "\n", 0, 0, 0)
547     ];
548 
549     // Empty comment is non-null
550     auto comment = extractDdocFromTrivia(tokens);
551     assert(comment !is null);
552     assert(comment == "");
553 
554     // Missing comment is null
555     comment = extractDdocFromTrivia(tokens[0 .. 1]);
556     assert(comment is null);
557     assert(comment == "");
558 }
559 
560 string extractLeadingDdoc(const Token token) pure nothrow @safe
561 {
562     return extractDdocFromTrivia(token.leadingTrivia);
563 }
564 
565 string extractTrailingDdoc(const Token token) pure nothrow @safe
566 {
567     return extractDdocFromTrivia(token.trailingTrivia.filter!(a => a.line == token.line));
568 }
569 
570 // test token trivia members
571 unittest
572 {
573     import std.conv : to;
574     import std.exception : enforce;
575 
576     static immutable src = `/// this is a module.
577 // mixed
578 /// it can do stuff
579 module foo.bar;
580 
581 // hello
582 
583 /**
584  * some doc
585  * hello
586  */
587 int x; /// very nice
588 
589 // TODO: do stuff
590 void main() {
591     #line 40
592     /// could be better
593     writeln(":)");
594 }
595 
596 ///
597 unittest {}
598 
599 /// end of file`;
600 
601     LexerConfig cf;
602     StringCache ca = StringCache(16);
603 
604     const tokens = getTokensForParser(src, cf, &ca);
605 
606     assert(tokens.length == 22);
607 
608     assert(tokens[0].type == tok!"module");
609     assert(tokens[0].leadingTrivia.length == 6);
610     assert(tokens[0].leadingTrivia[0].type == tok!"comment");
611     assert(tokens[0].leadingTrivia[0].text == "/// this is a module.");
612     assert(tokens[0].leadingTrivia[1].type == tok!"whitespace");
613     assert(tokens[0].leadingTrivia[1].text == "\n");
614     assert(tokens[0].leadingTrivia[2].type == tok!"comment");
615     assert(tokens[0].leadingTrivia[2].text == "// mixed");
616     assert(tokens[0].leadingTrivia[3].type == tok!"whitespace");
617     assert(tokens[0].leadingTrivia[3].text == "\n");
618     assert(tokens[0].leadingTrivia[4].type == tok!"comment");
619     assert(tokens[0].leadingTrivia[4].text == "/// it can do stuff");
620     assert(tokens[0].leadingTrivia[5].type == tok!"whitespace");
621     assert(tokens[0].leadingTrivia[5].text == "\n");
622     assert(tokens[0].trailingTrivia.length == 1);
623     assert(tokens[0].trailingTrivia[0].type == tok!"whitespace");
624     assert(tokens[0].trailingTrivia[0].text == " ");
625 
626     assert(tokens[1].type == tok!"identifier");
627     assert(tokens[1].text == "foo");
628     assert(!tokens[1].leadingTrivia.length);
629     assert(!tokens[1].trailingTrivia.length);
630 
631     assert(tokens[2].type == tok!".");
632     assert(!tokens[2].leadingTrivia.length);
633     assert(!tokens[2].trailingTrivia.length);
634 
635     assert(tokens[3].type == tok!"identifier");
636     assert(tokens[3].text == "bar");
637     assert(!tokens[3].leadingTrivia.length);
638     assert(!tokens[3].trailingTrivia.length);
639 
640     assert(tokens[4].type == tok!";");
641     assert(!tokens[4].leadingTrivia.length);
642     assert(tokens[4].trailingTrivia.length == 1);
643     assert(tokens[4].trailingTrivia[0].type == tok!"whitespace");
644     assert(tokens[4].trailingTrivia[0].text == "\n\n");
645 
646     assert(tokens[5].type == tok!"int");
647     assert(tokens[5].leadingTrivia.length == 4);
648     assert(tokens[5].leadingTrivia[0].text == "// hello");
649     assert(tokens[5].leadingTrivia[1].text == "\n\n");
650     assert(tokens[5].leadingTrivia[2].text == "/**\n * some doc\n * hello\n */");
651     assert(tokens[5].leadingTrivia[3].text == "\n");
652     assert(tokens[5].trailingTrivia.length == 1);
653     assert(tokens[5].trailingTrivia[0].text == " ");
654 
655     assert(tokens[6].type == tok!"identifier");
656     assert(tokens[6].text == "x");
657     assert(!tokens[6].leadingTrivia.length);
658     assert(!tokens[6].trailingTrivia.length);
659 
660     assert(tokens[7].type == tok!";");
661     assert(!tokens[7].leadingTrivia.length);
662     assert(tokens[7].trailingTrivia.length == 3);
663     assert(tokens[7].trailingTrivia[0].text == " ");
664     assert(tokens[7].trailingTrivia[1].text == "/// very nice");
665     assert(tokens[7].trailingTrivia[2].text == "\n\n");
666 
667     assert(tokens[8].type == tok!"void");
668     assert(tokens[8].leadingTrivia.length == 2);
669     assert(tokens[8].leadingTrivia[0].text == "// TODO: do stuff");
670     assert(tokens[8].leadingTrivia[1].text == "\n");
671     assert(tokens[8].trailingTrivia.length == 1);
672     assert(tokens[8].trailingTrivia[0].text == " ");
673 
674     assert(tokens[9].type == tok!"identifier");
675     assert(tokens[9].text == "main");
676     assert(!tokens[9].leadingTrivia.length);
677     assert(!tokens[9].trailingTrivia.length);
678 
679     assert(tokens[10].type == tok!"(");
680     assert(!tokens[10].leadingTrivia.length);
681     assert(!tokens[10].trailingTrivia.length);
682 
683     assert(tokens[11].type == tok!")");
684     assert(!tokens[11].leadingTrivia.length);
685     assert(tokens[11].trailingTrivia.length == 1);
686     assert(tokens[11].trailingTrivia[0].text == " ");
687 
688     assert(tokens[12].type == tok!"{");
689     assert(!tokens[12].leadingTrivia.length);
690     assert(tokens[12].trailingTrivia.length == 1);
691     assert(tokens[12].trailingTrivia[0].text == "\n    ");
692 
693     assert(tokens[13].type == tok!"identifier");
694     assert(tokens[13].text == "writeln");
695     assert(tokens[13].leadingTrivia.length == 4);
696     assert(tokens[13].leadingTrivia[0].type == tok!"specialTokenSequence");
697     assert(tokens[13].leadingTrivia[0].text == "#line 40");
698     assert(tokens[13].leadingTrivia[1].type == tok!"whitespace");
699     assert(tokens[13].leadingTrivia[1].text == "\n    ");
700     assert(tokens[13].leadingTrivia[2].type == tok!"comment");
701     assert(tokens[13].leadingTrivia[2].text == "/// could be better");
702     assert(tokens[13].leadingTrivia[3].type == tok!"whitespace");
703     assert(tokens[13].leadingTrivia[3].text == "\n    ");
704     assert(!tokens[13].trailingTrivia.length);
705 
706     assert(tokens[14].type == tok!"(");
707     assert(!tokens[14].leadingTrivia.length);
708     assert(!tokens[14].trailingTrivia.length);
709 
710     assert(tokens[15].type == tok!"stringLiteral");
711     assert(!tokens[15].leadingTrivia.length);
712     assert(!tokens[15].trailingTrivia.length);
713 
714     assert(tokens[16].type == tok!")");
715     assert(!tokens[16].leadingTrivia.length);
716     assert(!tokens[16].trailingTrivia.length);
717 
718     assert(tokens[17].type == tok!";");
719     assert(!tokens[17].leadingTrivia.length);
720     assert(tokens[17].trailingTrivia.length == 1);
721     assert(tokens[17].trailingTrivia[0].text == "\n");
722 
723     assert(tokens[18].type == tok!"}");
724     assert(!tokens[18].leadingTrivia.length);
725     assert(tokens[18].trailingTrivia.length == 1);
726     assert(tokens[18].trailingTrivia[0].type == tok!"whitespace");
727     assert(tokens[18].trailingTrivia[0].text == "\n\n");
728 
729     assert(tokens[19].type == tok!"unittest");
730     assert(tokens[19].leadingTrivia.length == 2);
731     assert(tokens[19].leadingTrivia[0].type == tok!"comment");
732     assert(tokens[19].leadingTrivia[0].text == "///");
733     assert(tokens[19].leadingTrivia[1].type == tok!"whitespace");
734     assert(tokens[19].leadingTrivia[1].text == "\n");
735 
736     assert(tokens[19].trailingTrivia.length == 1);
737     assert(tokens[19].trailingTrivia[0].type == tok!"whitespace");
738     assert(tokens[19].trailingTrivia[0].text == " ");
739 
740     assert(tokens[20].type == tok!"{");
741     assert(!tokens[20].leadingTrivia.length);
742     assert(!tokens[20].trailingTrivia.length);
743 
744     assert(tokens[21].type == tok!"}");
745     assert(!tokens[21].leadingTrivia.length);
746     assert(tokens[21].trailingTrivia.length == 2);
747     assert(tokens[21].trailingTrivia[0].type == tok!"whitespace");
748     assert(tokens[21].trailingTrivia[0].text == "\n\n");
749     assert(tokens[21].trailingTrivia[1].type == tok!"comment");
750     assert(tokens[21].trailingTrivia[1].text == "/// end of file");
751 }