ByteCombine

    import std.conv : to;
    import std.exception : enforce;

    static immutable src = `module foo.bar;

void main() {
    x = '';
}
`;

    LexerConfig cf;
    StringCache ca = StringCache(16);

    const tokens = getTokensForParser(src, cf, &ca);

    int i;
    assert(tokens[i++].type == tok!"module");
    assert(tokens[i++].type == tok!"identifier");
    assert(tokens[i++].type == tok!".");
    assert(tokens[i++].type == tok!"identifier");
    assert(tokens[i++].type == tok!";");
    assert(tokens[i++].type == tok!"void");
    assert(tokens[i++].type == tok!"identifier");
    assert(tokens[i++].type == tok!"(");
    assert(tokens[i++].type == tok!")");
    assert(tokens[i++].type == tok!"{");
    assert(tokens[i++].type == tok!"identifier");
    assert(tokens[i++].type == tok!"=");
    assert(tokens[i].type == tok!"");
    assert(tokens[i].line == tokens[i - 1].line);
    assert(tokens[i].column == tokens[i - 1].column + 2);
    i++;
    assert(tokens[i++].type == tok!";");
    assert(tokens[i++].type == tok!"}");

    void checkInvalidTrailingString(const Token[] tokens)
    {
        assert(tokens.length == 3);
        assert(tokens[2].index != 0);
        assert(tokens[2].column >= 4);
        assert(tokens[2].type == tok!"");
    }

    checkInvalidTrailingString(getTokensForParser(`x = "foo`, cf, &ca));
    checkInvalidTrailingString(getTokensForParser(`x = r"foo`, cf, &ca));
    checkInvalidTrailingString(getTokensForParser(`x = x"00`, cf, &ca));
    checkInvalidTrailingString(getTokensForParser("x = `foo", cf, &ca));
    checkInvalidTrailingString(getTokensForParser("x = q{foo", cf, &ca));
    checkInvalidTrailingString(getTokensForParser(`x = q"foo`, cf, &ca));
    checkInvalidTrailingString(getTokensForParser("x = '", cf, &ca));