dparse.lexer

Undocumented in source.

Members

Aliases

BasicTypes
alias BasicTypes = AliasSeq!(tok!"int", tok!"bool", tok!"byte", tok!"cdouble", tok!"cent", tok!"cfloat", tok!"char", tok!"creal", tok!"dchar", tok!"double", tok!"float", tok!"idouble", tok!"ifloat", tok!"ireal", tok!"long", tok!"real", tok!"short", tok!"ubyte", tok!"ucent", tok!"uint", tok!"ulong", tok!"ushort", tok!"void", tok!"wchar")

Basic type token types.

IdType
alias IdType = TokenIdType!(operators, dynamicTokens, keywords)

Token ID type for the D lexer.

Keywords
alias Keywords = AliasSeq!(tok!"abstract", tok!"alias", tok!"align", tok!"asm", tok!"assert", tok!"auto", tok!"body", tok!"break", tok!"case", tok!"cast", tok!"catch", tok!"class", tok!"const", tok!"continue", tok!"debug", tok!"default", tok!"delegate", tok!"delete", tok!"deprecated", tok!"do", tok!"else", tok!"enum", tok!"export", tok!"extern", tok!"false", tok!"final", tok!"finally", tok!"for", tok!"foreach", tok!"foreach_reverse", tok!"function", tok!"goto", tok!"if", tok!"immutable", tok!"import", tok!"in", tok!"inout", tok!"interface", tok!"invariant", tok!"is", tok!"lazy", tok!"macro", tok!"mixin", tok!"module", tok!"new", tok!"nothrow", tok!"null", tok!"out", tok!"override", tok!"package", tok!"pragma", tok!"private", tok!"protected", tok!"public", tok!"pure", tok!"ref", tok!"return", tok!"scope", tok!"shared", tok!"static", tok!"struct", tok!"super", tok!"switch", tok!"synchronized", tok!"template", tok!"this", tok!"throw", tok!"true", tok!"try", tok!"typedef", tok!"typeid", tok!"typeof", tok!"union", tok!"unittest", tok!"version", tok!"volatile", tok!"while", tok!"with", tok!"__DATE__", tok!"__EOF__", tok!"__FILE__", tok!"__FUNCTION__", tok!"__gshared", tok!"__LINE__", tok!"__MODULE__", tok!"__parameters", tok!"__PRETTY_FUNCTION__", tok!"__TIME__", tok!"__TIMESTAMP__", tok!"__traits", tok!"__vector", tok!"__VENDOR__", tok!"__VERSION__")

Keyword token types.

Literals
alias Literals = AliasSeq!(StringLiterals, NumberLiterals, tok!"characterLiteral", SpecialTokens, tok!"true", tok!"false", tok!"null", tok!"$")
Undocumented in source.
NumberLiterals
alias NumberLiterals = AliasSeq!(tok!"doubleLiteral", tok!"floatLiteral", tok!"idoubleLiteral", tok!"ifloatLiteral", tok!"intLiteral", tok!"longLiteral", tok!"realLiteral", tok!"irealLiteral", tok!"uintLiteral", tok!"ulongLiteral")

Number literal token types.

Operators
alias Operators = AliasSeq!(tok!",", tok!".", tok!"..", tok!"...", tok!"/", tok!"/=", tok!"!", tok!"!<", tok!"!<=", tok!"!<>", tok!"!<>=", tok!"!=", tok!"!>", tok!"!>=", tok!"$", tok!"%", tok!"%=", tok!"&", tok!"&&", tok!"&=", tok!"(", tok!")", tok!"*", tok!"*=", tok!"+", tok!"++", tok!"+=", tok!"-", tok!"--", tok!"-=", tok!":", tok!";", tok!"<", tok!"<<", tok!"<<=", tok!"<=", tok!"<>", tok!"<>=", tok!"=", tok!"==", tok!"=>", tok!">", tok!">=", tok!">>", tok!">>=", tok!">>>", tok!">>>=", tok!"?", tok!"@", tok!"[", tok!"]", tok!"^", tok!"^=", tok!"^^", tok!"^^=", tok!"{", tok!"|", tok!"|=", tok!"||", tok!"}", tok!"~", tok!"~=")

Operator token types.

Protections
alias Protections = AliasSeq!(tok!"export", tok!"package", tok!"private", tok!"public", tok!"protected")

Protection token types.

SpecialTokens
alias SpecialTokens = AliasSeq!(tok!"__DATE__", tok!"__TIME__", tok!"__TIMESTAMP__", tok!"__VENDOR__", tok!"__VERSION__", tok!"__FILE__", tok!"__LINE__", tok!"__MODULE__", tok!"__FUNCTION__", tok!"__PRETTY_FUNCTION__")
Undocumented in source.
StringLiterals
alias StringLiterals = AliasSeq!(tok!"dstringLiteral", tok!"stringLiteral", tok!"wstringLiteral")

String literal token types

Token
alias Token = std.experimental.lexer.TokenStructure!(IdType, extraFields)

The token type in the D lexer

str
alias str = tokenStringRepresentation!(IdType, operators, dynamicTokens, keywords)

Function used for converting an IdType to a string.

Enums

CommentBehavior
enum CommentBehavior
Undocumented in source.
StringBehavior
enum StringBehavior

Configure string lexing behavior

WhitespaceBehavior
enum WhitespaceBehavior

Configure whitespace handling

Functions

byToken
auto byToken(ubyte[] range)

Creates a token range from the given source code. Creates a default lexer configuration and a GC-managed string cache.

byToken
auto byToken(ubyte[] range, StringCache* cache)

Creates a token range from the given source code. Uses the given string cache.

byToken
auto byToken(ubyte[] range, LexerConfig config, StringCache* cache)

Creates a token range from the given source code. Uses the provided lexer configuration and string cache.

getTokensForParser
const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, StringCache* cache)
isBasicType
bool isBasicType(IdType type)
isKeyword
bool isKeyword(IdType type)
isLiteral
bool isLiteral(IdType type)
Undocumented in source. Be warned that the author may not have intended to support it.
isNumberLiteral
bool isNumberLiteral(IdType type)
isOperator
bool isOperator(IdType type)
isProtection
bool isProtection(IdType type)
isSpecialToken
bool isSpecialToken(IdType type)
Undocumented in source. Be warned that the author may not have intended to support it.
isStringLiteral
bool isStringLiteral(IdType type)
newlineMask
ushort newlineMask(ubyte* )
rangeMatch
ulong rangeMatch(ubyte* )
skip
void skip(ubyte* , ulong* , ulong* )

Skips between 0 and 16 bytes that match (or do not match) one of the given chars.

unDecorateComment
void unDecorateComment(string comment, T outputRange)

Removes "decoration" such as leading whitespace, leading + and * characters, and places the result into the given output range

Structs

DLexer
struct DLexer

The D lexer struct.

LexerConfig
struct LexerConfig

Lexer configuration struct

StringCache
struct StringCache

The string cache is used for string interning.

Templates

ByteCombine
template ByteCombine(c...)
Undocumented in source.
tok
template tok(string token)

Template used to refer to D token types.

Meta