dparse.lexer

Undocumented in source.

Public Imports

dparse.trivia
public import dparse.trivia;

Members

Aliases

BasicTypes
alias BasicTypes = AliasSeq!(tok!"int", tok!"bool", tok!"byte", tok!"cdouble", tok!"cent", tok!"cfloat", tok!"char", tok!"creal", tok!"dchar", tok!"double", tok!"float", tok!"idouble", tok!"ifloat", tok!"ireal", tok!"long", tok!"real", tok!"short", tok!"ubyte", tok!"ucent", tok!"uint", tok!"ulong", tok!"ushort", tok!"void", tok!"wchar")

Basic type token types.

IdType
alias IdType = TokenIdType!(operators, dynamicTokens, keywords)

Token ID type for the D lexer.

IntegerLiterals
alias IntegerLiterals = AliasSeq!(tok!"intLiteral", tok!"longLiteral", tok!"uintLiteral", tok!"ulongLiteral")

Number literal token types.

Keywords
alias Keywords = AliasSeq!(tok!"abstract", tok!"alias", tok!"align", tok!"asm", tok!"assert", tok!"auto", tok!"break", tok!"case", tok!"cast", tok!"catch", tok!"class", tok!"const", tok!"continue", tok!"debug", tok!"default", tok!"delegate", tok!"delete", tok!"deprecated", tok!"do", tok!"else", tok!"enum", tok!"export", tok!"extern", tok!"false", tok!"final", tok!"finally", tok!"for", tok!"foreach", tok!"foreach_reverse", tok!"function", tok!"goto", tok!"if", tok!"immutable", tok!"import", tok!"in", tok!"inout", tok!"interface", tok!"invariant", tok!"is", tok!"lazy", tok!"macro", tok!"mixin", tok!"module", tok!"new", tok!"nothrow", tok!"null", tok!"out", tok!"override", tok!"package", tok!"pragma", tok!"private", tok!"protected", tok!"public", tok!"pure", tok!"ref", tok!"return", tok!"scope", tok!"shared", tok!"static", tok!"struct", tok!"super", tok!"switch", tok!"synchronized", tok!"template", tok!"this", tok!"throw", tok!"true", tok!"try", tok!"typedef", tok!"typeid", tok!"typeof", tok!"union", tok!"unittest", tok!"version", tok!"while", tok!"with", tok!"__DATE__", tok!"__EOF__", tok!"__FILE__", tok!"__FILE_FULL_PATH__", tok!"__FUNCTION__", tok!"__gshared", tok!"__LINE__", tok!"__MODULE__", tok!"__parameters", tok!"__PRETTY_FUNCTION__", tok!"__TIME__", tok!"__TIMESTAMP__", tok!"__traits", tok!"__vector", tok!"__VENDOR__", tok!"__VERSION__")

Keyword token types.

Literals
alias Literals = AliasSeq!(StringLiterals, NumberLiterals, tok!"characterLiteral", SpecialTokens, tok!"true", tok!"false", tok!"null", tok!"$")
Undocumented in source.
NumberLiterals
alias NumberLiterals = AliasSeq!(tok!"doubleLiteral", tok!"floatLiteral", tok!"idoubleLiteral", tok!"ifloatLiteral", tok!"intLiteral", tok!"longLiteral", tok!"realLiteral", tok!"irealLiteral", tok!"uintLiteral", tok!"ulongLiteral")

Number literal token types.

Operators
alias Operators = AliasSeq!(tok!",", tok!".", tok!"..", tok!"...", tok!"/", tok!"/=", tok!"!", tok!"!<", tok!"!<=", tok!"!<>", tok!"!<>=", tok!"!=", tok!"!>", tok!"!>=", tok!"$", tok!"%", tok!"%=", tok!"&", tok!"&&", tok!"&=", tok!"(", tok!")", tok!"*", tok!"*=", tok!"+", tok!"++", tok!"+=", tok!"-", tok!"--", tok!"-=", tok!":", tok!";", tok!"<", tok!"<<", tok!"<<=", tok!"<=", tok!"<>", tok!"<>=", tok!"=", tok!"==", tok!"=>", tok!">", tok!">=", tok!">>", tok!">>=", tok!">>>", tok!">>>=", tok!"?", tok!"@", tok!"[", tok!"]", tok!"^", tok!"^=", tok!"^^", tok!"^^=", tok!"{", tok!"|", tok!"|=", tok!"||", tok!"}", tok!"~", tok!"~=")

Operator token types.

Protections
alias Protections = AliasSeq!(tok!"export", tok!"package", tok!"private", tok!"public", tok!"protected")

Protection token types.

SpecialTokens
alias SpecialTokens = AliasSeq!(tok!"__DATE__", tok!"__TIME__", tok!"__TIMESTAMP__", tok!"__VENDOR__", tok!"__VERSION__", tok!"__FILE__", tok!"__FILE_FULL_PATH__", tok!"__LINE__", tok!"__MODULE__", tok!"__FUNCTION__", tok!"__PRETTY_FUNCTION__")
Undocumented in source.
StringLiterals
alias StringLiterals = AliasSeq!(tok!"dstringLiteral", tok!"stringLiteral", tok!"wstringLiteral")

String literal token types

Token
alias Token = std.experimental.lexer.TokenStructure!(IdType, extraFields)

The token type in the D lexer

str
alias str = tokenStringRepresentation!(IdType, operators, dynamicTokens, keywords)

Function used for converting an IdType to a string.

Enums

CommentBehavior
enum CommentBehavior
Undocumented in source.
WhitespaceBehavior
enum WhitespaceBehavior

Configure whitespace handling

Functions

byToken
auto byToken(R range)

Creates a token range from the given source code. Creates a default lexer configuration and a GC-managed string cache.

byToken
auto byToken(R range, StringCache* cache)

Creates a token range from the given source code. Uses the given string cache.

byToken
auto byToken(R range, LexerConfig config, StringCache* cache)

Creates a token range from the given source code. Uses the provided lexer configuration and string cache.

getTokensForParser
const(Token)[] getTokensForParser(R sourceCode, LexerConfig config, StringCache* cache)
isBasicType
bool isBasicType(IdType type)
isIntegerLiteral
bool isIntegerLiteral(IdType type)
isKeyword
bool isKeyword(IdType type)
isLiteral
bool isLiteral(IdType type)
Undocumented in source. Be warned that the author may not have intended to support it.
isNumberLiteral
bool isNumberLiteral(IdType type)
isOperator
bool isOperator(IdType type)
isProtection
bool isProtection(IdType type)
isSpecialToken
bool isSpecialToken(IdType type)
Undocumented in source. Be warned that the author may not have intended to support it.
isStringLiteral
bool isStringLiteral(IdType type)
optimalBucketCount
size_t optimalBucketCount(size_t size)

Helper function used to avoid too much allocations while lexing.

rangeMatch
ulong rangeMatch(ubyte* )
skip
void skip(ubyte* , ulong* , ulong* )

Skips between 0 and 16 bytes that match (or do not match) one of the given chars.

Mixin templates

TokenTriviaFields
mixintemplate TokenTriviaFields()
Undocumented in source.

Structs

DLexer
struct DLexer

The D lexer struct.

LexerConfig
struct LexerConfig

Lexer configuration struct

StringBehavior
struct StringBehavior

Configure string lexing behavior

StringCache
struct StringCache

The string cache is used for string interning.

Templates

ByteCombine
template ByteCombine(c...)
Undocumented in source.
tok
template tok(string token)

Template used to refer to D token types.

Meta