class compiler::Tokenizer

sys::Obj
  compiler::CompilerSupport
    compiler::Tokenizer

Source

Tokenizer inputs a Str and output a list of Tokens

err

virtual override CompilerErr err(Str msg, Loc? loc := null)

Source

Return a CompilerException for current location in source.

escape

Int escape()

Source

Parse an escapse sequence which starts with a \

make

new make(Compiler compiler, Loc loc, Str buf, Bool isDoc)

Source

Construct with characters of source file. The buffer passed must be normalized in that all newlines must be represented strictly as \n and not \r or \r\n (see File.readAllStr). If isDoc is false, we skip all star-star Fandoc comments.

next

TokenVal? next()

Source

Return the next token in the buffer.

tokenize

TokenVal[] tokenize()

Source

Tokenize the entire input into a list of tokens.