private void TokeniseRemarks(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
writer.Write(reader.ReadUntil(Constants.RemarkTerminator));
}
private void Tokenise(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
var value = reader.SkipWhitespace();
if (value == -1)
{
return; // EOF
}
// read the line number
TokeniseLineNumber(reader, writer);
var allowJump = false;
var allowNumber = true;
var accpetClosingBracket = false;
while (true)
{
var current = reader.Peek();
// anything after NUL is ignored till EOL
if (current == '\0')
{
reader.Read();
reader.ReadUntil('\r');
break;
}
if (current == '\r' || current == -1) // EOF or
{
break;
}
if (Constants.Whitepsace.Contains(current)) // whitespace
{
reader.Read();
writer.Write((char)current);
}
else if (current == '"') // string literals
{
writer.Write(reader.ReadStringLiteral());
}
// handle jump numbers
else if (allowJump && allowNumber && (Constants.DecimalDigits.Contains(current) || current == '.'))
{
TokeniseJumpNumbers(reader, writer);
}
// numbers following var names with no operator or token in between
// should not be parsed, eg OPTION BASE 1
// note we don't include leading signs, encoded as unary operators
// number starting with . or & are always parsed
else if (current == '&' || current == '.' ||
(allowNumber && !allowJump && Constants.DecimalDigits.Contains(current)))
{
TokeniseNumber(reader, writer);
}
// operator keywords ('+', '-', '=', '/', '\\', '^', '*', '<', '>')
else if (Constants.Operators.Contains(current))
{
reader.Read();
// operators don't affect line number mode - can do line number
// arithmetic and RENUM will do the strangest things
// this allows for 'LIST 100-200' etc.
writer.Write(_keywordTokenMap[current.ToCharString()]);
allowNumber = true;
}
// special case ' -> :REM'
else if (current == '\'')
{
reader.Read();
writer.Write(':');
writer.Write(Token.KeywordRem);
writer.Write(Token.KeywordORem);
TokeniseRemarks(reader, writer);
}
// special case ? -> PRINT
else if (current == '?')
{
reader.Read();
writer.Write(Token.KeywordPrint);
if (!Constants.Whitepsace.Contains(reader.Peek()))
{
writer.Write(' ');
}
allowNumber = true;
}
// keywords & variable names
else if (Constants.Letters.Contains(current))
{
var word = reader.ReadWord();
writer.Write(word);
// handle non-parsing modes
switch (word)
{
case Token.KeywordRem:
case "'":
writer.Write(reader.ReadRemarks());
break;
case "DATA":
writer.Write(reader.ReadData());
break;
default:
allowJump = Constants.LineNumberPrefixTokens.Contains(word);
// numbers can follow tokenised keywords
// (which does not include the word 'AS')
allowNumber = _tokenKeywordMap.ContainsKey(word);
if (word == "SPC(" || word == "TAB(")
{
accpetClosingBracket = true;
}
break;
}
}
else
{
reader.Read();
if (current == ',' || current == '#' || current == ';')
{
allowNumber = true; // can separate numbers as well as jumpnums
}
else if (current == '(' || current == '[')
{
allowJump = false;
allowNumber = true;
}
else if (current == ')' && accpetClosingBracket)
{
accpetClosingBracket = false;
allowJump = false;
allowNumber = true;
}
else
{
allowJump = false;
allowNumber = false;
}
// replace all other nonprinting chars by spaces;
if (current >= 32 && current <= 127)
{
writer.Write((char)current);
}
else
{
writer.Write(' ');
}
}
}
}
