/// <summary>
/// Converts ASCII Program line to tokenised form
/// </summary>
/// <param name="line">ASCII Program line to tokenise</param>
/// <returns>Tokenised program line</returns>
public string Tokenise(string line)
{
using (var inputStream = new DetokenisedLineReader(line, _keywordTokenMap))
using (var outputStream = new MemoryStream())
{
using (var outputWriter = new TokenisedLineWriter(outputStream))
{
Tokenise(inputStream, outputWriter);
}
return(Encoding.UTF8.GetString(outputStream.ToArray()));
}
}
private void TokeniseJumpNumbers(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
var word = reader.ReadLineNumber();
if (!string.IsNullOrEmpty(word))
{
writer.Write((char)Token.LineNumber);
writer.Write(word);
}
else
{
reader.Read();
writer.Write('.');
}
}
/// <summary>
/// Convert an ascii line number to tokenised start-of-line
/// </summary>
private void TokeniseLineNumber(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
var lineNumber = reader.ReadLineNumber();
writer.WriteLineNumber(lineNumber);
if (!string.IsNullOrEmpty(lineNumber))
{
// ignore single whitespace after line number, if any,
// unless line number is zero (as does GW)
if (reader.Peek() == ' ' && lineNumber != "\0\0")
{
reader.Read();
}
}
}
private void TokeniseNumber(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
var current = reader.Peek();
if (current == -1)
{
return;
}
if (current == '&') // handle hex or oct constants
{
reader.ReadAsChar();
if (char.ToUpper((char)reader.Peek()) == 'H')
{
// hex constant
writer.WriteHex(reader.ReadHex());
}
else
{
writer.WriteOctal(reader.ReadOctal());
}
}
else if (Constants.DecimalDigits.Contains(current) ||
current == '.' || current == '+' || current == '-')
{
// handle other numbers
// note GW passes signs separately as a token
// and only stores positive numbers in the program
writer.Write(reader.ReadDecimal());
}
else
{
// why is this here?
// this looks wrong but hasn't hurt so far
reader.Seek(-1);
}
}
private void TokeniseRemarks(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
writer.Write(reader.ReadUntil(Constants.RemarkTerminator));
}
private void Tokenise(DetokenisedLineReader reader, TokenisedLineWriter writer)
{
var value = reader.SkipWhitespace();
if (value == -1)
{
return; // EOF
}
// read the line number
TokeniseLineNumber(reader, writer);
var allowJump = false;
var allowNumber = true;
var accpetClosingBracket = false;
while (true)
{
var current = reader.Peek();
// anything after NUL is ignored till EOL
if (current == '\0')
{
reader.Read();
reader.ReadUntil('\r');
break;
}
if (current == '\r' || current == -1) // EOF or
{
break;
}
if (Constants.Whitepsace.Contains(current)) // whitespace
{
reader.Read();
writer.Write((char)current);
}
else if (current == '"') // string literals
{
writer.Write(reader.ReadStringLiteral());
}
// handle jump numbers
else if (allowJump && allowNumber && (Constants.DecimalDigits.Contains(current) || current == '.'))
{
TokeniseJumpNumbers(reader, writer);
}
// numbers following var names with no operator or token in between
// should not be parsed, eg OPTION BASE 1
// note we don't include leading signs, encoded as unary operators
// number starting with . or & are always parsed
else if (current == '&' || current == '.' ||
(allowNumber && !allowJump && Constants.DecimalDigits.Contains(current)))
{
TokeniseNumber(reader, writer);
}
// operator keywords ('+', '-', '=', '/', '\\', '^', '*', '<', '>')
else if (Constants.Operators.Contains(current))
{
reader.Read();
// operators don't affect line number mode - can do line number
// arithmetic and RENUM will do the strangest things
// this allows for 'LIST 100-200' etc.
writer.Write(_keywordTokenMap[current.ToCharString()]);
allowNumber = true;
}
// special case ' -> :REM'
else if (current == '\'')
{
reader.Read();
writer.Write(':');
writer.Write(Token.KeywordRem);
writer.Write(Token.KeywordORem);
TokeniseRemarks(reader, writer);
}
// special case ? -> PRINT
else if (current == '?')
{
reader.Read();
writer.Write(Token.KeywordPrint);
if (!Constants.Whitepsace.Contains(reader.Peek()))
{
writer.Write(' ');
}
allowNumber = true;
}
// keywords & variable names
else if (Constants.Letters.Contains(current))
{
var word = reader.ReadWord();
writer.Write(word);
// handle non-parsing modes
switch (word)
{
case Token.KeywordRem:
case "'":
writer.Write(reader.ReadRemarks());
break;
case "DATA":
writer.Write(reader.ReadData());
break;
default:
allowJump = Constants.LineNumberPrefixTokens.Contains(word);
// numbers can follow tokenised keywords
// (which does not include the word 'AS')
allowNumber = _tokenKeywordMap.ContainsKey(word);
if (word == "SPC(" || word == "TAB(")
{
accpetClosingBracket = true;
}
break;
}
}
else
{
reader.Read();
if (current == ',' || current == '#' || current == ';')
{
allowNumber = true; // can separate numbers as well as jumpnums
}
else if (current == '(' || current == '[')
{
allowJump = false;
allowNumber = true;
}
else if (current == ')' && accpetClosingBracket)
{
accpetClosingBracket = false;
allowJump = false;
allowNumber = true;
}
else
{
allowJump = false;
allowNumber = false;
}
// replace all other nonprinting chars by spaces;
if (current >= 32 && current <= 127)
{
writer.Write((char)current);
}
else
{
writer.Write(' ');
}
}
}
}
