private int DetokeniseLine(TokenisedLineReader reader, DetokenisedLineWriter writer)
{
var currentLine = reader.ReadLineNumber();
if (currentLine < 0)
{
return(-1);
}
// ignore up to one space after line number 0
if (currentLine == 0 && reader.Peek() == ' ')
{
reader.Read();
}
writer.Write(currentLine.ToString());
// write one extra whitespace character after line number
// unless first char is TAB
if (reader.Peek() != '\t')
{
writer.Write(' ');
}
DetokeniseCompoundStatement(reader, writer);
return(currentLine);
}
/// <summary>
/// Convert a tokenised program line to ascii text
/// </summary>
/// <param name="stream">Tokenized line stream</param>
/// <returns>Detokenised ASCII line</returns>
public DetokeniserOutput DetokeniseLine(Stream stream)
{
using (var inputStream = new TokenisedLineReader(stream, _tokenKeywordMap))
using (var outputStream = new MemoryStream())
{
if (inputStream.Peek() == '\0')
{
inputStream.Read();
}
int lineNumber;
using (var outputWriter = new DetokenisedLineWriter(outputStream))
{
lineNumber = DetokeniseLine(inputStream, outputWriter);
}
return(new DetokeniserOutput(lineNumber, Encoding.UTF8.GetString(outputStream.ToArray())));
}
}
/// <summary>
/// Detokenise tokens until end of line.
/// </summary>
private void DetokeniseCompoundStatement(TokenisedLineReader reader, DetokenisedLineWriter writer)
{
var stringLiteral = false;
var comment = false;
while (true)
{
var current = reader.Read();
if (current == -1 || current == '\0')
{
// \x00 ends lines and comments when listed,
// if not inside a number constant
// stream ended or end of line
break;
}
if (current == '"')
{
// start of literal string, passed verbatim
// until a closing quote or EOL comes by
// however number codes are *printed* as the corresponding numbers,
// even inside comments & literals
writer.Write((char)current);
stringLiteral = !stringLiteral;
}
else if (Tokens.NumberTypeTokens.Contains(current))
{
reader.Seek(-1);
writer.Write(reader.ReadNumber());
}
else if (Tokens.LineNumberTokens.Contains(current))
{
// 0D: line pointer (unsigned int) - this token should not be here;
// interpret as line number and carry on
// 0E: line number (unsigned int)
writer.Write(reader.ReadUnsignedInteger().ToString());
}
else if (comment || stringLiteral || (current >= 0x20 && current <= 0x7E))
{
writer.Write((char)current);
}
else if (current == 0x0A)
{
writer.Write("\x0A\x0D");
}
else if (current <= 0x09)
{
writer.Write((char)current);
}
else
{
writer.Flush();
reader.Seek(-1);
if (writer.BaseStream.Length > 0)
{
// letter or number followed by token is separated by a space
writer.BaseStream.Seek(-1, SeekOrigin.Current);
var lastByte = writer.BaseStream.ReadByte();
if (Constants.DecimalDigits.Contains(lastByte) && !Constants.Operators.Contains(current))
{
writer.Write(' ');
}
}
var keyword = reader.ReadKeywords(out comment);
writer.Write(keyword);
writer.Flush();
// check for special cases
// [:REM'] -> [']
if (writer.ReadLast(4) == ":REM")
{
writer.Replace("'", 4);
}
// [WHILE+] -> [WHILE]
else if (writer.ReadLast(6) == "WHILE+")
{
writer.Replace("WHILE", 6);
}
// [:ELSE] -> [ELSE]
else if (writer.ReadLast(4) == "ELSE")
{
// note that anything before ELSE gets cut off,
// e.g. if we have 1ELSE instead of :ELSE it also becomes ELSE
var lastSix = writer.ReadLast(6);
if (lastSix[1] == ':' && Constants.DecimalDigits.Contains(lastSix[0]))
{
writer.Replace(": ELSE", 6);
}
else
{
writer.Replace(":ELSE", 6);
}
}
// token followed by token or number is separated by a space,
// except operator tokens and SPC(, TAB(, FN, USR
var next = reader.Peek();
var currentStr = current.ToCharString();
if (!comment && ShouldAddWhiteSpace(next) &&
!(Tokens.OperatorTokens.Contains(currentStr) ||
Tokens.WithBracketTokens.Contains(currentStr) ||
currentStr == Token.KeywordUsr ||
currentStr == Token.KeywordFn
))
{
writer.Write(' ');
}
}
}
}
