private int DetokeniseLine(TokenisedLineReader reader, DetokenisedLineWriter writer) { var currentLine = reader.ReadLineNumber(); if (currentLine < 0) { return(-1); } // ignore up to one space after line number 0 if (currentLine == 0 && reader.Peek() == ' ') { reader.Read(); } writer.Write(currentLine.ToString()); // write one extra whitespace character after line number // unless first char is TAB if (reader.Peek() != '\t') { writer.Write(' '); } DetokeniseCompoundStatement(reader, writer); return(currentLine); }
/// <summary> /// Convert a tokenised program line to ascii text /// </summary> /// <param name="stream">Tokenized line stream</param> /// <returns>Detokenised ASCII line</returns> public DetokeniserOutput DetokeniseLine(Stream stream) { using (var inputStream = new TokenisedLineReader(stream, _tokenKeywordMap)) using (var outputStream = new MemoryStream()) { if (inputStream.Peek() == '\0') { inputStream.Read(); } int lineNumber; using (var outputWriter = new DetokenisedLineWriter(outputStream)) { lineNumber = DetokeniseLine(inputStream, outputWriter); } return(new DetokeniserOutput(lineNumber, Encoding.UTF8.GetString(outputStream.ToArray()))); } }
/// <summary> /// Detokenise tokens until end of line. /// </summary> private void DetokeniseCompoundStatement(TokenisedLineReader reader, DetokenisedLineWriter writer) { var stringLiteral = false; var comment = false; while (true) { var current = reader.Read(); if (current == -1 || current == '\0') { // \x00 ends lines and comments when listed, // if not inside a number constant // stream ended or end of line break; } if (current == '"') { // start of literal string, passed verbatim // until a closing quote or EOL comes by // however number codes are *printed* as the corresponding numbers, // even inside comments & literals writer.Write((char)current); stringLiteral = !stringLiteral; } else if (Tokens.NumberTypeTokens.Contains(current)) { reader.Seek(-1); writer.Write(reader.ReadNumber()); } else if (Tokens.LineNumberTokens.Contains(current)) { // 0D: line pointer (unsigned int) - this token should not be here; // interpret as line number and carry on // 0E: line number (unsigned int) writer.Write(reader.ReadUnsignedInteger().ToString()); } else if (comment || stringLiteral || (current >= 0x20 && current <= 0x7E)) { writer.Write((char)current); } else if (current == 0x0A) { writer.Write("\x0A\x0D"); } else if (current <= 0x09) { writer.Write((char)current); } else { writer.Flush(); reader.Seek(-1); if (writer.BaseStream.Length > 0) { // letter or number followed by token is separated by a space writer.BaseStream.Seek(-1, SeekOrigin.Current); var lastByte = writer.BaseStream.ReadByte(); if (Constants.DecimalDigits.Contains(lastByte) && !Constants.Operators.Contains(current)) { writer.Write(' '); } } var keyword = reader.ReadKeywords(out comment); writer.Write(keyword); writer.Flush(); // check for special cases // [:REM'] -> ['] if (writer.ReadLast(4) == ":REM") { writer.Replace("'", 4); } // [WHILE+] -> [WHILE] else if (writer.ReadLast(6) == "WHILE+") { writer.Replace("WHILE", 6); } // [:ELSE] -> [ELSE] else if (writer.ReadLast(4) == "ELSE") { // note that anything before ELSE gets cut off, // e.g. if we have 1ELSE instead of :ELSE it also becomes ELSE var lastSix = writer.ReadLast(6); if (lastSix[1] == ':' && Constants.DecimalDigits.Contains(lastSix[0])) { writer.Replace(": ELSE", 6); } else { writer.Replace(":ELSE", 6); } } // token followed by token or number is separated by a space, // except operator tokens and SPC(, TAB(, FN, USR var next = reader.Peek(); var currentStr = current.ToCharString(); if (!comment && ShouldAddWhiteSpace(next) && !(Tokens.OperatorTokens.Contains(currentStr) || Tokens.WithBracketTokens.Contains(currentStr) || currentStr == Token.KeywordUsr || currentStr == Token.KeywordFn )) { writer.Write(' '); } } } }