private void TokeniseNumber(DetokenisedLineReader reader, TokenisedLineWriter writer) { var current = reader.Peek(); if (current == -1) { return; } if (current == '&') // handle hex or oct constants { reader.ReadAsChar(); if (char.ToUpper((char)reader.Peek()) == 'H') { // hex constant writer.WriteHex(reader.ReadHex()); } else { writer.WriteOctal(reader.ReadOctal()); } } else if (Constants.DecimalDigits.Contains(current) || current == '.' || current == '+' || current == '-') { // handle other numbers // note GW passes signs separately as a token // and only stores positive numbers in the program writer.Write(reader.ReadDecimal()); } else { // why is this here? // this looks wrong but hasn't hurt so far reader.Seek(-1); } }
/// <summary> /// Convert an ascii line number to tokenised start-of-line /// </summary> private void TokeniseLineNumber(DetokenisedLineReader reader, TokenisedLineWriter writer) { var lineNumber = reader.ReadLineNumber(); writer.WriteLineNumber(lineNumber); if (!string.IsNullOrEmpty(lineNumber)) { // ignore single whitespace after line number, if any, // unless line number is zero (as does GW) if (reader.Peek() == ' ' && lineNumber != "\0\0") { reader.Read(); } } }
private void Tokenise(DetokenisedLineReader reader, TokenisedLineWriter writer) { var value = reader.SkipWhitespace(); if (value == -1) { return; // EOF } // read the line number TokeniseLineNumber(reader, writer); var allowJump = false; var allowNumber = true; var accpetClosingBracket = false; while (true) { var current = reader.Peek(); // anything after NUL is ignored till EOL if (current == '\0') { reader.Read(); reader.ReadUntil('\r'); break; } if (current == '\r' || current == -1) // EOF or { break; } if (Constants.Whitepsace.Contains(current)) // whitespace { reader.Read(); writer.Write((char)current); } else if (current == '"') // string literals { writer.Write(reader.ReadStringLiteral()); } // handle jump numbers else if (allowJump && allowNumber && (Constants.DecimalDigits.Contains(current) || current == '.')) { TokeniseJumpNumbers(reader, writer); } // numbers following var names with no operator or token in between // should not be parsed, eg OPTION BASE 1 // note we don't include leading signs, encoded as unary operators // number starting with . or & are always parsed else if (current == '&' || current == '.' || (allowNumber && !allowJump && Constants.DecimalDigits.Contains(current))) { TokeniseNumber(reader, writer); } // operator keywords ('+', '-', '=', '/', '\\', '^', '*', '<', '>') else if (Constants.Operators.Contains(current)) { reader.Read(); // operators don't affect line number mode - can do line number // arithmetic and RENUM will do the strangest things // this allows for 'LIST 100-200' etc. writer.Write(_keywordTokenMap[current.ToCharString()]); allowNumber = true; } // special case ' -> :REM' else if (current == '\'') { reader.Read(); writer.Write(':'); writer.Write(Token.KeywordRem); writer.Write(Token.KeywordORem); TokeniseRemarks(reader, writer); } // special case ? -> PRINT else if (current == '?') { reader.Read(); writer.Write(Token.KeywordPrint); if (!Constants.Whitepsace.Contains(reader.Peek())) { writer.Write(' '); } allowNumber = true; } // keywords & variable names else if (Constants.Letters.Contains(current)) { var word = reader.ReadWord(); writer.Write(word); // handle non-parsing modes switch (word) { case Token.KeywordRem: case "'": writer.Write(reader.ReadRemarks()); break; case "DATA": writer.Write(reader.ReadData()); break; default: allowJump = Constants.LineNumberPrefixTokens.Contains(word); // numbers can follow tokenised keywords // (which does not include the word 'AS') allowNumber = _tokenKeywordMap.ContainsKey(word); if (word == "SPC(" || word == "TAB(") { accpetClosingBracket = true; } break; } } else { reader.Read(); if (current == ',' || current == '#' || current == ';') { allowNumber = true; // can separate numbers as well as jumpnums } else if (current == '(' || current == '[') { allowJump = false; allowNumber = true; } else if (current == ')' && accpetClosingBracket) { accpetClosingBracket = false; allowJump = false; allowNumber = true; } else { allowJump = false; allowNumber = false; } // replace all other nonprinting chars by spaces; if (current >= 32 && current <= 127) { writer.Write((char)current); } else { writer.Write(' '); } } } }