Ejemplo n.º 1
0
        /// <summary>
        /// Converts ASCII Program line to tokenised form
        /// </summary>
        /// <param name="line">ASCII Program line to tokenise</param>
        /// <returns>Tokenised program line</returns>
        public string Tokenise(string line)
        {
            using (var inputStream = new DetokenisedLineReader(line, _keywordTokenMap))
                using (var outputStream = new MemoryStream())
                {
                    using (var outputWriter = new TokenisedLineWriter(outputStream))
                    {
                        Tokenise(inputStream, outputWriter);
                    }

                    return(Encoding.UTF8.GetString(outputStream.ToArray()));
                }
        }
Ejemplo n.º 2
0
        private void TokeniseJumpNumbers(DetokenisedLineReader reader, TokenisedLineWriter writer)
        {
            var word = reader.ReadLineNumber();

            if (!string.IsNullOrEmpty(word))
            {
                writer.Write((char)Token.LineNumber);
                writer.Write(word);
            }
            else
            {
                reader.Read();
                writer.Write('.');
            }
        }
Ejemplo n.º 3
0
        /// <summary>
        /// Convert an ascii line number to tokenised start-of-line
        /// </summary>
        private void TokeniseLineNumber(DetokenisedLineReader reader, TokenisedLineWriter writer)
        {
            var lineNumber = reader.ReadLineNumber();

            writer.WriteLineNumber(lineNumber);

            if (!string.IsNullOrEmpty(lineNumber))
            {
                // ignore single whitespace after line number, if any,
                // unless line number is zero (as does GW)
                if (reader.Peek() == ' ' && lineNumber != "\0\0")
                {
                    reader.Read();
                }
            }
        }
Ejemplo n.º 4
0
        private void TokeniseNumber(DetokenisedLineReader reader, TokenisedLineWriter writer)
        {
            var current = reader.Peek();

            if (current == -1)
            {
                return;
            }

            if (current == '&') // handle hex or oct constants
            {
                reader.ReadAsChar();
                if (char.ToUpper((char)reader.Peek()) == 'H')
                {
                    // hex constant
                    writer.WriteHex(reader.ReadHex());
                }
                else
                {
                    writer.WriteOctal(reader.ReadOctal());
                }
            }
            else if (Constants.DecimalDigits.Contains(current) ||
                     current == '.' || current == '+' || current == '-')
            {
                // handle other numbers
                // note GW passes signs separately as a token
                // and only stores positive numbers in the program
                writer.Write(reader.ReadDecimal());
            }
            else
            {
                // why is this here?
                // this looks wrong but hasn't hurt so far
                reader.Seek(-1);
            }
        }
Ejemplo n.º 5
0
 private void TokeniseRemarks(DetokenisedLineReader reader, TokenisedLineWriter writer)
 {
     writer.Write(reader.ReadUntil(Constants.RemarkTerminator));
 }
Ejemplo n.º 6
0
        private void Tokenise(DetokenisedLineReader reader, TokenisedLineWriter writer)
        {
            var value = reader.SkipWhitespace();

            if (value == -1)
            {
                return; // EOF
            }
            // read the line number
            TokeniseLineNumber(reader, writer);

            var allowJump            = false;
            var allowNumber          = true;
            var accpetClosingBracket = false;

            while (true)
            {
                var current = reader.Peek();

                // anything after NUL is ignored till EOL
                if (current == '\0')
                {
                    reader.Read();
                    reader.ReadUntil('\r');
                    break;
                }

                if (current == '\r' || current == -1) // EOF or
                {
                    break;
                }

                if (Constants.Whitepsace.Contains(current)) // whitespace
                {
                    reader.Read();
                    writer.Write((char)current);
                }
                else if (current == '"') // string literals
                {
                    writer.Write(reader.ReadStringLiteral());
                }

                // handle jump numbers
                else if (allowJump && allowNumber && (Constants.DecimalDigits.Contains(current) || current == '.'))
                {
                    TokeniseJumpNumbers(reader, writer);
                }
                // numbers following var names with no operator or token in between
                // should not be parsed, eg OPTION BASE 1
                // note we don't include leading signs, encoded as unary operators
                // number starting with . or & are always parsed

                else if (current == '&' || current == '.' ||
                         (allowNumber && !allowJump && Constants.DecimalDigits.Contains(current)))
                {
                    TokeniseNumber(reader, writer);
                }

                // operator keywords ('+', '-', '=', '/', '\\', '^', '*', '<', '>')
                else if (Constants.Operators.Contains(current))
                {
                    reader.Read();

                    // operators don't affect line number mode - can do line number
                    // arithmetic and RENUM will do the strangest things
                    // this allows for 'LIST 100-200' etc.
                    writer.Write(_keywordTokenMap[current.ToCharString()]);
                    allowNumber = true;
                }

                // special case ' -> :REM'
                else if (current == '\'')
                {
                    reader.Read();
                    writer.Write(':');
                    writer.Write(Token.KeywordRem);
                    writer.Write(Token.KeywordORem);

                    TokeniseRemarks(reader, writer);
                }

                // special case ? -> PRINT
                else if (current == '?')
                {
                    reader.Read();
                    writer.Write(Token.KeywordPrint);
                    if (!Constants.Whitepsace.Contains(reader.Peek()))
                    {
                        writer.Write(' ');
                    }

                    allowNumber = true;
                }

                // keywords & variable names
                else if (Constants.Letters.Contains(current))
                {
                    var word = reader.ReadWord();
                    writer.Write(word);

                    // handle non-parsing modes
                    switch (word)
                    {
                    case Token.KeywordRem:
                    case "'":
                        writer.Write(reader.ReadRemarks());
                        break;

                    case "DATA":
                        writer.Write(reader.ReadData());
                        break;

                    default:
                        allowJump = Constants.LineNumberPrefixTokens.Contains(word);
                        // numbers can follow tokenised keywords
                        // (which does not include the word 'AS')

                        allowNumber = _tokenKeywordMap.ContainsKey(word);
                        if (word == "SPC(" || word == "TAB(")
                        {
                            accpetClosingBracket = true;
                        }
                        break;
                    }
                }
                else
                {
                    reader.Read();
                    if (current == ',' || current == '#' || current == ';')
                    {
                        allowNumber = true; // can separate numbers as well as jumpnums
                    }
                    else if (current == '(' || current == '[')
                    {
                        allowJump   = false;
                        allowNumber = true;
                    }
                    else if (current == ')' && accpetClosingBracket)
                    {
                        accpetClosingBracket = false;
                        allowJump            = false;
                        allowNumber          = true;
                    }
                    else
                    {
                        allowJump   = false;
                        allowNumber = false;
                    }

                    // replace all other nonprinting chars by spaces;
                    if (current >= 32 && current <= 127)
                    {
                        writer.Write((char)current);
                    }
                    else
                    {
                        writer.Write(' ');
                    }
                }
            }
        }