예제 #1
0
        public void Char()
        {
            string text = "aa 'x' bb '\\'' cc";
            var scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("aa", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("'x'", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("bb", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("'\\''", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("cc", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);

            text = "'\\t' '\\x21' '\\u0041' '\"' '\\\\' ";
            scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("'\\t'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("'\\x21'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("'\\u0041'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("'\"'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(@"'\\'", scanner.Token.Text()); Assert.AreEqual(TokenKind.Char, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #2
0
        public void Basics()
        {
            string text = @"using System.IO;

            protected override Tuple2<NSMenu, int> GetScriptsLocation()
            {
            }

            ";
            var scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("using", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("System", scanner.Token.Text());
            Assert.AreEqual(1, scanner.Token.Line);
            scanner.Advance(); Assert.AreEqual(".", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("IO", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(";", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("protected", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("override", scanner.Token.Text());
            Assert.AreEqual(3, scanner.Token.Line);
            scanner.Advance(); Assert.AreEqual("Tuple2", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("<", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("NSMenu", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(",", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("int", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(">", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("GetScriptsLocation", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("(", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(")", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("{", scanner.Token.Text());
            Assert.AreEqual(4, scanner.Token.Line);
            scanner.Advance(); Assert.AreEqual("}", scanner.Token.Text());
            Assert.AreEqual(5, scanner.Token.Line);
            scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #3
0
        public string ParseType(string text, int start, int stop, ref Token token)
        {
            Contract.Requires(text != null, "text is null");
            Contract.Requires(start >= 0, "start is negative");
            Contract.Requires(start <= text.Length, "start is too big");
            Contract.Requires(start <= stop, "stop is too small");

            var result = new StringBuilder();

            try
            {
                m_text = text.Substring(start, stop - start);
                m_scanner = new Scanner();
                m_scanner.Init(m_text);

                if (m_scanner.Token.Kind == TokenKind.Identifier)
                {
                    DoParseType(result);
                    token = m_scanner.Token;
                }
            }
            catch (LocalException)
            {
                result = null;
            }
            catch (ScannerException)
            {
                result = null;
            }
            catch (Exception e)
            {
                Log.WriteLine(TraceLevel.Warning, "LocalsParser", "Unexpected error:");
                Log.WriteLine(TraceLevel.Warning, "LocalsParser", "{0}", e);
                result = null;
            }

            return result != null && result.Length > 0 ? result.ToString() : null;
        }
예제 #4
0
        // This is an ad-hoc parser used with auto-complete. It would also be possible
        // to write a real method body parser but that is an awful lot of work just to
        // extract local declarations and is made trickier because the method body is
        // being edited so it may be very malformed.
        //
        // What we do here is treat everything that could be a local variable as a
        // local variable. This may result in a few false positives, but that should
        // happen rarely and should not be a problem unless the false local hides
        // a real one.
        public Local[] Parse(string text, int start, int stop)
        {
            Contract.Requires(text != null, "text is null");
            Contract.Requires(start >= 0, "start is negative");
            Contract.Requires(start <= text.Length, "start is too big");
            Contract.Requires(start <= stop, "stop is too small");

            var locals = new List<Local>();
            m_numErrors = 0;

            try
            {
                m_text = text.Substring(start, stop - start);
                m_scanner = new Scanner();
                m_scanner.Init(m_text);

                while (m_scanner.Token.IsValid())
                {
                    if (m_scanner.Token.IsPunct("{"))
                        DoParseBlock(locals, 0);
                    else
                        m_scanner.Advance();
                }
            }
            catch (ScannerException)
            {
                ++m_numErrors;
            }

            return locals.ToArray();
        }
예제 #5
0
        public void VerbatimString()
        {
            string text = @"foo @""some
            text
            goes here"" yyy";
            var scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("foo", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("\"some\ntext\n	goes here\"", scanner.Token.Text());
            Assert.AreEqual(1, scanner.Token.Line);
            scanner.Advance(); Assert.AreEqual("yyy", scanner.Token.Text());
            Assert.AreEqual(3, scanner.Token.Line);
            scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);

            text = "aa @\"bb\"\"cc\" dd";
            scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("aa", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual("\"bb\"\"cc\"", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual("dd", scanner.Token.Text());
        }
예제 #6
0
 public void SingleLineComment()
 {
     string text = @"alpha // text goes here @"" hmm
     some more // foo";
     var scanner = new Scanner();
     scanner.Init(text);
     Assert.AreEqual("alpha", scanner.Token.Text());
     scanner.Advance(); Assert.AreEqual("some", scanner.Token.Text());
     scanner.Advance(); Assert.AreEqual("more", scanner.Token.Text());
     scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
 }
예제 #7
0
        public void String()
        {
            string text = "foo \"hmm\" xx \"aa { \" ";
            var scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("foo", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("\"hmm\"", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("xx", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("\"aa { \"", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);

            text = "xxx \"aa\\\"b\" yyy";
            scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("xxx", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("\"aa\\\"b\"", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual("yyy", scanner.Token.Text());
            scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #8
0
        public void Integers()
        {
            string text = "15 15U 0x1F 0x1FUL";

            var scanner = new Scanner();
            scanner.Init(text);

            Assert.AreEqual("15", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("15U", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("0x1F", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("0x1FUL", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #9
0
        public void Preprocess()
        {
            string text = "alpha\n#region Your Name Here	\n#define Foo\n#endif\n#endregion\n";

            var scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("alpha", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);

            CsPreprocess[] preprocess = scanner.Preprocess;
            Assert.AreEqual(4, preprocess.Length);

            Assert.AreEqual("region", preprocess[0].Name);
            Assert.AreEqual("Your Name Here", preprocess[0].Text);

            Assert.AreEqual("define", preprocess[1].Name);
            Assert.AreEqual("Foo", preprocess[1].Text);

            Assert.AreEqual("endif", preprocess[2].Name);
            Assert.AreEqual("", preprocess[2].Text);

            Assert.AreEqual("endregion", preprocess[3].Name);
            Assert.AreEqual("", preprocess[3].Text);
        }
예제 #10
0
        public void Identifier()
        {
            string text = @"alpha _beta @gamma";

            var scanner = new Scanner();
            scanner.Init(text);
            Assert.AreEqual("alpha", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("_beta", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("gamma", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Identifier, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #11
0
        public void IckyString()
        {
            string ick = @"#!/bin/sh
            app=yyy

            if [ ""$1"" = ""-?"" ] || [ ""$1"" = ""-h"" ] || [ ""$1"" = ""--help"" ] ; then
            echo ""Usage: zzz [files]""
            exit 0
            fi

            if [ -d ""$app"" ] ; then
            open -a ""$app"" $@
            else
            echo ""Couldn't find $app: try re-installing the tool.""
            fi
            ";
            string text = string.Format("private const string Script = @\"{0}\"", ick.Replace("\"", "\"\""));

            var scanner = new Scanner();
            scanner.Init(text);

            Assert.AreEqual("private", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual("const", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual("string", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual("Script", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual("=", scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual('"' + ick.Replace("\"", "\"\"") + '"', scanner.Token.Text());

            scanner.Advance();
            Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #12
0
        public void Floats()
        {
            string text = "10.4 10.4M 10.4e100 .33 .33e-3 15f 15e100";

            var scanner = new Scanner();
            scanner.Init(text);

            Assert.AreEqual("10.4", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("10.4M", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("10.4e100", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(".33", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(".33e-3", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("15f", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual("15e100", scanner.Token.Text());
            Assert.AreEqual(TokenKind.Number, scanner.Token.Kind);

            scanner.Advance();
            Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
        }
예제 #13
0
 public void DelimitedComment()
 {
     string text = @"alpha /* this
     is a comment */
      foo";
     var scanner = new Scanner();
     scanner.Init(text);
     Assert.AreEqual("alpha", scanner.Token.Text());
     scanner.Advance(); Assert.AreEqual("foo", scanner.Token.Text());
     Assert.AreEqual(3, scanner.Token.Line);
     scanner.Advance(); Assert.AreEqual(TokenKind.Invalid, scanner.Token.Kind);
 }
예제 #14
0
 private void DoInit(string text)
 {
     m_text = text;
     m_scanner = new Scanner();
     m_scanner.Init(text);
 }