public void SetUp()
        {
            Lexer lexer;

            _emptyDictionary = new TestingTokenDictionary();

            lexer = new Lexer();
            _filledDictionary = lexer.Dictionary;
            Assert.That(_filledDictionary, Is.Not.Null);
            Assert.That(_filledDictionary.Count, Is.GreaterThan(0));

            return;
        }
示例#2
0
        private void _checkOutput(string sequence, LexToken[] expected)
        {
            Lexer lexer;
            StringBuilder recognized;
            string error;
            int i;
            int j;

            lexer = new Lexer();
            lexer.Parse(sequence);

            recognized = new StringBuilder();

            i = 0;
            while (lexer.Next())
            {
                recognized.Append(lexer.CurrentToken.Text);

                error = String.Format("Token [{0}] was expected, but lexer returned [{1}] instead, near: [{2}].",
                    expected[i],
                    lexer.CurrentToken,
                    recognized.ToString());

                Assert.That(lexer.CurrentToken, Is.EqualTo(expected[i]), error);

                i++;
            }

            Assert.That(lexer.Next(), Is.False, "Error, there are unvisited tokens left.");

            error = "missing ";
            j = i;
            while (j < expected.Length)
            {
                error += expected[j].ToString();
                error += ", ";
                ++j;
            }

            Assert.That(i == expected.Length, "Error, more tokens were expected. {0}", error);

            return;
        }
        private void _checkClassification(string sequence, Couple[] expected)
        {
            Lexer lexer;
            ClassificationTag returned_tag;
            TestingClassifier classifier;
            StringBuilder recognized;
            string error;

            int i;

            classifier = new TestingClassifier();
            lexer = new Lexer();
            lexer.Parse(sequence);

            recognized = new StringBuilder();

            i = 0;
            while (lexer.Next())
            {
                recognized.Append(lexer.CurrentToken.Text);

                error = String.Format(
                    "Token [{0}] was expected, but [{1}] was returned instead, near: [{2}].",
                    expected[i].Text,
                    lexer.CurrentToken.Text,
                    recognized.ToString());
                Assert.That(lexer.CurrentToken.Text, Is.EqualTo(expected[i].Text), error);

                returned_tag = classifier.Classify(lexer.CurrentToken);

                error = String.Format(
                    "ClassificationTag [{0}] was expected, but [{1}] was returned instead, near: [{2}].",
                    expected[i].Value,
                    returned_tag,
                    recognized.ToString());
                Assert.That(returned_tag, Is.EqualTo(expected[i].Value), error);

                i++;
            }

            Assert.That(lexer.Next(), Is.False, "Error, there are unvisited tokens left.");
            Assert.That(i == expected.Length, "Error, more tokens were expected.");

            return;
        }
        public void Test_Escaping_sequence()
        {
            Lexer _lexer;

            _lexer = new Lexer();

            // this ensure that escaping can be set in string context only

            _lexer.Parse("\\\\");
            _classifier.Reset();
            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));
            Assert.That(_classifier.Escaping, Is.False);
            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));
            Assert.That(_classifier.Escaping, Is.False);


            // this ensure that parsing "\\\\" two times
            // set and unset Escaping flag correctly

            _lexer.Parse("\"\\\\\"");

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_classifier.Escaping, Is.False);
            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
            Assert.That(_classifier.Escaping, Is.True);

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
            Assert.That(_classifier.Escaping, Is.False);

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
            Assert.That(_classifier.Escaping, Is.False);


            // this ensure that first 'a' is considered as string, second as code

            _lexer.Parse("\"\\\"a\"a");
            _classifier.Reset();

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
            Assert.That(_classifier.Escaping, Is.True);

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));
            Assert.That(_classifier.Escaping, Is.False);

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("a"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("a"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));


            // another test, this time 'a' should be considered as code

            _lexer.Parse("\"\\\\\"a\"");
            _classifier.Reset();

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\\"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("\""));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.String));

            Assert.That(_lexer.Next(), Is.True);
            Assert.That(_lexer.CurrentToken.Text, Is.EqualTo("a"));
            Assert.That(_classifier.Classify(_lexer.CurrentToken), Is.EqualTo(ClassificationTag.Code));


            // this ensure that Reset() reset escaping to false

            _lexer.Parse("\"\\");
            _lexer.Next();
            _classifier.Classify(_lexer.CurrentToken);
            _lexer.Next();
            _classifier.Classify(_lexer.CurrentToken);
            Assert.That(_classifier.Escaping, Is.True);
            _classifier.Reset();
            Assert.That(_classifier.Escaping, Is.False);

            return;
        }
        public void Test_Reset()
        {
            Lexer lexer;

            lexer = new Lexer();
            lexer.Parse("/*int");

            lexer.Next();
            _classifier.Classify(lexer.CurrentToken);

            _classifier.Reset();
            lexer.Next();
            Assert.That(_classifier.Classify(lexer.CurrentToken), Is.EqualTo(ClassificationTag.Keyword));

            return;
        }
        public void Test_Classify_As_Keyword()
        {
            TokenClassifier classifier;
            ClassificationTag result;
            string error;
            Lexer lexer;

            lexer = new Lexer();
            lexer.Parse(
                "abstract event new struct as explicit null switch " +
                "base extern object this bool false operator throw " +
                "break finally out true byte fixed override try case " +
                "float params typeof catch for private uint char " +
                "foreach protected ulong checked goto public unchecked " +
                "class if readonly unsafe const implicit ref ushort " +
                "continue in return using decimal int sbyte virtual " +
                "default interface sealed volatile delegate internal " +
                "short void do is sizeof while double lock stackalloc " +
                "else long static enum namespace string get set region " +
                "endregion ");

            classifier = new TokenClassifier();

            while (lexer.Next())
            {
                if (lexer.CurrentToken.Text.Trim() == "")
                    continue;

                result = classifier.Classify(lexer.CurrentToken);

                error = String.Format("Classification: [{0}] was expected for token [{1}] but [{2}] was returned.",
                    ClassificationTag.Keyword,
                    lexer.CurrentToken,
                    result);

                Assert.That(
                    result,
                    Is.EqualTo(ClassificationTag.Keyword),
                    error);
            }

            return;
        }
示例#7
0
        /// <summary>
        /// Analyzes the input text as C# code. This method doesn't return anything.
        /// Callers may retrieve the result of this process by querying the FormattedCode property.
        ///   Passing null results in raising an exception.
        /// </summary>
        /// <param name="csharp">The text to be analyzed.</param>
        protected void Parse(string csharp)
        {
            TokenClassifier classifier;
            ConcreteToken csToken;
            ClassificationTag tag;
            Lexer lexer;
            StringBuilder text;
            int tokenIndex;

            UiExceptionHelper.CheckNotNull(csharp, "csharp");

            csharp = PreProcess(csharp);

            lexer = new Lexer();
            lexer.Parse(csharp);

            classifier = new TokenClassifier();
            text = new StringBuilder();
            csToken = null;
            tokenIndex = 0;

            // loop through each token in the text
            while (lexer.Next())
            {
                // classify the current token 
                tag = classifier.Classify(lexer.CurrentToken);

                // if the tag cannot be merged with current csToken
                // we flush csToken into _info and build a new instance
                // from the current tag.
                if (csToken == null ||
                    !csToken.CanMerge(_info.LineArray.Count, tag))
                {
                    _flushToken(csToken, _info);
                    csToken = new ConcreteToken(
                        lexer.CurrentToken.Text, tag,
                        lexer.CurrentToken.IndexStart,
                        _info.LineArray.Count);
                }

                // append token's text into text
                text.Append(lexer.CurrentToken.Text);

                // handle newline character. Appends tokenIndex to LineArray
                // and set tokenIndex to the start of the newline.
                if (lexer.CurrentToken.Text == "\n")
                {
                    _info.LineArray.Add(tokenIndex);
                    tokenIndex = _info.IndexArray.Count + 1;
                }
            }

            // flush terminal token
            _flushToken(csToken, _info);

            if (csToken != null &&
                _info.LineArray.Count == 0)
                _info.LineArray.Add(tokenIndex);

            _info.Text = csharp;

            return;
        }
示例#8
0
        /// <summary>
        /// Analyzes the input text as C# code. This method doesn't return anything.
        /// Callers may retrieve the result of this process by querying the FormattedCode property.
        ///   Passing null results in raising an exception.
        /// </summary>
        /// <param name="csharp">The text to be analyzed.</param>
        protected void Parse(string csharp)
        {
            TokenClassifier   classifier;
            ConcreteToken     csToken;
            ClassificationTag tag;
            Lexer             lexer;
            StringBuilder     text;
            int tokenIndex;

            UiExceptionHelper.CheckNotNull(csharp, "csharp");

            csharp = PreProcess(csharp);

            lexer = new Lexer();
            lexer.Parse(csharp);

            classifier = new TokenClassifier();
            text       = new StringBuilder();
            csToken    = null;
            tokenIndex = 0;

            // loop through each token in the text
            while (lexer.Next())
            {
                // classify the current token
                tag = classifier.Classify(lexer.CurrentToken);

                // if the tag cannot be merged with current csToken
                // we flush csToken into _info and build a new instance
                // from the current tag.
                if (csToken == null ||
                    !csToken.CanMerge(_info.LineArray.Count, tag))
                {
                    _flushToken(csToken, _info);
                    csToken = new ConcreteToken(
                        lexer.CurrentToken.Text, tag,
                        lexer.CurrentToken.IndexStart,
                        _info.LineArray.Count);
                }

                // append token's text into text
                text.Append(lexer.CurrentToken.Text);

                // handle newline character. Appends tokenIndex to LineArray
                // and set tokenIndex to the start of the newline.
                if (lexer.CurrentToken.Text == "\n")
                {
                    _info.LineArray.Add(tokenIndex);
                    tokenIndex = _info.IndexArray.Count + 1;
                }
            }

            // flush terminal token
            _flushToken(csToken, _info);

            if (csToken != null &&
                _info.LineArray.Count == 0)
            {
                _info.LineArray.Add(tokenIndex);
            }

            _info.Text = csharp;

            return;
        }