Example #1
0
        static bool IsSpecialIdentifier(Symbol name, out bool backquote)
        {
            backquote = name.Name.Length == 0;
            bool special = false, first = true;

            foreach (char c in name.Name)
            {
                if (!LesLexer.IsIdContChar(c))
                {
                    if (LesLexer.IsSpecialIdChar(c))
                    {
                        special = true;
                    }
                    else
                    {
                        backquote = true;
                    }
                }
                else if (first && !LesLexer.IsIdStartChar(c))
                {
                    special = true;
                }
                first = false;
            }

            // Watch out for @`-inf_d` and @`-inf_f`, because they will be
            // interpreted as named literals if we don't backquote them.
            if (special && !backquote && (name.Name == "-inf_d" || name.Name == "-inf_f"))
            {
                backquote = true;
            }
            return(special || backquote);
        }
Example #2
0
        void Case(string input, TokenType[] tokenTypes, params object[] values)
        {
            Debug.Assert(values.Length <= tokenTypes.Length);

            bool error = false;
            var  lexer = new LesLexer(input, new MessageSinkFromDelegate((type, ctx, msg, args) => {
                MessageSink.Trace.Write(type, ctx, msg, args); error = true;
            }));

            int index = 0;

            for (int i = 0; i < tokenTypes.Length; i++)
            {
                error = false;
                Token token = lexer.NextToken().Value;
                Assert.LessOrEqual(index, token.StartIndex);
                Assert.AreEqual(tokenTypes[i], token.Type());
                if (i < values.Length)
                {
                    Assert.AreEqual(values[i] == (object)ERROR, error);
                    if (!error)
                    {
                        Assert.AreEqual(values[i], token.Value);
                    }
                }
                index = token.EndIndex;
            }
            var nothing = lexer.NextToken();

            Assert.That(!nothing.HasValue);
        }
Example #3
0
 protected override void StartToken(char nextCh)
 {
     if (_newlinePending)
     {
         Newline();
     }
     if (LesLexer.IsIdContChar(_lastCh) && LesLexer.IsIdContChar(nextCh))
     {
         _out.Write(' ');
     }
     else if (LesLexer.IsOpContChar(_lastCh) && LesLexer.IsOpContChar(nextCh))
     {
         _out.Write(' ');
     }
 }
 protected override void StartToken(char nextCh)
 {
     if (_newlinePending)
     {
         Newline();
     }
     if (LesLexer.IsIdContChar(_lastCh) && LesLexer.IsIdContChar(nextCh))
     {
         _out.Write(' ');
     }
     else if (LesLexer.IsOpContChar(_lastCh) && LesLexer.IsOpContChar(nextCh))
     {
         _out.Write(' ');
     }
     else if (_lastCh == '-' && (nextCh >= '0' && nextCh <= '9'))             // - 2 is different from -2 (-(2) vs integer literal)
     {
         _out.Write(' ');
     }
 }
Example #5
0
        private void PrintIdOrSymbol(Symbol name, bool isSymbol)
        {
            // Figure out what style we need to use: plain, \\special, or \\`backquoted`
            bool special = isSymbol, backquote = name.Name.Length == 0, first = true;

            foreach (char c in name.Name)
            {
                if (!LesLexer.IsIdContChar(c))
                {
                    if (LesLexer.IsSpecialIdChar(c))
                    {
                        special = true;
                    }
                    else
                    {
                        backquote = true;
                    }
                }
                else if (first && !LesLexer.IsIdStartChar(c))
                {
                    special = true;
                }
                first = false;
            }

            if (special || backquote)
            {
                _out.Write(isSymbol ? "@@" : "@", false);
            }
            if (backquote)
            {
                PrintStringCore('`', false, name.Name);
            }
            else
            {
                _out.Write(name.Name, true);
            }
        }
		private void TestLes(string input, string expectOutput, int expectMessages = 0, Severity expectSev = 0)
		{
			using (G.PushTLV(Token.ToStringStrategyTLV, TokenExt.ToString))
			{
				MessageHolder errorList;
				var input2 = StripInitialNewline(input);
				var lexer = new LesLexer(input2, errorList = new MessageHolder());
				var wrapr = new LesIndentTokenGenerator(lexer);
				var output = new DList<Token>();
				for (var t = wrapr.NextToken(); t.HasValue; t = wrapr.NextToken())
					output.Add(t.Value);
				var expectTokens = new LesLexer(expectOutput, MessageSink.Console).Buffered().Select(t =>
					t.Type() == TokenType.LBrack ? t.WithType((int)TokenType.Indent) :
					t.Type() == TokenType.RBrack ? t.WithType((int)TokenType.Dedent) : t).ToList();

				AreEqual(expectMessages, errorList.List.Count);
				if (expectMessages > 0)
					AreEqual(expectSev, errorList.List.Max(m => m.Severity));
				ExpectList(output, expectTokens, false);
			}
		}
Example #7
0
        public ILexer <Token> Tokenize(ICharSource text, string fileName, IMessageSink msgs)
        {
            var lexer = new LesLexer(text, fileName, msgs);

            return(new LesIndentTokenGenerator(new WhitespaceFilter(lexer)));
        }
Example #8
0
		List<Token> Lex(string input, bool skipWS = true)
		{
			var lexer = new LesLexer(input, MessageSink.Trace);
			var lexer2 = new TokensToTree(lexer, skipWS);
			var list = new List<Token>();
			Maybe<Token> token;
			while ((token = lexer2.NextToken()).HasValue)
				list.Add(token.Value);
			return list;
		}
Example #9
0
		public ILexer<Token> Tokenize(ICharSource text, string fileName, IMessageSink msgs)
		{
			var lexer = new LesLexer(text, fileName, msgs);
			return new LesIndentTokenGenerator(new WhitespaceFilter(lexer));
		}