Пример #1
0
        static bool IsSpecialIdentifier(Symbol name, out bool backquote)
        {
            backquote = name.Name.Length == 0;
            bool special = false, first = true;

            foreach (char c in name.Name)
            {
                if (!Les2Lexer.IsIdContChar(c))
                {
                    if (Les2Lexer.IsSpecialIdChar(c))
                    {
                        special = true;
                    }
                    else
                    {
                        backquote = true;
                    }
                }
                else if (first && !Les2Lexer.IsIdStartChar(c))
                {
                    special = true;
                }
                first = false;
            }

            // Watch out for @`-inf_d` and @`-inf_f`, because they will be
            // interpreted as named literals if we don't backquote them.
            if (special && !backquote && (name.Name == "-inf_d" || name.Name == "-inf_f"))
            {
                backquote = true;
            }
            return(special || backquote);
        }
Пример #2
0
        protected string ParseStringValue(bool parseNeeded, bool isTripleQuoted)
        {
            if (SkipValueParsing)
            {
                return(null);
            }
            string value;

            if (parseNeeded)
            {
                UString original = CharSource.Slice(_startPosition, InputPosition - _startPosition);
                value = Les2Lexer.UnescapeQuotedString(ref original, Error, IndentString, true);
                Debug.Assert(original.IsEmpty);
            }
            else
            {
                Debug.Assert(CharSource.TryGet(InputPosition - 1, '?') == CharSource.TryGet(_startPosition, '!'));
                if (isTripleQuoted)
                {
                    value = CharSource.Slice(_startPosition + 3, InputPosition - _startPosition - 6).ToString();
                }
                else
                {
                    value = CharSource.Slice(_startPosition + 1, InputPosition - _startPosition - 2).ToString();
                }
            }
            return(value);
        }
Пример #3
0
        void Case(string input, TokenType[] tokenTypes, params object[] values)
        {
            Debug.Assert(values.Length <= tokenTypes.Length);

            bool error = false;
            var  lexer = new Les2Lexer(input, new MessageSinkFromDelegate((type, ctx, msg, args) => {
                TraceMessageSink.Value.Write(type, ctx, msg, args); error = true;
            }));

            int index = 0;

            for (int i = 0; i < tokenTypes.Length; i++)
            {
                error = false;
                Token token = lexer.NextToken().Value;
                Assert.LessOrEqual(index, token.StartIndex);
                Assert.AreEqual(tokenTypes[i], token.Type());
                if (i < values.Length)
                {
                    Assert.AreEqual(values[i] == (object)ERROR, error);
                    if (!error)
                    {
                        Assert.AreEqual(values[i], token.Value);
                    }
                }
                index = token.EndIndex;
            }
            var nothing = lexer.NextToken();

            Assert.That(!nothing.HasValue);
        }
Пример #4
0
        static bool IsSpecialIdentifier(UString name, out bool backquote)
        {
            backquote = name.Length == 0;
            bool special = false, first = true;

            foreach (char c in name)
            {
                if (!Les2Lexer.IsIdContChar(c))
                {
                    if (Les2Lexer.IsSpecialIdChar(c))
                    {
                        special = true;
                    }
                    else
                    {
                        backquote = true;
                    }
                }
                else if (first && !Les2Lexer.IsIdStartChar(c))
                {
                    special = true;
                }
                first = false;
            }

            // Watch out for named literals with punctuation e.g. @`-inf_d` and @`-inf_f`:
            // they will be interpreted as named literals if we don't backquote them.
            if (special && !backquote && Les2Lexer.NamedLiterals.ContainsKey(name))
            {
                backquote = true;
            }
            return(special || backquote);
        }
Пример #5
0
 protected override void StartToken(char nextCh)
 {
     if (_newlinePending)
     {
         Newline();
     }
     if (Les2Lexer.IsIdContChar(_lastCh) && Les2Lexer.IsIdContChar(nextCh))
     {
         _out.Write(' ');
     }
     else if (Les2Lexer.IsOpContChar(_lastCh) && Les2Lexer.IsOpContChar(nextCh))
     {
         _out.Write(' ');
     }
     else if (_lastCh == '-' && (nextCh >= '0' && nextCh <= '9'))             // - 2 is different from -2 (-(2) vs integer literal)
     {
         _out.Write(' ');
     }
 }
Пример #6
0
        protected override void OnNodeChanged(char nextCh)
        {
            var lastCh = LastCharWritten;

            if (Les2Lexer.IsIdContChar(lastCh) && Les2Lexer.IsIdContChar(nextCh))
            {
                StringBuilder.Append(' ');
            }
            else if (Les2Lexer.IsOpContChar(lastCh) && Les2Lexer.IsOpContChar(nextCh))
            {
                StringBuilder.Append(' ');
            }
            else if (JustWroteSymbolOrSpecialId && Les2Lexer.IsSpecialIdChar(nextCh))
            {
                StringBuilder.Append(' ');
            }

            JustWroteSymbolOrSpecialId = false;
        }
Пример #7
0
 protected object ParseSQStringValue(bool parseNeeded)
 {
     if (SkipValueParsing)
     {
         return(null);
     }
     else
     {
         var text = Text();
         if (!parseNeeded && text.Length == 3)
         {
             return(text[1]);
         }
         else
         {
             var value = Les2Lexer.ParseSQStringValue(text, Error);
             if (value is string)
             {
                 return(new CustomLiteral(value, (Symbol)"char"));
             }
             return(value);
         }
     }
 }
Пример #8
0
		List<Token> Lex(string input, bool skipWS = true)
		{
			var lexer = new Les2Lexer(input, TraceMessageSink.Value);
			var lexer2 = new TokensToTree(lexer, skipWS);
			var list = new List<Token>();
			Maybe<Token> token;
			while ((token = lexer2.NextToken()).HasValue)
				list.Add(token.Value);
			return list;
		}