Esempio n. 1
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            string         value = null;

            if (data.CurrentItem == Delimiter)
            {
                data.Advance();
                bool inEscape = false;
                for (; !data.AtEnd(); data.Advance())
                {
                    if (inEscape)
                    {
                        inEscape = false;
                        switch (data.CurrentItem)
                        {
                        case "\\":
                        case Delimiter:
                            value += data.CurrentItem;
                            continue;

                        default:
                            log.LogError(@$ "Unrecognized escape sequence: '\{data.CurrentItem}'", new SourcePosition(streamPos));
                            return(null);
                        }
                    }

                    if (data.CurrentItem == "\\")
                    {
                        inEscape = true;
                        continue;
                    }
                    if (data.CurrentItem == Delimiter)
                    {
                        break;
                    }
                    if (data.CurrentItem == "\n")
                    {
                        streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                        log.LogError("Name Literals can not contain line breaks!", start, new SourcePosition(streamPos));
                        return(null);
                    }
                    value += data.CurrentItem;
                }

                if (data.CurrentItem == Delimiter)
                {
                    data.Advance();
                    value ??= "None"; //empty name literals should be interpreted as 'None'
                }
                else
                {
                    streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                    log.LogError("Name Literal was not terminated properly!", start, new SourcePosition(streamPos));
                    return(null);
                }
            }
Esempio n. 2
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            String         value = null;

            if (data.CurrentItem == "\"")
            {
                data.Advance();
                String prev = "";
                while (!data.AtEnd())
                {
                    if (data.CurrentItem == "\"" && prev != "\\")
                    {
                        break;
                    }
                    else if (data.CurrentItem == "\n")
                    {
                        streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                        log.LogError("String Literals can not contain line breaks!", start, new SourcePosition(streamPos));
                        return(null);
                    }
                    value += data.CurrentItem;
                    prev   = data.CurrentItem;
                    data.Advance();
                }

                if (data.CurrentItem == "\"")
                {
                    data.Advance();
                    if (value == null)
                    {
                        value = "";
                    }
                }
                else
                {
                    streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                    log.LogError("String Literal was not terminated properly!", start, new SourcePosition(streamPos));
                    return(null);
                }
            }

            if (value != null)
            {
                streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                SourcePosition end = new SourcePosition(streamPos);
                return(new Token <String>(TokenType.String, value, start, end));
            }
            return(null);
        }
Esempio n. 3
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start   = new SourcePosition(streamPos);
            string         comment = null;

            if (data.CurrentItem == "/")
            {
                data.Advance();
                if (data.CurrentItem == "/")
                {
                    data.Advance();
                    while (!data.AtEnd())
                    {
                        if (data.CurrentItem == "\n")
                        {
                            break;
                        }

                        comment += data.CurrentItem;
                        data.Advance();
                    }


                    comment ??= "";


                    streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                    SourcePosition end = new SourcePosition(streamPos);
                    return(new Token <string>(TokenType.SingleLineComment, comment, start, end));
                }
            }

            return(null);
        }
Esempio n. 4
0
        public override Token <string> GetNextToken()
        {
            if (Data.AtEnd())
            {
                return(new Token <string>(TokenType.EOF, null, StreamPosition, StreamPosition));
            }

            Token <string> result = null;

            foreach (ITokenMatcher <string> matcher in TokenMatchers)
            {
                Token <string> token = matcher.MatchNext(Data, ref StreamPosition, Log);
                if (token != null)
                {
                    result = token;
                    break;
                }
            }

            if (result == null)
            {
                Log.LogError("Could not lex '" + Data.CurrentItem + "'",
                             StreamPosition, StreamPosition.GetModifiedPosition(0, 1, 1));
                Data.Advance();
                return(new Token <string>(TokenType.INVALID));
            }
            return(result);
        }
Esempio n. 5
0
        protected override Token <String> Match(TokenizableDataStream <String> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            TokenType      type;
            String         value;
            String         first = SubNumber(data, new Regex("[0-9]"));

            if (first == null)
            {
                return(null);
            }

            if (data.CurrentItem == "x")
            {
                if (first != "0")
                {
                    return(null);
                }

                data.Advance();
                String hex = SubNumber(data, new Regex("[0-9a-fA-F]"));
                if (hex == null || data.CurrentItem == "." || data.CurrentItem == "x")
                {
                    return(null);
                }

                hex   = Convert.ToInt32(hex, 16).ToString("D");
                type  = TokenType.IntegerNumber;
                value = hex;
            }
            else if (data.CurrentItem == ".")
            {
                data.Advance();
                String second = SubNumber(data, new Regex("[0-9]"));
                if (second == null || data.CurrentItem == "." || data.CurrentItem == "x")
                {
                    return(null);
                }

                type  = TokenType.FloatingNumber;
                value = first + "." + second;
            }
            else
            {
                type  = TokenType.IntegerNumber;
                value = first;
            }

            streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
            SourcePosition end = new SourcePosition(streamPos);

            return(new Token <String>(type, value, start, end));
        }
Esempio n. 6
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            String         value = null;
            Regex          regex = new Regex("[0-9a-zA-Z_]");

            if (data.CurrentItem == "'")
            {
                data.Advance();
                while (!data.AtEnd() && regex.IsMatch(data.CurrentItem))
                {
                    value += data.CurrentItem;
                    data.Advance();
                }

                if (data.CurrentItem == "'")
                {
                    data.Advance();
                    if (value == null)
                    {
                        value = "";
                    }
                }
                else
                {
                    streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                    log.LogError("Name Literal was not terminated properly!", start, new SourcePosition(streamPos));
                    return(null);
                }
            }

            if (value != null)
            {
                streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                SourcePosition end = new SourcePosition(streamPos);
                return(new Token <String>(TokenType.Name, value, start, end));
            }
            return(null);
        }
Esempio n. 7
0
 public void LogError(string msg, SourcePosition start = null, SourcePosition end = null)
 {
     if (start == null && end == null)
     {
         content.Add(new Error(msg));
     }
     else if (end == null)
     {
         content.Add(new LineError(msg, start, start.GetModifiedPosition(0, 1, 1)));
     }
     else
     {
         content.Add(new LineError(msg, start, end));
     }
 }
Esempio n. 8
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);

            string peek = data.CurrentItem;

            if (peek != "$")
            {
                return(null);
            }
            data.Advance();
            peek = data.CurrentItem;
            bool isNegative = false;

            if (peek == "-")
            {
                isNegative = true;
                data.Advance();
                peek = data.CurrentItem;
            }
            string number = null;
            var    regex  = new Regex("[0-9]");

            while (!data.AtEnd() && regex.IsMatch(peek))
            {
                number += peek;
                data.Advance();
                peek = data.CurrentItem;
            }

            if (number == null)
            {
                return(null);
            }

            if (isNegative)
            {
                number = $"-{number}";
            }

            streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
            SourcePosition end = new SourcePosition(streamPos);

            return(new Token <string>(TokenType.StringRefLiteral, number, start, end));
        }
Esempio n. 9
0
        public override Token <String> GetNextToken()
        {
            if (Data.AtEnd())
            {
                return(new Token <String>(TokenType.EOF));
            }

            Token <String> result =
                (from matcher in TokenMatchers
                 let token = matcher.MatchNext(Data, ref StreamPosition, Log)
                             where token != null
                             select token).FirstOrDefault();

            if (result == null)
            {
                Log.LogError("Could not lex '" + Data.CurrentItem + "'",
                             StreamPosition, StreamPosition.GetModifiedPosition(0, 1, 1));
                Data.Advance();
                return(new Token <String>(TokenType.INVALID));
            }
            return(result);
        }
Esempio n. 10
0
        protected override Token <String> Match(TokenizableDataStream <String> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            String         peek  = data.CurrentItem;
            String         word  = null;

            while (!data.AtEnd() && !String.IsNullOrWhiteSpace(peek) &&
                   Delimiters.All(d => d.Keyword != peek) &&
                   peek != "\"" && peek != "'")
            {
                word += peek;
                data.Advance();
                peek = data.CurrentItem;
            }

            if (word != null)
            {
                streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                SourcePosition end = new SourcePosition(streamPos);
                return(new Token <String>(TokenType.Word, word, start, end));
            }
            return(null);
        }
Esempio n. 11
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);

            foreach (char c in Keyword)
            {
                if (!string.Equals(data.CurrentItem, c.ToString(CultureInfo.InvariantCulture), StringComparison.CurrentCultureIgnoreCase))
                {
                    return(null);
                }
                data.Advance();
            }

            string peek         = data.CurrentItem;
            bool   hasDelimiter = string.IsNullOrWhiteSpace(peek) || Delimiters.Any(c => c.Keyword == peek);

            if (SubString || (!SubString && hasDelimiter))
            {
                streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                SourcePosition end = new SourcePosition(streamPos);
                return(new Token <string>(Type, Keyword, start, end));
            }
            return(null);
        }
Esempio n. 12
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            string         peek  = data.CurrentItem;
            string         word  = null;

loopStart:
            while (!data.AtEnd() && !string.IsNullOrWhiteSpace(peek) &&
                   (Delimiters.All(d => d.Keyword != peek)) &&
                   peek != "\"" && peek != "'")
            {
                word += peek;
                data.Advance();
                peek = data.CurrentItem;
            }

            //HACK: there are variable names that include the c++ scope operator '::' for some godforsaken reason
            if (peek == ":" && data.LookAhead(1) == ":")
            {
                word += peek;
                data.Advance();
                peek  = data.CurrentItem;
                word += peek;
                data.Advance();
                peek = data.CurrentItem;
                goto loopStart;
            }

            if (word != null)
            {
                streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
                SourcePosition end = new SourcePosition(streamPos);
                return(new Token <string>(TokenType.Word, word, start, end));
            }
            return(null);
        }
Esempio n. 13
0
        protected override Token <string> Match(TokenizableDataStream <string> data, ref SourcePosition streamPos, MessageLog log)
        {
            SourcePosition start = new SourcePosition(streamPos);
            TokenType      type;
            string         value;

            string first = SubNumber(data, digits);

            if (first == null)
            {
                return(null);
            }

            if (data.CurrentItem == "x")
            {
                if (first != "0")
                {
                    return(null);
                }

                data.Advance();
                string hex = SubNumber(data, hexDigits);
                if (hex == null || data.CurrentItem == "." || data.CurrentItem == "x")
                {
                    return(null);
                }

                hex   = Convert.ToInt32(hex, 16).ToString("D");
                type  = TokenType.IntegerNumber;
                value = hex;
            }
            else if (data.CurrentItem == "." || data.CurrentItem.CaseInsensitiveEquals("e") || data.CurrentItem.CaseInsensitiveEquals("d"))
            {
                type = TokenType.FloatingNumber;
                string second = null;
                if (data.CurrentItem == ".")
                {
                    data.Advance();
                    second = SubNumber(data, digits);
                }
                if (data.CurrentItem.CaseInsensitiveEquals("e") || data.CurrentItem.CaseInsensitiveEquals("d"))
                {
                    data.Advance();
                    string exponent = SubNumber(data, digits);
                    if (exponent == null || data.CurrentItem == "." || data.CurrentItem == "x")
                    {
                        return(null);
                    }
                    value = $"{first}.{second ?? "0"}e{exponent}";
                }
                else if (second == null && data.CurrentItem == "f")
                {
                    data.Advance();
                    value = $"{first}.0";
                }
                else
                {
                    if (second == null && data.CurrentItem == "f")
                    {
                        data.Advance();
                    }
                    if (second == null || data.CurrentItem == "." || data.CurrentItem == "x")
                    {
                        return(null);
                    }

                    value = $"{first}.{second}";
                }

                if (data.CurrentItem == "f")
                {
                    data.Advance();
                }
            }
            else
            {
                type  = TokenType.IntegerNumber;
                value = first;
            }

            string peek         = data.CurrentItem;
            bool   hasDelimiter = string.IsNullOrWhiteSpace(peek) || Delimiters.Any(c => c.Keyword == peek);

            if (!hasDelimiter)
            {
                return(null);
            }
            streamPos = streamPos.GetModifiedPosition(0, data.CurrentIndex - start.CharIndex, data.CurrentIndex - start.CharIndex);
            SourcePosition end = new SourcePosition(streamPos);

            return(new Token <string>(type, value, start, end));
        }