Пример #1
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            string tokenText = string.Empty;

            while (true)
            {
                //Find next position
                var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
                if (newPos == -1)
                {
                    if (IsSet(FreeTextOptions.AllowEof))
                    {
                        source.PreviewPosition = source.Text.Length;
                        return(source.CreateToken(this.OutputTerminal));
                    }
                    else
                    {
                        return(null);
                    }
                }
                tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
                source.PreviewPosition = newPos;
                //if it is escape, add escaped text and continue search
                if (CheckEscape(source, ref tokenText))
                {
                    continue;
                }
                //check terminators
                if (CheckTerminators(source, ref tokenText))
                {
                    break; //from while (true)
                }
            }
            return(source.CreateToken(this.OutputTerminal, tokenText));
        }
Пример #2
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            var isEscape = source.PreviewChar == EscapeChar && EscapeChar != NoEscape;

            if (isEscape)
            {
                //return a token containing only escaped char
                var value = source.NextPreviewChar.ToString();
                source.PreviewPosition += 2;
                return(source.CreateToken(OutputTerminal, value));
            }

            var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position + 1);

            if (stopIndex == source.Location.Position)
            {
                return(null);
            }

            if (stopIndex < 0)
            {
                stopIndex = source.Text.Length;
            }

            source.PreviewPosition = stopIndex;
            return(source.CreateToken(OutputTerminal));
        }
Пример #3
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   string tokenText = string.Empty;
   while (true) {
     //Find next position
     var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
     if(newPos == -1) {
       if(IsSet(FreeTextOptions.AllowEof)) {
         source.PreviewPosition = source.Text.Length;
         return source.CreateToken(this.OutputTerminal);
       }  else
         return null;
     }
     if (newPos == source.PreviewPosition)   // DC
     {
         context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {});
         break;                              // DC
     }
     tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
     source.PreviewPosition = newPos;
     //if it is escape, add escaped text and continue search
     if (CheckEscape(source, ref tokenText)) 
       continue;
     //check terminators
     if (CheckTerminators(source, ref tokenText))
       break; //from while (true)        
   }
   return source.CreateToken(this.OutputTerminal, tokenText);
 }
Пример #4
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
        {
            if (commentLevel == 0)
            {
                var line_breaks  = new char[] { '\n', '\r', '\v' };
                var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
                if (firstCharPos > 0)
                {
                    source.PreviewPosition = firstCharPos;
                }
                else
                {
                    source.PreviewPosition = source.Text.Length;
                }

                return(source.CreateToken(this.OutputTerminal));
            }

            while (!source.EOF())
            {
                string text    = source.Text.Substring(source.PreviewPosition);
                var    matches = Regex.Matches(text, @"\](=*)\]");
                foreach (Match match in matches)
                {
                    if (match.Groups[1].Value.Length == (int)commentLevel - 1)
                    {
                        source.PreviewPosition += match.Index + match.Length;

                        if (context.VsLineScanState.Value != 0)
                        {
                            //We are using line-mode and begin terminal was on previous line.
                            SourceLocation tokenStart = new SourceLocation();
                            tokenStart.Position = 0;

                            string lexeme = source.Text.Substring(0, source.PreviewPosition);

                            context.VsLineScanState.Value = 0;
                            return(new Token(this, tokenStart, lexeme, null));
                        }
                        else
                        {
                            return(source.CreateToken(this.OutputTerminal));
                        }
                    }
                }

                source.PreviewPosition++;
            }
            //The full match wasn't found, store the state for future parsing.
            //   context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            context.VsLineScanState.TokenSubType = commentLevel;
            return(null);
        }
Пример #5
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            Match m = Expression.Match(source.Text, source.PreviewPosition);
            if (!m.Success || m.Index != source.PreviewPosition)
                return null;

            source.PreviewPosition += m.Length;

            if (ValueSelector != null)
                return source.CreateToken(this, ValueSelector.Invoke(m));

            return source.CreateToken(OutputTerminal);
        }
Пример #6
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   bool isEscape = source.PreviewChar == EscapeChar && EscapeChar != NoEscape;
   if(isEscape) {
     //return a token containing only escaped char
     var value = source.NextPreviewChar.ToString(); 
     source.PreviewPosition += 2; 
     return source.CreateToken(this.OutputTerminal, value);  
   }
   var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position + 1);
   if (stopIndex == source.Location.Position) return null; 
   if (stopIndex < 0) stopIndex = source.Text.Length; 
   source.PreviewPosition = stopIndex;
   return source.CreateToken(this.OutputTerminal);
 }//method
Пример #7
0
 private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
 {
     if (commentLevel == 0)
     {
         char[] anyOf = new char[]
         {
             '\n',
             '\r',
             '\v'
         };
         int num = source.Text.IndexOfAny(anyOf, source.PreviewPosition);
         if (num > 0)
         {
             source.PreviewPosition = num;
         }
         else
         {
             source.PreviewPosition = source.Text.Length;
         }
         return(source.CreateToken(base.OutputTerminal));
     }
     while (!source.EOF())
     {
         foreach (Match match in Regex.Matches(source.Text.Substring(source.PreviewPosition), "\\](=*)\\]"))
         {
             if (match.Groups[1].Value.Length == (int)(commentLevel - 1))
             {
                 source.PreviewPosition += match.Index + match.Length;
                 Token result;
                 if (context.VsLineScanState.Value != 0)
                 {
                     SourceLocation location = default(SourceLocation);
                     location.Position = 0;
                     string text = source.Text.Substring(0, source.PreviewPosition);
                     context.VsLineScanState.Value = 0;
                     result = new Token(this, location, text, null);
                     return(result);
                 }
                 result = source.CreateToken(base.OutputTerminal);
                 return(result);
             }
         }
         int previewPosition = source.PreviewPosition;
         source.PreviewPosition = previewPosition + 1;
     }
     context.VsLineScanState.TokenSubType = commentLevel;
     return(null);
 }
Пример #8
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            var text = source.Text;

            foreach (var entry in Constants)
            {
                source.PreviewPosition = source.Position;
                var constant = entry.Key;
                if (source.PreviewPosition + constant.Length > text.Length)
                {
                    continue;
                }

                if (source.MatchSymbol(constant))
                {
                    source.PreviewPosition += constant.Length;
                    if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
                    {
                        continue; //make sure it is delimiter
                    }

                    return(source.CreateToken(OutputTerminal, entry.Value));
                }
            }
            return(null);
        }
Пример #9
0
        /// <summary>
        /// Tries to match the current position in the <paramref name="source"/> with the rules for a TYPENAME terminal;
        /// returning null if there is no match, returning an  error token if the TYPENAME is malformed or there is no
        /// match and returning a TYPENAME token
        /// otherwise.
        /// </summary>
        /// <param name="context">
        /// The context in which the match is occuring.
        /// </param>
        /// <param name="source">
        /// The source to try to match.
        /// </param>
        /// <returns>
        /// An error token if the TYPENAME is malformed or there is no match and returning a TYPENAME token
        /// otherwise.
        /// </returns>
        public override Token TryMatch([NotNull] ParsingContext context, [NotNull] ISourceStream source)
        {
            if (context == null)
            {
                throw new ArgumentNullException(nameof(context));
            }

            if (source == null)
            {
                throw new ArgumentNullException(nameof(source));
            }

            while (true)
            {
                if (source.EOF())
                {
                    return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
                }

                if (source.PreviewChar == '}' || char.IsWhiteSpace(source.PreviewChar))
                {
                    var token = source.CreateToken(this);
                    if (string.IsNullOrWhiteSpace(token.ValueString))
                    {
                        return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
                    }

                    return(token);
                }

                ++source.PreviewPosition;
            }
        }
Пример #10
0
        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;
            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return null;
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return source.CreateToken(this.OutputTerminal);
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced quoted string");
        }
Пример #11
0
 public override Token TryMatch(ParsingContext context, ISourceStream source)
 {
     foreach (var prefix in prefixes)
     {
         if (!PrefixMatches(source, prefix))
         {
             continue;
         }
         if (DirectivePrefix.StartsWith(prefix) && PrefixMatches(source, DirectivePrefix))
         {
             continue;
         }
         var          startIndex = source.Location.Position + prefix.Length;
         const string suffix     = "%>";
         var          endIndex   = source.Text.IndexOf(suffix, startIndex, StringComparison.Ordinal);
         if (endIndex < 0)
         {
             continue;
         }
         source.PreviewPosition = endIndex + suffix.Length;
         var tokenValue = source.Text.Substring(startIndex, endIndex - startIndex);
         return(source.CreateToken(OutputTerminal, tokenValue));
     }
     return(null);
 }
Пример #12
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            if (context.VsLineScanState.Value != 0)
            {
                // we are continuing in line mode - restore internal env (none in this case)
                context.VsLineScanState.Value = 0;
            }
            else
            {
                //we are starting from scratch
                if (!BeginMatch(context, source))
                {
                    return(null);
                }
            }
            var result = CompleteMatch(context, source);

            if (result != null)
            {
                return(result);
            }
            //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            if (_isLineComment)
            {
                return(source.CreateToken(OutputTerminal));
            }

            if (context.Mode == ParseMode.VsLineScan)
            {
                return(CreateIncompleteToken(context, source));
            }

            return(context.CreateErrorToken(Resources.ErrUnclosedComment));
        }
Пример #13
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   Match m = _expression.Match(source.Text, source.PreviewPosition);
   if (!m.Success || m.Index != source.PreviewPosition) 
     return null;
   source.PreviewPosition += m.Length;
   return source.CreateToken(this.OutputTerminal); 
 }
Пример #14
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
        {
            string text    = source.Text.Substring(source.PreviewPosition);
            var    matches = Regex.Matches(text, @"\](=*)\]");

            foreach (Match match in matches)
            {
                if (match.Groups[1].Value.Length == (int)level)
                {
                    source.PreviewPosition += match.Index + match.Length;


                    if (context.VsLineScanState.Value != 0)
                    {
                        SourceLocation tokenStart = new SourceLocation();
                        tokenStart.Position = 0;

                        string lexeme = source.Text.Substring(0, source.PreviewPosition);

                        context.VsLineScanState.Value = 0;
                        return(new Token(this, tokenStart, lexeme, null));
                    }
                    else
                    {
                        return(source.CreateToken(this.OutputTerminal));
                    }
                }
            }

            context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            context.VsLineScanState.TokenSubType  = level;
            return(null);
        }
Пример #15
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (true)
            {
                //Find next position
                var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition + 1);
                //we either didn't find it
                if (newPos == -1)
                {
                    return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrNoEndForRegex));// "No end symbol for regex literal."
                }
                source.PreviewPosition = newPos;
                if (source.PreviewChar != EndSymbol)
                {
                    //we hit CR or LF, this is an error
                    return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrNoEndForRegex));
                }
                if (!CheckEscaped(source))
                {
                    break;
                }
            }
            source.PreviewPosition++;                                               //move after end symbol
            //save pattern length, we will need it
            var patternLen = source.PreviewPosition - source.Location.Position - 2; //exclude start and end symbol
            //read switches and turn them into options
            RegexOptions options  = RegexOptions.None;
            var          switches = string.Empty;

            while (ReadSwitch(source, ref options))
            {
                if (IsSet(RegexTermOptions.UniqueSwitches) && switches.Contains(source.PreviewChar))
                {
                    return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrDupRegexSwitch, source.PreviewChar)); // "Duplicate switch '{0}' for regular expression"
                }
                switches += source.PreviewChar.ToString();
                source.PreviewPosition++;
            }
            //check following symbol
            if (!IsSet(RegexTermOptions.AllowLetterAfter))
            {
                var currChar = source.PreviewChar;
                if (char.IsLetter(currChar) || currChar == '_')
                {
                    return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrInvRegexSwitch, currChar)); // "Invalid switch '{0}' for regular expression"
                }
            }
            var token = source.CreateToken(this.OutputTerminal);
            //we have token, now what's left is to set its Value field. It is either pattern itself, or Regex instance
            string pattern = token.Text.Substring(1, patternLen); //exclude start and end symbol
            object value   = pattern;

            if (IsSet(RegexTermOptions.CreateRegExObject))
            {
                value = new Regex(pattern, options);
            }
            token.Value   = value;
            token.Details = switches; //save switches in token.Details
            return(token);
        }
Пример #16
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;

            if (isHeadingOrList)
            {
                bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
                if (!isAfterNewLine)
                {
                    return(null);
                }
            }
            if (!source.MatchSymbol(OpenTag, true))
            {
                return(null);
            }
            source.PreviewPosition += OpenTag.Length;
            //For headings and lists require space after
            if (TermType == WikiTermType.Heading || TermType == WikiTermType.List)
            {
                const string whitespaces = " \t\r\n\v";
                if (!whitespaces.Contains(source.PreviewChar))
                {
                    return(null);
                }
            }
            var token = source.CreateToken(this.OutputTerminal);

            return(token);
        }
Пример #17
0
        //Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9
        // so we try to do a quick parse for these, without starting the whole general process
        protected override Token QuickParse(ParsingContext context, ISourceStream source)
        {
            if (IsSet(NumberOptions.DisableQuickParse))
            {
                return(null);
            }
            char current = source.PreviewChar;

            //it must be a digit followed by a terminator
            if (!char.IsDigit(current) || GrammarData.WhitespaceAndDelimiters.IndexOf(source.NextPreviewChar) < 0)
            {
                return(null);
            }
            int    iValue = current - '0';
            object value  = null;

            switch (DefaultIntTypes[0])
            {
            case TypeCode.Int32: value = iValue; break;

            case TypeCode.UInt32: value = (UInt32)iValue; break;

            case TypeCode.Byte: value = (byte)iValue; break;

            case TypeCode.SByte: value = (sbyte)iValue; break;

            case TypeCode.Int16: value = (Int16)iValue; break;

            case TypeCode.UInt16: value = (UInt16)iValue; break;

            default: return(null);
            }
            source.PreviewPosition++;
            return(source.CreateToken(this.OutputTerminal, value));
        }
        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;

            if (quoteChar != '\'' && quoteChar != '"')
            {
                return(null);
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return(source.CreateToken(this.OutputTerminal));
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }
                source.PreviewPosition++;
            }

            return(context.CreateErrorToken("Unbalanced quoted string"));
        }
Пример #19
0
 public override Token TryMatch(ParsingContext context, ISourceStream source)
 {
     try
     {
         var left  = GetByte(source.PreviewChar);
         var right = GetByte(source.NextPreviewChar);
         if (left >= 0 && right >= 0)
         {
             var list = new List <byte>();
             do
             {
                 list.Add(Convert.ToByte(left * 16 + right));
                 source.PreviewPosition += 2;
                 left  = GetByte(source.PreviewChar);
                 right = GetByte(source.NextPreviewChar);
             } while (left >= 0 && right >= 0);
             return(source.CreateToken(this.OutputTerminal, list.ToArray()));
         }
         return(null);
     }
     catch (Exception ex)
     {
         return(context.CreateErrorToken(ex.Message));
     }
 }
Пример #20
0
 private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
 {
     foreach (Match match in Regex.Matches(source.Text.Substring(source.PreviewPosition), "\\](=*)\\]"))
     {
         if (match.Groups[1].Value.Length == (int)level)
         {
             source.PreviewPosition += match.Index + match.Length;
             Token result;
             if (context.VsLineScanState.Value != 0)
             {
                 SourceLocation location = default(SourceLocation);
                 location.Position = 0;
                 string text = source.Text.Substring(0, source.PreviewPosition);
                 context.VsLineScanState.Value = 0;
                 result = new Token(this, location, text, null);
                 return(result);
             }
             result = source.CreateToken(base.OutputTerminal);
             return(result);
         }
     }
     context.VsLineScanState.TerminalIndex = MultilineIndex;
     context.VsLineScanState.TokenSubType  = level;
     return(null);
 }
Пример #21
0
 private Token CreateIncompleteToken(ParsingContext context, ISourceStream source) {
   source.PreviewPosition = source.Text.Length;
   Token result = source.CreateToken(this.OutputTerminal);
   result.Flags |= TokenFlags.IsIncomplete;
   context.VsLineScanState.TerminalIndex = this.MultilineIndex;
   return result; 
 }
        public override Token?TryMatch(ParsingContext context, ISourceStream source)
        {
            var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position);

            if (stopIndex == source.Location.Position)
            {
                return(null);
            }
            if (stopIndex < 0)
            {
                stopIndex = source.Text.Length;
            }
            source.PreviewPosition = stopIndex;

            // preserve leading whitespace, if present.
            int start = source.Location.Position;

            if (preserveLeadingWhitespace)
            {
                while (start > 0 && char.IsWhiteSpace(source.Text, start - 1))
                {
                    start--;
                }
            }
            var content = source.Text.Substring(start, stopIndex - start);

            return(source.CreateToken(this.OutputTerminal, content));
        }
Пример #23
0
        private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
        {
            var startPos = source.PreviewPosition;
            var termLen  = _singleTerminator.Length;

#if NETSTANDARD
            var stringComp = Grammar.CaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
#else
            var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
#endif
            int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
            if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
            {
                termPos = source.Text.Length;
            }
            if (termPos < 0)
            {
                return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
            }
            var textEnd = termPos;
            if (IsSet(FreeTextOptions.IncludeTerminator))
            {
                textEnd += termLen;
            }
            var tokenText = source.Text.Substring(startPos, textEnd - startPos);
            // The following line is a fix submitted by user rmcase
            source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
            return(source.CreateToken(this.OutputTerminal, tokenText));
        }
        protected override Token QuickParse(ParsingContext context, ISourceStream source)
        {
            if (AllFirstChars.IndexOf(source.PreviewChar) < 0)
            {
                return(null);
            }
            source.PreviewPosition++;
            while (AllChars.IndexOf(source.PreviewChar) >= 0 && !source.EOF())
            {
                source.PreviewPosition++;
            }
            //if it is not a terminator then cancel; we need to go through full algorithm
            if (GrammarData.WhitespaceAndDelimiters.IndexOf(source.PreviewChar) < 0)
            {
                return(null);
            }
            var token = source.CreateToken(this.OutputTerminal);

            if (CaseRestriction != CaseRestriction.None && !CheckCaseRestriction(token.ValueString))
            {
                return(null);
            }
            //!!! Do not convert to common case (all-lower) for case-insensitive grammar. Let identifiers remain as is,
            //  it is responsibility of interpreter to provide case-insensitive read/write operations for identifiers
            // if (!this.GrammarData.Grammar.CaseSensitive)
            //    token.Value = token.Text.ToLower(CultureInfo.InvariantCulture);
            CheckReservedWord(token);
            token.Symbol = SymbolTable.Symbols.TextToSymbol(token.ValueString);
            return(token);
        }
Пример #25
0
        /// <summary>
        /// Creates a MEMBERNAME token.
        /// </summary>
        /// <param name="source">
        /// The source from which to create the token.
        /// </param>
        /// <param name="builder">
        /// The builder that contains the MEMBERNAME value.
        /// </param>
        /// <returns>
        /// The <see cref="Token"/>.
        /// </returns>
        private Token CreateMemberNameToken(ISourceStream source, StringBuilder builder)
        {
            var terminal = this.TerminalType == MemberNameOrStringTerminalType.String
                ? this.OtherTerminal.OutputTerminal
                : this.OutputTerminal;

            return(source.CreateToken(terminal, builder.ToString().Trim()));
        }
Пример #26
0
 public Token CreateErrorToken(string message, params object[] args)
 {
     if (args != null && args.Length > 0)
     {
         message = string.Format(message, args);
     }
     return(Source.CreateToken(Language.Grammar.SyntaxError, message));
 }
Пример #27
0
        private Token TryMatchContentExtended(ParsingContext context, ISourceStream source)
        {
            StringBuilder tokenText = new StringBuilder();

            while (true)
            {
                //Find next position of one of stop chars
                var nextPos = source.IndexOfAny(_stopChars, source.PreviewPosition);
                if (nextPos == -1)
                {
                    if (IsSet(FreeTextOptions.AllowEof))
                    {
                        source.PreviewPosition = source.Length;
                        return(source.CreateToken(this.OutputTerminal));
                    }
                    else
                    {
                        return(null);
                    }
                }
                var newText = source.GetText(source.PreviewPosition, nextPos - source.PreviewPosition);
                tokenText.Append(newText);
                source.PreviewPosition = nextPos;
                //if it is escape, add escaped text and continue search
                if (CheckEscape(source, tokenText))
                {
                    continue;
                }
                //check terminators
                if (CheckTerminators(source, tokenText))
                {
                    break; //from while (true); we reached
                }
                //The current stop is not at escape or terminator; add this char to token text and move on
                tokenText.Append(source.PreviewChar);
                source.PreviewPosition++;
            }//while
            var text = tokenText.ToString();

            if (string.IsNullOrEmpty(text) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
            {
                return(null);
            }
            return(source.CreateToken(this.OutputTerminal, text));
        }
Пример #28
0
        private Token CreateIncompleteToken(ParsingContext context, ISourceStream source)
        {
            source.PreviewPosition = source.Text.Length;
            Token expr_1D = source.CreateToken(base.OutputTerminal);

            expr_1D.Flags |= TokenFlags.IsIncomplete;
            context.VsLineScanState.TerminalIndex = MultilineIndex;
            return(expr_1D);
        }
        private Token CreateIncompleteToken(ParsingContext context, ISourceStream source)
        {
            source.PreviewPosition = source.Text.Length;
            Token result = source.CreateToken(this.OutputTerminal);

            result.Flags |= TokenFlags.IsIncomplete;
            context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            return(result);
        }
Пример #30
0
        protected virtual Token CreateToken(ParsingContext context, ISourceStream source, CompoundTokenDetails details)
        {
            var token = source.CreateToken(this.OutputTerminal, details.Value);

            token.Details = details;
            if (details.IsPartial)
            {
                token.Flags |= TokenFlags.IsIncomplete;
            }
            return(token);
        }
Пример #31
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            foreach (var m in _matchers)
            {
                var backupOffset = source.PreviewPosition;

                var start = m.TryGetInitiator(source.Text, source.PreviewPosition, !context.Language.Grammar.CaseSensitive);
                if (start != null)
                {
                    //Locate the end terminal
                    var end = m.TryGetTerminator(start, !context.Language.Grammar.CaseSensitive);

                    if (end == null)
                    {
                        continue;
                    }

                    source.PreviewPosition += start.Length;

                    var startPos = source.PreviewPosition;
                    var endPos   = -2;
                    while (endPos == -2)
                    {
                        endPos = source.Text.IndexOf(end, source.PreviewPosition);

                        if ((m.Flags & StringOptions.NoEscapes) != 0)
                        {
                            continue;
                        }

                        if (IsEndQuoteEscaped(source.Text, endPos))
                        {
                            source.PreviewPosition = endPos + end.Length;
                            endPos = -2;
                        }
                    }
                    if (endPos == -1)
                    {
                        continue;
                    }

                    source.PreviewPosition = endPos + end.Length;
                    var tokenText = source.Text.Substring(startPos, endPos - startPos);

                    if ((m.Flags & StringOptions.NoEscapes) == 0)
                    {
                        tokenText = ConvertValue(context, tokenText, m);
                    }

                    return(source.CreateToken(this.OutputTerminal, tokenText));
                }
            }
            return(null);
        }
Пример #32
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   try {
     var textValue = ReadBody(context, source);
     if (textValue == null) return null; 
     var value = ConvertValue(context, textValue);
     return source.CreateToken(this.OutputTerminal, value);
   } catch(Exception ex) {
     //we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
     return source.CreateErrorToken(ex.Message);
   }
 }//method
Пример #33
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            Match m = _expression.Match(source.Text, source.PreviewPosition);

            if (!m.Success || m.Index != source.PreviewPosition)
            {
                return(null);
            }
            source.PreviewPosition += m.Length;
            return(source.CreateToken(this.OutputTerminal));
        }
Пример #34
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   char current = source.PreviewChar;
   if (!LineTerminators.Contains(current)) return null;
   //Treat \r\n as a single terminator
   bool doExtraShift = (current == '\r' && source.NextPreviewChar == '\n');
   source.PreviewPosition++; //main shift
   if (doExtraShift)
     source.PreviewPosition++;
   Token result = source.CreateToken(this.OutputTerminal);
   return result;
 }
Пример #35
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            var match = binaryRegex.Match(source.Text, source.PreviewPosition);

            if (!match.Success || match.Index != source.PreviewPosition)
            {
                return(null);
            }
            source.Position        += 2;
            source.PreviewPosition += match.Length;
            return(source.CreateToken(OutputTerminal));
        }
Пример #36
0
        /// <summary>
        /// Creates a STRING token.
        /// </summary>
        /// <param name="context">
        /// The context in which the parse is occuring.
        /// </param>
        /// <param name="source">
        /// The source from which to create the token.
        /// </param>
        /// <param name="builder">
        /// The builder that contains the STRING value.
        /// </param>
        /// <returns>
        /// The STRING <see cref="Token"/>.
        /// </returns>
        private Token CreateStringToken(ParsingContext context, ISourceStream source, StringBuilder builder)
        {
            var terminal = this.TerminalType == MemberNameOrStringTerminalType.MemberName
                ? this.OtherTerminal.OutputTerminal
                : this.OutputTerminal;

            var text      = builder.ToString();
            var parseTree = new Parser(context.Language).Parse(text);
            var value     = parseTree.Status == ParseTreeStatus.Parsed ? (object)parseTree : text;

            return(source.CreateToken(terminal, value));
        }
Пример #37
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source)
        {
            //Find end symbol
            while (!source.EOF())
            {
                var firstCharPos = source.Text.IndexOf('{', source.PreviewPosition);
                if (firstCharPos < 0)
                {
                    source.PreviewPosition = source.Text.Length;
                    return(source.CreateToken(this.OutputTerminal));
                }
                //We found a character that might start an end symbol; let's see if it is true.
                source.PreviewPosition = firstCharPos;
                if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
                {
                    return(source.CreateToken(this.OutputTerminal));
                }
                source.PreviewPosition++; //move to the next char and try again
            }

            return(source.CreateToken(this.OutputTerminal));
        }
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   if (!source.MatchSymbol(OpenTag, true)) return null;
   source.PreviewPosition += OpenTag.Length;
   var endPos = source.Text.IndexOf(CloseTag, source.PreviewPosition);
   string content; 
   if(endPos > 0) {
     content = source.Text.Substring(source.PreviewPosition, endPos - source.PreviewPosition); 
     source.PreviewPosition = endPos + CloseTag.Length;
   } else {
     content = source.Text.Substring(source.PreviewPosition, source.Text.Length - source.PreviewPosition); 
     source.PreviewPosition = source.Text.Length;
   }
   var token = source.CreateToken(this.OutputTerminal, content); 
   return token;      
 }
Пример #39
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
   if(isHeadingOrList) {
       bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
       if(!isAfterNewLine)  return null;
   }
   if(!source.MatchSymbol(OpenTag)) return null;
   source.PreviewPosition += OpenTag.Length;
   //For headings and lists require space after
   if(TermType == WikiTermType.Heading || TermType == WikiTermType.List) {
     const string whitespaces = " \t\r\n\v";
     if (!whitespaces.Contains(source.PreviewChar)) return null; 
   }
   var token = source.CreateToken(this.OutputTerminal);
   return token; 
 }
        protected Token CreateToken(ISourceStream source, int lastNonWhitespacePosition)
        {
            if (lastNonWhitespacePosition > source.Position)
            {
                // Remember position
                var pos = source.PreviewPosition;

                // Move position back to last non whitespace (or significant whitespace)
                source.PreviewPosition = lastNonWhitespacePosition;
                var token = source.CreateToken(this.OutputTerminal);

                // Restore position
                source.PreviewPosition = pos;
                return token;
            }
            return null;
        }
Пример #41
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   Token result;
   if (context.VsLineScanState.Value != 0) {
     // we are continuing in line mode - restore internal env (none in this case)
     context.VsLineScanState.Value = 0;
   } else {
     //we are starting from scratch
     if (!BeginMatch(context, source)) return null;
   }
   result = CompleteMatch(context, source);
   if (result != null) return result;
   //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
   if (_isLineComment)
     return source.CreateToken(this.OutputTerminal);
   if (context.Mode == ParseMode.VsLineScan)
     return CreateIncompleteToken(context, source);
   return context.CreateErrorToken(Resources.ErrUnclosedComment);
 }
 Token MatchCodeTerminal(Terminal terminal, ParsingContext context, ISourceStream source)
 {
     var remaining = source.Text.Substring(source.Location.Position);
     var stack = 1;
     var token = "";
     while (stack > 0)
     {
         var index = remaining.IndexOf('}') + 1;
         var cut = remaining.Substring(0, index);
         token += cut;
         remaining = remaining.Substring(index);
         stack += cut.Count(x => x == '{');
         stack--;
     }
     token = token.Substring(0, token.Length - 1);
     source.PreviewPosition += token.Length;
     return source.CreateToken(terminal.OutputTerminal);
 }
Пример #43
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\n':
                    case '\r':
                    case ' ':
                    case '}':
                        if (source.PreviewPosition > source.Position)
                            return source.CreateToken(this.OutputTerminal);
                        return context.CreateErrorToken(Name + " was expected");
                }
                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced " + Name);
        }
Пример #44
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
        {
            string text = source.Text.Substring(source.PreviewPosition);
            var matches = Regex.Matches(text, @"\](=*)\]");
            foreach(Match match in matches) 
            {
                if (match.Groups[1].Value.Length == (int)level)
                {
                    source.PreviewPosition += match.Index + match.Length;


                    if (context.VsLineScanState.Value != 0)
                    {
                        SourceLocation tokenStart = new SourceLocation();
                        tokenStart.Position = 0;

                        string lexeme = source.Text.Substring(0, source.PreviewPosition);

                        context.VsLineScanState.Value = 0;
                        return new Token(this, tokenStart, lexeme, null);
                    }
                    else
                    {
                        return source.CreateToken(this.OutputTerminal); 
                    }
                }
            }

            context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            context.VsLineScanState.TokenSubType = level;
            return null;
        }
Пример #45
0
 private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
 {
     var startPos = source.PreviewPosition;
       int p = source.Text.IndexOf(_singleTerminator, startPos, Grammar.StringComparisonMode);
       if (p < 0 && IsSet(FreeTextOptions.AllowEof))
     p = source.Text.Length;
       if (p < 0)
     return source.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
       var tokenText = source.Text.Substring(startPos, p - startPos);
       return source.CreateToken(this.OutputTerminal, tokenText);
 }
Пример #46
0
 public override Token TryMatch(ParsingContext context, ISourceStream source)
 {
     var Match = MatchRegex.Match(source.Text, source.PreviewPosition);
     if (!Match.Success || Match.Index != source.PreviewPosition) return null;
     source.PreviewPosition += Match.Length;
     return source.CreateToken(this.OutputTerminal);
 }
Пример #47
0
    private Token CompleteMatch(ISourceStream source) {
      if (source.EOF())
        return null;

      do {
        // Match NewLine
        var lookAhead = source.PreviewChar;
        if (LineTerminators.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          // Treat \r\n as single NewLine
          if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
            source.PreviewPosition++;
          break;
        }

        // Eat up whitespace
        if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          continue;
        }

        // Fail on anything else
        return null;
      }
      while (!source.EOF());

      // Create output token
      return source.CreateToken(this.OutputTerminal);
    }
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   return source.CreateToken(this); //Create an empty token representing an implied symbol.
 }
Пример #49
0
 private Token CompleteMatch(ParsingContext context, ISourceStream source) {
   //Find end symbol
   while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
     else 
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
     if (firstCharPos < 0) {
       source.PreviewPosition = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.PreviewPosition = firstCharPos;
     foreach (string endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol)) {
         //We found end symbol; eat end symbol only if it is not line comment.
         // For line comment, leave LF symbol there, it might be important to have a separate LF token
         if (!_isLineComment)
           source.PreviewPosition += endSymbol.Length;
         return source.CreateToken(this.OutputTerminal); 
       }//if
     }//foreach endSymbol
     source.PreviewPosition++; //move to the next char and try again    
   }//while
   return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }//method
Пример #50
0
 private Token TryMatchContentExtended(ParsingContext context, ISourceStream source) {
   StringBuilder tokenText = new StringBuilder();
   while (true) {
     //Find next position of one of stop chars
     var nextPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
     if(nextPos == -1) {
       if(IsSet(FreeTextOptions.AllowEof)) {
         source.PreviewPosition = source.Text.Length;
         return source.CreateToken(this.OutputTerminal);
       }  else
         return null;
     }
     var newText = source.Text.Substring(source.PreviewPosition, nextPos - source.PreviewPosition);
     tokenText.Append(newText);
     source.PreviewPosition = nextPos;
     //if it is escape, add escaped text and continue search
     if (CheckEscape(source, tokenText)) 
       continue;
     //check terminators
     if (CheckTerminators(source, tokenText))
       break; //from while (true); we reached 
     //The current stop is not at escape or terminator; add this char to token text and move on 
     tokenText.Append(source.PreviewChar);
     source.PreviewPosition++; 
   }//while
   var text = tokenText.ToString(); 
   if (string.IsNullOrEmpty(text) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
     return null;
   return source.CreateToken(this.OutputTerminal, text);
 }
Пример #51
0
 private Token TryMatchContentSimple(ParsingContext context, ISourceStream source) {
   var startPos = source.PreviewPosition;
   var termLen = _singleTerminator.Length;
   var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
   int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
   if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
     termPos = source.Text.Length;
   if (termPos < 0)
     return context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
   var textEnd = termPos;
   if (IsSet(FreeTextOptions.IncludeTerminator))
     textEnd += termLen;
   var tokenText = source.Text.Substring(startPos, textEnd - startPos);
   if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
     return null; 
   // The following line is a fix submitted by user rmcase
   source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos; 
   return source.CreateToken(this.OutputTerminal, tokenText); 
 }
Пример #52
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
        {
            if (commentLevel == 0)
            {
                var line_breaks = new char[] { '\n', '\r', '\v' };
                var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
                if (firstCharPos > 0)
                {
                    source.PreviewPosition = firstCharPos;
                }
                else
                {
                    source.PreviewPosition = source.Text.Length;
                }

                return source.CreateToken(this.OutputTerminal);
            }

            while (!source.EOF())
            {      
               string text = source.Text.Substring(source.PreviewPosition);
                var matches = Regex.Matches(text, @"\](=*)\]");
                foreach (Match match in matches)
                {
                    if (match.Groups[1].Value.Length == (int)commentLevel - 1)
                    {
                        source.PreviewPosition += match.Index + match.Length;

                        if (context.VsLineScanState.Value != 0)
                        {
                            SourceLocation tokenStart = new SourceLocation();
                            tokenStart.Position = 0;

                            string lexeme = source.Text.Substring(0, source.PreviewPosition);

                            context.VsLineScanState.Value = 0;
                            return new Token(this, tokenStart, lexeme, null);
                        }
                        else
                        {
                            return source.CreateToken(this.OutputTerminal);
                        }
                    }
                }

                source.PreviewPosition++;
            }
            
			context.VsLineScanState.TokenSubType = commentLevel;
            return null;
        }