Exemple #1
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (true)
            {
                //Find next position
                var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition + 1);
                //we either didn't find it
                if (newPos == -1)
                {
                    return(context.CreateErrorToken(Resources.ErrNoEndForRegex));// "No end symbol for regex literal."
                }
                source.PreviewPosition = newPos;
                if (source.PreviewChar != EndSymbol)
                {
                    //we hit CR or LF, this is an error
                    return(context.CreateErrorToken(Resources.ErrNoEndForRegex));
                }
                if (!CheckEscaped(source))
                {
                    break;
                }
            }
            source.PreviewPosition++;                                               //move after end symbol
            //save pattern length, we will need it
            var patternLen = source.PreviewPosition - source.Location.Position - 2; //exclude start and end symbol
            //read switches and turn them into options
            RegexOptions options  = RegexOptions.None;
            var          switches = string.Empty;

            while (ReadSwitch(source, ref options))
            {
                if (IsSet(RegexTermOptions.UniqueSwitches) && switches.Contains(source.PreviewChar))
                {
                    return(context.CreateErrorToken(Resources.ErrDupRegexSwitch, source.PreviewChar)); // "Duplicate switch '{0}' for regular expression"
                }
                switches += source.PreviewChar.ToString();
                source.PreviewPosition++;
            }
            //check following symbol
            if (!IsSet(RegexTermOptions.AllowLetterAfter))
            {
                var currChar = source.PreviewChar;
                if (char.IsLetter(currChar) || currChar == '_')
                {
                    return(context.CreateErrorToken(Resources.ErrInvRegexSwitch, currChar)); // "Invalid switch '{0}' for regular expression"
                }
            }
            var token = source.CreateToken(this.OutputTerminal);
            //we have token, now what's left is to set its Value field. It is either pattern itself, or Regex instance
            string pattern = token.Text.Substring(1, patternLen); //exclude start and end symbol
            object value   = pattern;

            if (IsSet(RegexTermOptions.CreateRegExObject))
            {
                value = new Regex(pattern, options);
            }
            token.Value   = value;
            token.Details = switches; //save switches in token.Details
            return(token);
        }
Exemple #2
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            // Quick check
            var lookAhead  = source.PreviewChar;
            var startIndex = _startSymbolsFirsts.IndexOf(lookAhead);

            if (startIndex < 0)
            {
                return(null);
            }

            // Match start symbols
            if (!BeginMatch(source, startIndex, lookAhead))
            {
                return(null);
            }

            // Match NewLine
            var result = CompleteMatch(source);

            if (result != null)
            {
                return(result);
            }

            // Report an error
            return(context.CreateErrorToken(Resources.ErrNewLineExpected));
        }
Exemple #3
0
        private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
        {
            var startPos = source.PreviewPosition;
            var termLen  = _singleTerminator.Length;

#if NETSTANDARD
            var stringComp = Grammar.CaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
#else
            var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
#endif
            int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
            if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
            {
                termPos = source.Text.Length;
            }
            if (termPos < 0)
            {
                return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
            }
            var textEnd = termPos;
            if (IsSet(FreeTextOptions.IncludeTerminator))
            {
                textEnd += termLen;
            }
            var tokenText = source.Text.Substring(startPos, textEnd - startPos);
            // The following line is a fix submitted by user rmcase
            source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
            return(source.CreateToken(this.OutputTerminal, tokenText));
        }
        public override Token TryMatch(ParsingContext context, ISourceStream source) 
        {
            Token result;
            if (context.VsLineScanState.Value != 0)
            {
                byte commentLevel = context.VsLineScanState.TokenSubType;
                result = CompleteMatch(context, source, commentLevel);
            } 
            else 
            {
                byte commentLevel = 0;
                if (!BeginMatch(context, source, ref commentLevel)) 
                    return null;

                result = CompleteMatch(context, source, commentLevel);
            }
            
            if (result != null) 
                return result;

            if (context.Mode == ParseMode.VsLineScan)
                return CreateIncompleteToken(context, source);

            return context.CreateErrorToken("unclosed comment");
        }
Exemple #5
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            if (context.VsLineScanState.Value != 0)
            {
                // we are continuing in line mode - restore internal env (none in this case)
                context.VsLineScanState.Value = 0;
            }
            else
            {
                //we are starting from scratch
                if (!BeginMatch(context, source))
                {
                    return(null);
                }
            }
            var result = CompleteMatch(context, source);

            if (result != null)
            {
                return(result);
            }
            //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            if (_isLineComment)
            {
                return(source.CreateToken(OutputTerminal));
            }

            if (context.Mode == ParseMode.VsLineScan)
            {
                return(CreateIncompleteToken(context, source));
            }

            return(context.CreateErrorToken(Resources.ErrUnclosedComment));
        }
        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;
            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return null;
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return source.CreateToken(this.OutputTerminal);
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced quoted string");
        }
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\n':
                    case '\r':
                    case ' ':
                    case '}':
                        if (source.PreviewPosition > source.Position)
                            return source.CreateToken(this.OutputTerminal);
                        return context.CreateErrorToken(Name + " was expected");
                }
                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced " + Name);
        }
Exemple #8
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   try {
     var textValue = ReadBody(context, source);
     if (textValue == null) return null; 
     var value = ConvertValue(context, textValue);
     return source.CreateToken(this.OutputTerminal, value);
   } catch(Exception ex) {
     //we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
     return context.CreateErrorToken(ex.Message);
   }
 }//method
Exemple #9
0
 protected internal override void OnValidateToken(ParsingContext context)
 {
     if (!IsSet(NumberOptions.AllowLetterAfter))
     {
         var current = context.Source.PreviewChar;
         if (char.IsLetter(current) || current == '_')
         {
             context.CurrentToken = context.CreateErrorToken(Resources.ErrNoLetterAfterNum); // "Number cannot be followed by a letter."
         }
     }
     base.OnValidateToken(context);
 }
Exemple #10
0
 public override Token TryMatch(ParsingContext context, ISourceStream source)
 {
     try {
         var textValue = ReadBody(context, source);
         if (textValue == null)
         {
             return(null);
         }
         var value = ConvertValue(context, textValue);
         return(source.CreateToken(this.OutputTerminal, value));
     } catch (Exception ex) {
         //we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
         return(context.CreateErrorToken(ex.Message));
     }
 }//method
Exemple #11
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   Token result;
   if (context.VsLineScanState.Value != 0) {
     // we are continuing in line mode - restore internal env (none in this case)
     context.VsLineScanState.Value = 0;
   } else {
     //we are starting from scratch
     if (!BeginMatch(context, source)) return null;
   }
   result = CompleteMatch(context, source);
   if (result != null) return result;
   //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
   if (_isLineComment)
     return source.CreateToken(this.OutputTerminal);
   if (context.Mode == ParseMode.VsLineScan)
     return CreateIncompleteToken(context, source);
   return context.CreateErrorToken(Resources.ErrUnclosedComment);
 }
    public override Token TryMatch(ParsingContext context, ISourceStream source) {
      // Quick check
      var lookAhead = source.PreviewChar;
      var startIndex = _startSymbolsFirsts.IndexOf(lookAhead);
      if (startIndex < 0)
        return null;

      // Match start symbols
      if (!BeginMatch(source, startIndex, lookAhead))
        return null;

      // Match NewLine
      var result = CompleteMatch(source);
      if (result != null)
        return result;

      // Report an error
      return context.CreateErrorToken(Resources.ErrNewLineExpected);
    }
Exemple #13
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            Token token;

            //Try quick parse first, but only if we're not continuing
            if (context.VsLineScanState.Value == 0)
            {
                token = QuickParse(context, source);
                if (token != null)
                {
                    return(token);
                }
                source.PreviewPosition = source.Position; //revert the position
            }

            CompoundTokenDetails details = new CompoundTokenDetails();

            InitDetails(context, details);

            if (context.VsLineScanState.Value == 0)
            {
                ReadPrefix(source, details);
            }
            if (!ReadBody(source, details))
            {
                return(null);
            }
            if (details.Error != null)
            {
                return(context.CreateErrorToken(details.Error));
            }
            if (details.IsPartial)
            {
                details.Value = details.Body;
            }
            else
            {
                ReadSuffix(source, details);

                if (!ConvertValue(details))
                {
                    if (string.IsNullOrEmpty(details.Error))
                    {
                        details.Error = Resources.ErrInvNumber;
                    }
                    return(context.CreateErrorToken(details.Error)); // "Failed to convert the value: {0}"
                }
            }
            token = CreateToken(context, source, details);

            if (details.IsPartial)
            {
                //Save terminal state so we can continue
                context.VsLineScanState.TokenSubType  = (byte)details.SubTypeIndex;
                context.VsLineScanState.TerminalFlags = (short)details.Flags;
                context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            }
            else
            {
                context.VsLineScanState.Value = 0;
            }
            return(token);
        }
 private Token TryMatchContentSimple(ParsingContext context, ISourceStream source) {
   var startPos = source.PreviewPosition;
   var termLen = _singleTerminator.Length;
   var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
   int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
   if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
     termPos = source.Text.Length;
   if (termPos < 0)
     return context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
   var textEnd = termPos;
   if (IsSet(FreeTextOptions.IncludeTerminator))
     textEnd += termLen;
   var tokenText = source.Text.Substring(startPos, textEnd - startPos);
   if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
     return null; 
   // The following line is a fix submitted by user rmcase
   source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos; 
   return source.CreateToken(this.OutputTerminal, tokenText); 
 }
        private Token MatchUnquoted(ParsingContext context, ISourceStream source)
        {
            if (source.PreviewChar == '{')
            {
                // Member names can't start with {
                if (this.IsMemberName)
                {
                    return null;
                }

                // Check for special {} at start of token indicating that this is a STRING token.
                if (source.NextPreviewChar != '}')
                {
                    return null;
                }

                source.PreviewPosition += 2;
            }

            var runningBraceTotal = 0;

            // This variable tracks the position of the last non whitespace (or significant whitespace).
            var lastNonWhitespacePosition = source.PreviewPosition;
            while (!source.EOF())
            {
                bool isWhiteSpace = false;
                switch (source.PreviewChar)
                {
                    case '{':
                        runningBraceTotal++;
                        break;

                    case '}':
                        if (--runningBraceTotal < 0)
                        {
                            return this.CreateToken(source, lastNonWhitespacePosition);
                        }

                        break;

                    case ',':
                        if (runningBraceTotal == 0)
                        {
                            return this.CreateToken(source, lastNonWhitespacePosition);
                        }

                        break;

                    case '=':
                        if (runningBraceTotal == 0)
                        {
                            // Equal sign. Only allowed after MemberNames.
                            return this.IsMemberName
                                ? this.CreateToken(source, lastNonWhitespacePosition)
                                : null;
                        }

                        break;

                    case '\\':
                        source.PreviewPosition++;
                        break;

                    default:
                        isWhiteSpace = Char.IsWhiteSpace(source.PreviewChar);
                        break;
                }

                source.PreviewPosition++;

                if (!isWhiteSpace)
                {
                    lastNonWhitespacePosition = source.PreviewPosition;
                }
            }

            return context.CreateErrorToken("Unterminated string terminal");
        }