Example #1
0
        private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
        {
            var startPos   = source.PreviewPosition;
            var termLen    = _singleTerminator.Length;
            var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
            int termPos    = source.IndexOf(_singleTerminator, startPos, stringComp);

            if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
            {
                termPos = source.Length;
            }
            if (termPos < 0)
            {
                return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
            }
            var textEnd = termPos;

            if (IsSet(FreeTextOptions.IncludeTerminator))
            {
                textEnd += termLen;
            }
            var tokenText = source.GetText(startPos, textEnd - startPos);

            if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
            {
                return(null);
            }
            // The following line is a fix submitted by user rmcase
            source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
            return(source.CreateToken(this.OutputTerminal, tokenText));
        }
Example #2
0
        private string ReadQuotedBody(ParsingContext context, ISourceStream source)
        {
            const char    dQuoute = '"';
            StringBuilder sb      = null;
            var           from    = source.Location.Position + 1; //skip initial double quote

            while (true)
            {
                var until = source.IndexOf(dQuoute, from);
                if (until < 0)
                {
                    throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value."
                }
                source.PreviewPosition = until;                          //now points at double-quote
                var piece = source.GetText(from, until - from);
                source.PreviewPosition++;                                //move after double quote
                if (source.PreviewChar != dQuoute && sb == null)
                {
                    return(piece); //quick path - if sb (string builder) was not created yet, we are looking at the very first segment;
                }
                // and if we found a standalone dquote, then we are done - the "piece" is the result.
                if (sb == null)
                {
                    sb = new StringBuilder(100);
                }
                sb.Append(piece);
                if (source.PreviewChar != dQuoute)
                {
                    return(sb.ToString());
                }
                //we have doubled double-quote; add a single double-quoute char to the result and move over both symbols
                sb.Append(dQuoute);
                from = source.PreviewPosition + 1;
            }
        }
Example #3
0
        protected override string ReadBody(ParsingContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(StartSymbol))
            {
                return(null);                            //this will result in null returned from TryMatch, no token
            }
            var start = source.Location.Position + StartSymbol.Length;
            var end   = source.IndexOf(EndSymbol, start);

            if (end < 0)
            {
                return(null);
            }
            var body = source.GetText(start, end - start);

            source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
            return(body);
        }
Example #4
0
 private Token CompleteMatch(ParsingContext context, ISourceStream source)
 {
     //Find end symbol
     while (!source.EOF())
     {
         int firstCharPos;
         if (EndSymbols.Count == 1)
         {
             firstCharPos = source.IndexOf(EndSymbols[0], source.PreviewPosition);
         }
         else
         {
             firstCharPos = source.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
         }
         if (firstCharPos < 0)
         {
             source.PreviewPosition = source.Length;
             return(null); //indicating error
         }
         //We found a character that might start an end symbol; let's see if it is true.
         source.PreviewPosition = firstCharPos;
         foreach (string endSymbol in EndSymbols)
         {
             if (source.MatchSymbol(endSymbol))
             {
                 //We found end symbol; eat end symbol only if it is not line comment.
                 // For line comment, leave LF symbol there, it might be important to have a separate LF token
                 if (!_isLineComment)
                 {
                     source.PreviewPosition += endSymbol.Length;
                 }
                 return(source.CreateToken(this.OutputTerminal));
             } //if
         }     //foreach endSymbol
         source.PreviewPosition++; //move to the next char and try again
     }         //while
     return(null); //might happen if we found a start char of end symbol, but not the full endSymbol
 }             //method
Example #5
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(OpenTag))
            {
                return(null);
            }
            source.PreviewPosition += OpenTag.Length;
            var    endPos = source.IndexOf(CloseTag, source.PreviewPosition);
            string content;

            if (endPos > 0)
            {
                content = source.GetText(source.PreviewPosition, endPos - source.PreviewPosition);
                source.PreviewPosition = endPos + CloseTag.Length;
            }
            else
            {
                content = source.GetText(source.PreviewPosition, source.Length - source.PreviewPosition);
                source.PreviewPosition = source.Length;
            }
            var token = source.CreateToken(this.OutputTerminal, content);

            return(token);
        }
Example #6
0
 public static int IndexOf(this ISourceStream source, char chr, int index)
 {
     return(source.IndexOf(chr, index, source.Length - index));
 }
Example #7
0
 public static int IndexOf(this ISourceStream source, string str, int index, StringComparison comparison = StringComparison.Ordinal)
 {
     return(source.IndexOf(str, index, source.Length - index, comparison));
 }
Example #8
0
        private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
        {
            bool   escapeEnabled    = !details.IsSet((short)StringOptions.NoEscapes);
            int    start            = source.PreviewPosition;
            string endQuoteSymbol   = details.EndSymbol;
            string endQuoteDoubled  = endQuoteSymbol + endQuoteSymbol; //doubled quote symbol
            bool   lineBreakAllowed = details.IsSet((short)StringOptions.AllowsLineBreak);
            //1. Find the string end
            // first get the position of the next line break; we are interested in it to detect malformed string,
            //  therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
            int nlPos = lineBreakAllowed ? -1 : source.IndexOf('\n', source.PreviewPosition);

            //fix by ashmind for EOF right after opening symbol
            while (true)
            {
                int endPos = source.IndexOf(endQuoteSymbol, source.PreviewPosition);
                //Check for partial token in line-scanning mode
                if (endPos < 0 && details.PartialOk && lineBreakAllowed)
                {
                    ProcessPartialBody(source, details);
                    return(true);
                }
                //Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
                bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
                if (malformed)
                {
                    //Set source position for recovery: move to the next line if linebreak is not allowed.
                    if (nlPos > 0)
                    {
                        endPos = nlPos;
                    }
                    if (endPos > 0)
                    {
                        source.PreviewPosition = endPos + 1;
                    }
                    details.Error = Resources.ErrBadStrLiteral; //    "Mal-formed  string literal - cannot find termination symbol.";
                    return(true);                               //we did find start symbol, so it is definitely string, only malformed
                }//if malformed

                if (source.EOF())
                {
                    return(true);
                }

                //We found EndSymbol - check if it is escaped; if yes, skip it and continue search
                if (escapeEnabled && IsEndQuoteEscaped(source, endPos))
                {
                    source.PreviewPosition = endPos + endQuoteSymbol.Length;
                    continue; //searching for end symbol
                }

                //Check if it is doubled end symbol
                source.PreviewPosition = endPos;
                if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && source.MatchSymbol(endQuoteDoubled))
                {
                    source.PreviewPosition = endPos + endQuoteDoubled.Length;
                    continue;
                }//checking for doubled end symbol

                //Ok, this is normal endSymbol that terminates the string.
                // Advance source position and get out from the loop
                details.Body           = source.GetText(start, endPos - start);
                source.PreviewPosition = endPos + endQuoteSymbol.Length;
                return(true); //if we come here it means we're done - we found string end.
            } //end of loop to find string end;
        }