Example #1
0
        public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(_startSymbol, false))
            {
                return(null);
            }
            source.Position += _startSymbol.Length;

            while (!source.EOF())
            {
                int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

                if (firstCharPos < 0)
                {
                    source.Position = source.Text.Length;

                    return(_isLineComment
                   ? TokenAst.Create(this, context, source.TokenStart, source.GetLexeme())
                   : LRParser.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
                }

                source.Position = firstCharPos;
                if (source.MatchSymbol(_endSymbol, false))
                {
                    source.Position += _endSymbol.Length;
                    return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
                }

                source.Position++;
            }

            throw new NotSupportedException();
        }
Example #2
0
    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(_startSymbol, false)) return null;
      source.Position += _startSymbol.Length;

      while (!source.EOF())
      {
        int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

        if (firstCharPos < 0)
        {
          source.Position = source.Text.Length;

          if (_isLineComment)
            return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
          else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
        }
      
        source.Position = firstCharPos;
        if (source.MatchSymbol(_endSymbol, false))
        {
          source.Position += _endSymbol.Length;
          return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
        }
        
        source.Position++; 
      }

      throw new NotSupportedException();
    }
Example #3
0
        private bool BeginMatch(ParsingContext context, ISourceStream source)
        {
            //Check starting symbol
            if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
            {
                return(false);
            }

            source.PreviewPosition += 1;
            return(true);
        }
Example #4
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;

            //Check starting symbol
            if (!source.MatchSymbol(StartSymbol, ignoreCase))
            {
                return(null);
            }
            //Find end symbol
            source.Position += StartSymbol.Length;

            while (!source.EOF())
            {
                int firstCharPos;
                if (EndSymbols.Count == 1)
                {
                    firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
                }
                else
                {
                    firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
                }
                if (firstCharPos < 0)
                {
                    source.Position = source.Text.Length;
                    if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
                    {
                        return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }
                    else
                    {
                        return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
                    }
                }
                //We found a character that might start an end symbol; let's see if it is true.
                source.Position = firstCharPos;
                foreach (string endSymbol in EndSymbols)
                {
                    if (source.MatchSymbol(endSymbol, ignoreCase))
                    {
                        //We found end symbol
                        source.Position += endSymbol.Length;
                        return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }//if
                }
                source.Position++; //move to the next char and try again
            } //while
            return(null); //never happens
        }     //method
 private bool BeginMatch(ParsingContext context, ISourceStream source)
 {
     //Check starting symbol
     if (!source.MatchSymbol(StartSymbol)) return false;
     source.PreviewPosition += StartSymbol.Length;
     return true;
 }
Example #6
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            var text = source.Text;

            foreach (var entry in Constants)
            {
                source.PreviewPosition = source.Position;
                var constant = entry.Key;
                if (source.PreviewPosition + constant.Length > text.Length)
                {
                    continue;
                }

                if (source.MatchSymbol(constant))
                {
                    source.PreviewPosition += constant.Length;
                    if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
                    {
                        continue; //make sure it is delimiter
                    }

                    return(source.CreateToken(OutputTerminal, entry.Value));
                }
            }
            return(null);
        }
        }     //method

        protected virtual void ReadSuffix(ISourceStream source, ScanDetails details)
        {
            if (_suffixesFirsts.IndexOf(source.CurrentChar) < 0)
            {
                return;
            }
            bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);

            foreach (string sfx in Suffixes)
            {
                if (!source.MatchSymbol(sfx, ignoreCase))
                {
                    continue;
                }
                //We found suffix
                details.Suffix   = sfx;
                source.Position += sfx.Length;
                //Set TypeCode from suffix
                TypeCode[] codes;
                if (!string.IsNullOrEmpty(details.Suffix) && SuffixTypeCodes.TryGetValue(details.Suffix, out codes))
                {
                    details.TypeCodes = codes;
                }
                return;
            } //foreach
        }     //method
        protected virtual void ReadPrefix(ISourceStream source, ScanDetails details)
        {
            if (_prefixesFirsts.IndexOf(source.CurrentChar) < 0)
            {
                return;
            }
            bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);

            foreach (string pfx in Prefixes)
            {
                if (!source.MatchSymbol(pfx, ignoreCase))
                {
                    continue;
                }
                //We found prefix
                details.Prefix   = pfx;
                source.Position += pfx.Length;
                //Set numeric base flag from prefix
                ScanFlags pfxFlags;
                if (!string.IsNullOrEmpty(details.Prefix) && PrefixFlags.TryGetValue(details.Prefix, out pfxFlags))
                {
                    details.Flags |= pfxFlags;
                }
                return;
            } //foreach
        }     //method
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            if (!ReadStartSymbol(source, details))
            {
                return(false);
            }

            bool   escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
            bool   ignoreCase    = IsSet(TermOptions.SpecialIgnoreCase);
            int    start         = source.Position;
            string startS        = details.ControlSymbol;
            string startS2       = startS + startS; //doubled start symbol
            //1. Find the string end
            // first get the position of the next line break; we are interested in it to detect malformed string,
            //  therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
            int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);

            while (!source.EOF())
            {
                int endPos = source.Text.IndexOf(startS, source.Position);
                //Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
                bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
                if (malformed)
                {
                    //Set source position for recovery: move to the next line if linebreak is not allowed.
                    if (nlPos > 0)
                    {
                        endPos = nlPos;
                    }
                    if (endPos > 0)
                    {
                        source.Position = endPos + 1;
                    }
                    details.Error = "Mal-formed  string literal - cannot find termination symbol.";
                    return(true);
                }

                //We found EndSymbol - check if it is escaped; if yes, skip it and continue search
                if (escapeEnabled && source.Text[endPos - 1] == EscapeChar)
                {
                    source.Position = endPos + startS.Length;
                    continue; //searching for end symbol
                }

                //Check if it is doubled end symbol
                source.Position = endPos;
                if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase))
                {
                    source.Position = endPos + startS.Length * 2;
                    continue;
                }//checking for doubled end symbol

                //Ok, this is normal endSymbol that terminates the string.
                // Advance source position and get out from the loop
                details.Body    = source.Text.Substring(start, endPos - start);
                source.Position = endPos + startS.Length;
                return(true); //if we come here it means we're done - we found string end.
            } //end of loop to find string end;
            return(false);
        }
Example #10
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;
              //Check starting symbol
              if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
              //Find end symbol
              source.Position += StartSymbol.Length;

              while(!source.EOF()) {
            int firstCharPos;
            if (EndSymbols.Count == 1)
              firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
            else
              firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
            if (firstCharPos < 0) {
              source.Position = source.Text.Length;
              if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
            }
            //We found a character that might start an end symbol; let's see if it is true.
            source.Position = firstCharPos;
            foreach (string endSymbol in EndSymbols)
              if (source.MatchSymbol(endSymbol, ignoreCase)) {
            //We found end symbol
            source.Position += endSymbol.Length;
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              }//if
            source.Position++; //move to the next char and try again
              }//while
              return null; //never happens
        }
Example #11
0
        private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte commentLevel)
        {
            if (!source.MatchSymbol(StartSymbol, !base.Grammar.CaseSensitive))
            {
                return(false);
            }
            int num    = source.PreviewPosition + StartSymbol.Length;
            int length = source.Text.Length;

            if (length > num + 1 && source.Text[num] == '[')
            {
                byte b    = 1;
                int  num2 = num + 255;
                num++;
                while (num < num2 && length > num)
                {
                    if (source.Text[num] == '=')
                    {
                        b += 1;
                        num++;
                    }
                    else
                    {
                        if (source.Text[num] == '[')
                        {
                            commentLevel = b;
                            break;
                        }
                        break;
                    }
                }
            }
            source.PreviewPosition += StartSymbol.Length + (int)commentLevel;
            return(true);
        }
Example #12
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;

            if (isHeadingOrList)
            {
                bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
                if (!isAfterNewLine)
                {
                    return(null);
                }
            }
            if (!source.MatchSymbol(OpenTag, true))
            {
                return(null);
            }
            source.PreviewPosition += OpenTag.Length;
            //For headings and lists require space after
            if (TermType == WikiTermType.Heading || TermType == WikiTermType.List)
            {
                const string whitespaces = " \t\r\n\v";
                if (!whitespaces.Contains(source.PreviewChar))
                {
                    return(null);
                }
            }
            var token = source.CreateToken(this.OutputTerminal);

            return(token);
        }
Example #13
0
 protected override string ReadBody(ParsingContext context, ISourceStream source) {
   if (!source.MatchSymbol(StartSymbol)) return null; //this will result in null returned from TryMatch, no token
   var start = source.Location.Position + StartSymbol.Length;
   var end = source.Text.IndexOf(EndSymbol, start);
   if (end < 0) return null;
   var body = source.Text.Substring(start, end - start);
   source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
   return body; 
 }
Example #14
0
 private bool TryMatchPrefixes(ParsingContext context, ISourceStream source) {
   if (Firsts.Count == 0) 
     return true;  
   foreach (var first in Firsts)
     if (source.MatchSymbol(first)) {
       source.PreviewPosition += first.Length;
       return true;
     }
   return false; 
 }
Example #15
0
    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(Symbol, false))
        return null;
    
      source.Position += Symbol.Length;
      TokenAst tokenAst = TokenAst.Create(this, context, source.TokenStart, Symbol);

      return tokenAst;
    }
 private bool BeginMatch(CompilerContext context, ISourceStream source)
 {
     //Check starting symbol
     if (!source.MatchSymbol(StartSymbol, !OwnerGrammar.CaseSensitive))
     {
         return(false);
     }
     source.Position += StartSymbol.Length;
     return(true);
 }
Example #17
0
 private bool CheckEscape(ISourceStream source, ref string tokenText) {
   foreach (var dictEntry in Escapes) {
     if (source.MatchSymbol(dictEntry.Key, !Grammar.CaseSensitive)) {
       source.PreviewPosition += dictEntry.Key.Length;
       tokenText += dictEntry.Value;
       return true; 
     }
   }//foreach
   return false; 
 }
Example #18
0
 private bool BeginMatch(ParsingContext context, ISourceStream source)
 {
     //Check starting symbol
     if (!source.MatchSymbol(StartSymbol))
     {
         return(false);
     }
     source.PreviewPosition += StartSymbol.Length;
     return(true);
 }
Example #19
0
 private bool BeginMatch(ISourceStream source, int startFrom, char lookAhead) {
   foreach (var startSymbol in StartSymbols.Skip(startFrom)) {
     if (startSymbol[0] != lookAhead)
       continue;
     if (source.MatchSymbol(startSymbol, !Grammar.CaseSensitive)) {
       source.PreviewPosition += startSymbol.Length;
       return true;
     }
   }
   return false;
 }
Example #20
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(_symbol, !OwnerGrammar.CaseSensitive))
            {
                return(null);
            }
            source.Position += _symbol.Length;
            Token tkn = new Token(this, source.TokenStart, Symbol, null);

            return(tkn);
        }
Example #21
0
        public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(Symbol, false))
            {
                return(null);
            }

            source.Position += Symbol.Length;
            TokenAst tokenAst = TokenAst.Create(this, context, source.TokenStart, Symbol);

            return(tokenAst);
        }
Example #22
0
 Token Match_compile_end_tag(Terminal terminal, ParsingContext context, ISourceStream source)
 {
     if (source.MatchSymbol("__End"))
     {
         var p       = source.Location.Position;
         var lineEnd = source.Text.IndexOf("\n", p);
         var text    = source.Text.Substring(p, lineEnd - p + 1);
         return(new Token(_comment, source.Location, text, text));
     }
     //otherwise return null
     return(null);
 }
Example #23
0
 private bool CheckEscape(ISourceStream source, StringBuilder tokenText)
 {
     foreach (var dictEntry in Escapes)
     {
         if (source.MatchSymbol(dictEntry.Key))
         {
             source.PreviewPosition += dictEntry.Key.Length;
             tokenText.Append(dictEntry.Value);
             return(true);
         }
     }//foreach
     return(false);
 }
Example #24
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source)
        {
            //Find end symbol
            while (!source.EOF())
            {
                var firstCharPos = source.Text.IndexOf('{', source.PreviewPosition);
                if (firstCharPos < 0)
                {
                    source.PreviewPosition = source.Text.Length;
                    return(source.CreateToken(this.OutputTerminal));
                }
                //We found a character that might start an end symbol; let's see if it is true.
                source.PreviewPosition = firstCharPos;
                if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
                {
                    return(source.CreateToken(this.OutputTerminal));
                }
                source.PreviewPosition++; //move to the next char and try again
            }

            return(source.CreateToken(this.OutputTerminal));
        }
Example #25
0
 private bool CheckEscape(ISourceStream source, ref string tokenText)
 {
     foreach (var dictEntry in Escapes)
     {
         if (source.MatchSymbol(dictEntry.Key, !Grammar.CaseSensitive))
         {
             source.PreviewPosition += dictEntry.Key.Length;
             tokenText += dictEntry.Value;
             return(true);
         }
     }//foreach
     return(false);
 }
Example #26
0
        private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
        {
            if (!source.MatchSymbol(StartSymbol, !base.Grammar.CaseSensitive))
            {
                return(false);
            }
            Match match = Regex.Match(source.Text.Substring(source.PreviewPosition + StartSymbol.Length), "^(=*)\\[");

            if (match.Value != string.Empty)
            {
                level = (byte)match.Groups[1].Value.Length;
                return(true);
            }
            return(false);
        }
 private void MoveSourcePositionAfterTerminator(ISourceStream source)
 {
     while (!source.EOF())
     {
         while (source.PreviewChar != Terminator[0])
         {
             source.PreviewPosition++;
         }
         if (source.MatchSymbol(Terminator, !Grammar.CaseSensitive))
         {
             source.PreviewPosition += Terminator.Length;
             return;
         } //if
     }     //while
 }         //method
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   if (!source.MatchSymbol(OpenTag, true)) return null;
   source.PreviewPosition += OpenTag.Length;
   var endPos = source.Text.IndexOf(CloseTag, source.PreviewPosition);
   string content; 
   if(endPos > 0) {
     content = source.Text.Substring(source.PreviewPosition, endPos - source.PreviewPosition); 
     source.PreviewPosition = endPos + CloseTag.Length;
   } else {
     content = source.Text.Substring(source.PreviewPosition, source.Text.Length - source.PreviewPosition); 
     source.PreviewPosition = source.Text.Length;
   }
   var token = source.CreateToken(this.OutputTerminal, content); 
   return token;      
 }
        private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
        {
            if (!source.MatchSymbol(StartSymbol)) 
                return false;

            string text = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
            var match = Regex.Match(text, @"^(=*)\[");
            if(match.Value != string.Empty)
            {
                level = (byte)match.Groups[1].Value.Length;
                return true;
            }
           
            return false; 
        }
 private bool BeginMatch(ISourceStream source, int startFrom, char lookAhead)
 {
     foreach (var startSymbol in StartSymbols.Skip(startFrom))
     {
         if (startSymbol[0] != lookAhead)
         {
             continue;
         }
         if (source.MatchSymbol(startSymbol))
         {
             source.PreviewPosition += startSymbol.Length;
             return(true);
         }
     }
     return(false);
 }
Example #31
0
 private bool TryMatchPrefixes(ParsingContext context, ISourceStream source)
 {
     if (Firsts.Count == 0)
     {
         return(true);
     }
     foreach (var first in Firsts)
     {
         if (source.MatchSymbol(first))
         {
             source.PreviewPosition += first.Length;
             return(true);
         }
     }
     return(false);
 }
Example #32
0
 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
   if(isHeadingOrList) {
       bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
       if(!isAfterNewLine)  return null;
   }
   if(!source.MatchSymbol(OpenTag)) return null;
   source.PreviewPosition += OpenTag.Length;
   //For headings and lists require space after
   if(TermType == WikiTermType.Heading || TermType == WikiTermType.List) {
     const string whitespaces = " \t\r\n\v";
     if (!whitespaces.Contains(source.PreviewChar)) return null; 
   }
   var token = source.CreateToken(this.OutputTerminal);
   return token; 
 }
Example #33
0
        private void MoveSourcePositionAfterTerminator(ISourceStream source)
        {
            while (!source.EOF)
            {
                while (source.PreviewChar != Terminator[0])
                {
                    source.PreviewPosition++;
                }

                if (source.MatchSymbol(Terminator))
                {
                    source.PreviewPosition += Terminator.Length;
                    return;
                }
            }
        }
        private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
        {
            //Check starting symbol
            if (!source.MatchSymbol(StartSymbol, !Grammar.CaseSensitive))
                return false;

            //Found starting --, now determine whether this is a long comment.
            string text = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
            var match = Regex.Match(text, @"^(=*)\[");
            if(match.Value != string.Empty)
            {
                level = (byte)match.Groups[1].Value.Length;
                return true;
            }

            return false;
        }
Example #35
0
        protected override string ReadBody(ParsingContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(StartSymbol))
            {
                return(null);                            //this will result in null returned from TryMatch, no token
            }
            var start = source.Location.Position + StartSymbol.Length;
            var end   = source.IndexOf(EndSymbol, start);

            if (end < 0)
            {
                return(null);
            }
            var body = source.GetText(start, end - start);

            source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
            return(body);
        }
        private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
        {
            if (!source.MatchSymbol(StartSymbol))
            {
                return(false);
            }

            string text  = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
            var    match = Regex.Match(text, @"^(=*)\[");

            if (match.Value != string.Empty)
            {
                level = (byte)match.Groups[1].Value.Length;
                return(true);
            }

            return(false);
        }
        private bool BeginMatch(ParsingContext context, ISourceStream source, out int numberOfEquals)
        {
            numberOfEquals = 0;

            //Check starting symbol
            if (!source.MatchSymbol("--["))
            {
                return(false);
            }

            //Do our regex magic
            var match = counterRegex.Match(source.Text, source.PreviewPosition);

            numberOfEquals = match.Groups[1].Length;

            source.PreviewPosition += match.Length;
            return(true);
        }
Example #38
0
 private bool CheckTerminators(ISourceStream source, ref string tokenText)
 {
     foreach (var term in Terminators)
     {
         if (source.MatchSymbol(term, !Grammar.CaseSensitive))
         {
             if (IsSet(FreeTextOptions.IncludeTerminator))
             {
                 tokenText += term;
             }
             if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
             {
                 source.PreviewPosition += term.Length;
             }
             return(true);
         }
     }
     return(false);
 }
Example #39
0
 private bool CheckTerminators(ISourceStream source, StringBuilder tokenText)
 {
     foreach (var term in Terminators)
     {
         if (source.MatchSymbol(term))
         {
             if (IsSet(FreeTextOptions.IncludeTerminator))
             {
                 tokenText.Append(term);
             }
             if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
             {
                 source.PreviewPosition += term.Length;
             }
             return(true);
         }
     }
     return(false);
 }
Example #40
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            string text = source.Text;

            foreach (var entry in Constants)
            {
                var constant = entry.Key;
                if (source.PreviewPosition + constant.Length > text.Length)
                {
                    continue;
                }
                if (source.MatchSymbol(constant, !Grammar.CaseSensitive))
                {
                    source.PreviewPosition += constant.Length;
                    return(source.CreateToken(this.OutputTerminal, entry.Value));
                }
            }
            return(null);
        }
Example #41
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source)
        {
            //Find end symbol
            while (!source.EOF)
            {
                int firstCharPos;
                if (EndSymbols.Count == 1)
                {
                    firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition, StringComparison.InvariantCulture);
                }
                else
                {
                    firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
                }

                if (firstCharPos < 0)
                {
                    source.PreviewPosition = source.Text.Length;
                    return(null); //indicating error
                }

                //We found a character that might start an end symbol; let's see if it is true.
                source.PreviewPosition = firstCharPos;
                foreach (var endSymbol in EndSymbols)
                {
                    if (!source.MatchSymbol(endSymbol))
                    {
                        continue;
                    }
                    //We found end symbol; eat end symbol only if it is not line comment.
                    // For line comment, leave LF symbol there, it might be important to have a separate LF token
                    if (!_isLineComment)
                    {
                        source.PreviewPosition += endSymbol.Length;
                    }

                    return(source.CreateToken(OutputTerminal));
                }
                source.PreviewPosition++; //move to the next char and try again
            }
            return(null);                 //might happen if we found a start char of end symbol, but not the full endSymbol
        }
Example #42
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            string text = source.Text;

            foreach (string lexeme in Table.Keys)
            {
                if (source.Position + lexeme.Length > text.Length)
                {
                    continue;
                }
                if (!source.MatchSymbol(lexeme, !OwnerGrammar.CaseSensitive))
                {
                    continue;
                }
                Token tkn = new Token(this, source.TokenStart, lexeme, Table[lexeme]);
                source.Position += lexeme.Length;
                return(tkn);
            }
            return(null);
        }
Example #43
0
 private bool ReadStartSymbol(ISourceStream source, ScanDetails details)
 {
     if (_startEndFirsts.IndexOf(source.CurrentChar) < 0)
     return false;
       bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
       foreach (string startEnd in _startEndSymbols) {
     if (!source.MatchSymbol(startEnd, ignoreCase))
       continue;
     //We found start symbol
     details.ControlSymbol = startEnd;
     details.Flags |= StartEndSymbolTable[startEnd];
     source.Position += startEnd.Length;
     return true;
       }//foreach
       return false;
 }
Example #44
0
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            if (!ReadStartSymbol(source, details)) return false;

              bool escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
              bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
              int start = source.Position;
              string startS = details.ControlSymbol;
              string startS2 = startS + startS; //doubled start symbol
              //1. Find the string end
              // first get the position of the next line break; we are interested in it to detect malformed string,
              //  therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
              int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);
              while (!source.EOF()) {
            int endPos = source.Text.IndexOf(startS, source.Position);
            //Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
            bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
            if (malformed) {
              //Set source position for recovery: move to the next line if linebreak is not allowed.
              if (nlPos > 0) endPos = nlPos;
              if (endPos > 0) source.Position = endPos + 1;
              details.Error = "Mal-formed  string literal - cannot find termination symbol.";
              return true;
            }

            //We found EndSymbol - check if it is escaped; if yes, skip it and continue search
            if (escapeEnabled && source.Text[endPos - 1] == EscapeChar) {
              source.Position = endPos + startS.Length;
              continue; //searching for end symbol
            }

            //Check if it is doubled end symbol
            source.Position = endPos;
            if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase)) {
              source.Position = endPos + startS.Length * 2;
              continue;
            }//checking for doubled end symbol

            //Ok, this is normal endSymbol that terminates the string.
            // Advance source position and get out from the loop
            details.Body = source.Text.Substring(start, endPos - start);
            source.Position = endPos + startS.Length;
            return true; //if we come here it means we're done - we found string end.
              }  //end of loop to find string end;
              return false;
        }
Example #45
0
 private bool BeginMatch(CompilerContext context, ISourceStream source)
 {
     //Check starting symbol
       if (!source.MatchSymbol(StartSymbol, !Grammar.CaseSensitive)) return false;
       source.Position += StartSymbol.Length;
       return true;
 }
Example #46
0
 private bool CheckEscape(ISourceStream source, StringBuilder tokenText) {
   foreach (var dictEntry in Escapes) {
     if (source.MatchSymbol(dictEntry.Key)) {
       source.PreviewPosition += dictEntry.Key.Length;
       tokenText.Append(dictEntry.Value);
       return true; 
     }
   }//foreach
   return false; 
 }
Example #47
0
 private bool CheckTerminators(ISourceStream source, StringBuilder tokenText) {
   foreach(var term in Terminators)
     if(source.MatchSymbol(term)) {
       if (IsSet(FreeTextOptions.IncludeTerminator))
         tokenText.Append(term); 
       if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
         source.PreviewPosition += term.Length;
       return true;
     }
   return false; 
 }
Example #48
0
 private void MoveSourcePositionAfterTerminator(ISourceStream source) {
   while(!source.EOF()) {
     while(source.PreviewChar != Terminator[0])
       source.PreviewPosition++;
     if(source.MatchSymbol(Terminator, !Grammar.CaseSensitive)) {
       source.PreviewPosition += Terminator.Length;
       return;
     }//if
   }//while
 }//method
Example #49
0
 Token Match_compile_end_tag(Terminal terminal, ParsingContext context, ISourceStream source) {
   if (source.MatchSymbol("__End")) {
     var p = source.Location.Position; 
     var lineEnd = source.Text.IndexOf("\n", p); 
     var text = source.Text.Substring(p, lineEnd - p + 1);
     return new Token(_comment, source.Location, text, text);
   }
   //otherwise return null
   return null; 
 }
Example #50
0
 private Token CompleteMatch(ParsingContext context, ISourceStream source) {
   //Find end symbol
   while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
     else 
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
     if (firstCharPos < 0) {
       source.PreviewPosition = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.PreviewPosition = firstCharPos;
     foreach (string endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol)) {
         //We found end symbol; eat end symbol only if it is not line comment.
         // For line comment, leave LF symbol there, it might be important to have a separate LF token
         if (!_isLineComment)
           source.PreviewPosition += endSymbol.Length;
         return source.CreateToken(this.OutputTerminal); 
       }//if
     }//foreach endSymbol
     source.PreviewPosition++; //move to the next char and try again    
   }//while
   return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }//method
Example #51
0
 private bool CheckTerminators(ISourceStream source, ref string tokenText) {
   foreach(var term in Terminators)
     if(source.MatchSymbol(term, !Grammar.CaseSensitive)) {
       if (IsSet(FreeTextOptions.IncludeTerminator))
         tokenText += term; 
       if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
         source.PreviewPosition += term.Length;
       return true;
     }
   return false; 
 }