protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { //Console.WriteLine("Attempting to match Comment String"); var str = new StringBuilder (); int location = -1; //Match to comment start foreach (char character in COMMENT_START) { if (character.ToString().Equals (tokenizer.Current)) { tokenizer.Consume (); } else { return null; } } location = tokenizer.Location; bool ProperEnd = false; while (!tokenizer.End()) { //Console.WriteLine (tokenizer.Current + " " + tokenizer.End()); bool MatchedEnd = true; foreach (char character in COMMENT_END) { if (character.ToString().Equals (tokenizer.Current)) { str.Append (tokenizer.Current); tokenizer.Consume (); } else { MatchedEnd = false; break; } } if (!MatchedEnd) { str.Append (tokenizer.Current); tokenizer.Consume (); } else { str.Remove (str.Length - COMMENT_END.Length, (COMMENT_END.Length)); ProperEnd = true; break; } } if (tokenizer.End ()) return null; if (ProperEnd) { return new YSToken(location, YSToken.TokenType.Comment, str.ToString()); } return null; }
public YSToken IsMatch(YSTokenizer tokenizer) { if (tokenizer.End()) { return(new YSToken(tokenizer.Location, YSToken.TokenType.EOF, null)); } //YSLexer.Debug ("Current Token: " + tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount); var match = IsMatchImpl(tokenizer); //YSLexer.Debug (tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount); if (match == null) { YSLexer.Debug("No match for " + Identifier()); } else { YSLexer.Debug(Identifier() + " matched " + match.Content); } if (match == null) { tokenizer.Rollback(); } else { tokenizer.Commit(); } return(match); }
public YSLexer(string source) { Tokens = new List<YSToken> (); Tokenizer = new YSTokenizer (() => source.ToCharArray ().Select (i => i.ToString ()).ToList ()); Matchers = new List<YSMatcherBase> (); Matchers.Add (new YSQuotedStringMatcher ()); Matchers.Add (new YSCommentMatcher ()); Matchers.AddRange (YSToken.SpecialCharacters); Matchers.AddRange (YSToken.KeywordCharacters); Matchers.Add (new YSNumberMatcher ()); Matchers.Add (new YSWhiteSpaceMatcher()); Matchers.Add (new YSIdentityMatcher ()); Debug(((YSMatcherBase) Matchers[0]).Identifier()); int safety = 0; while (!Tokenizer.End () && ++safety < 1000) { int location = Tokenizer.Location; foreach (YSMatcherBase Matcher in Matchers) { YSToken token = Matcher.IsMatch (Tokenizer); if (token != null) { Debug (token.Type + " @ " + token.Position + " Content: " + token.Content); if(token.Type != YSToken.TokenType.WhiteSpace && token.Type != YSToken.TokenType.Newline) Tokens.Add (token); break; } } if (Tokenizer.Location == location) { Debug ("Unrecognized character " + Tokenizer.Current + " @ " + Tokenizer.Location); break; } } Debug ("Finished lexing"); }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { int location = tokenizer.Location; StringBuilder str = new StringBuilder(); //first character of an identity cannot be a number int intval = 0; if (int.TryParse(tokenizer.Current, out intval)) { return(null); } while (!tokenizer.End() && !String.IsNullOrWhiteSpace(tokenizer.Current) && !YSToken.SpecialCharacters.Any(character => character.Word == tokenizer.Current)) { str.Append(tokenizer.Current); tokenizer.Consume(); } if (str.Length > 0) { return(new YSToken(location, YSToken.TokenType.Identity, str.ToString())); } return(null); }
public YSToken IsMatch(YSTokenizer tokenizer) { if (tokenizer.End()) { return new YSToken(tokenizer.Location, YSToken.TokenType.EOF, null); } //YSLexer.Debug ("Current Token: " + tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount); var match = IsMatchImpl(tokenizer); //YSLexer.Debug (tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount); if (match == null) YSLexer.Debug ("No match for " + Identifier ()); else YSLexer.Debug (Identifier() + " matched " + match.Content); if (match == null) { tokenizer.Rollback(); } else { tokenizer.Commit(); } return match; }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { //Console.WriteLine("Attempting to match Quoted String"); var str = new StringBuilder(); bool completeString = false; int location = -1; //Console.WriteLine (Delims.Contains (tokenizer.Current)); if (Delims.Contains(tokenizer.Current)) { String current_delim = tokenizer.Current; location = tokenizer.Location; tokenizer.Consume(); while (!tokenizer.End() && !tokenizer.Current.Equals(current_delim)) { //Console.WriteLine (tokenizer.Current + " " + tokenizer.End()); str.Append(tokenizer.Current); tokenizer.Consume(); } //Console.WriteLine (str.ToString ()); if (tokenizer.End()) { return(null); } if (tokenizer.Current.Equals(current_delim)) { tokenizer.Consume(); completeString = true; } } if (completeString) { return(new YSToken(location, YSToken.TokenType.TextData, str.ToString())); } return(null); }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { int location = tokenizer.Location; bool dotfound = false; StringBuilder str = new StringBuilder(); //first character of an identity cannot be a number while (!tokenizer.End() && !String.IsNullOrWhiteSpace(tokenizer.Current)) { bool specialcharfound = YSToken.SpecialCharacters.Any(character => character.Word == tokenizer.Current); if (tokenizer.Current == DOT) { if (!dotfound) { dotfound = true; str.Append(tokenizer.Current); tokenizer.Consume(); continue; } else if (dotfound) { break; } } else if (specialcharfound) { break; } int intval = 0; if (int.TryParse(tokenizer.Current, out intval)) { str.Append(tokenizer.Current); tokenizer.Consume(); } else { return(null); } } if (str.Length > 0) { return(new YSToken(location, YSToken.TokenType.NumberData, str.ToString())); } return(null); }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { //Console.WriteLine("Attempting to match Quoted String"); var str = new StringBuilder (); bool completeString = false; int location = -1; //Console.WriteLine (Delims.Contains (tokenizer.Current)); if (Delims.Contains (tokenizer.Current)) { String current_delim = tokenizer.Current; location = tokenizer.Location; tokenizer.Consume (); while (!tokenizer.End() && !tokenizer.Current.Equals (current_delim)) { //Console.WriteLine (tokenizer.Current + " " + tokenizer.End()); str.Append (tokenizer.Current); tokenizer.Consume(); } //Console.WriteLine (str.ToString ()); if (tokenizer.End ()) return null; if(tokenizer.Current.Equals(current_delim)) { tokenizer.Consume(); completeString = true; } } if (completeString) { return new YSToken(location, YSToken.TokenType.TextData, str.ToString()); } return null; }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { bool ws = false; int location = tokenizer.Location; while (!tokenizer.End () && String.IsNullOrWhiteSpace (tokenizer.Current)) { ws = true; tokenizer.Consume (); } if (ws) { return new YSToken (location, YSToken.TokenType.WhiteSpace, null); } return null; }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { bool ws = false; int location = tokenizer.Location; while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current)) { ws = true; tokenizer.Consume(); } if (ws) { return(new YSToken(location, YSToken.TokenType.WhiteSpace, null)); } return(null); }
public YSLexer(string source) { Tokens = new List <YSToken> (); Tokenizer = new YSTokenizer(() => source.ToCharArray().Select(i => i.ToString()).ToList()); Matchers = new List <YSMatcherBase> (); Matchers.Add(new YSQuotedStringMatcher()); Matchers.Add(new YSCommentMatcher()); Matchers.AddRange(YSToken.SpecialCharacters); Matchers.AddRange(YSToken.KeywordCharacters); Matchers.Add(new YSNumberMatcher()); Matchers.Add(new YSWhiteSpaceMatcher()); Matchers.Add(new YSIdentityMatcher()); Debug(((YSMatcherBase)Matchers[0]).Identifier()); int safety = 0; while (!Tokenizer.End() && ++safety < 1000) { int location = Tokenizer.Location; foreach (YSMatcherBase Matcher in Matchers) { YSToken token = Matcher.IsMatch(Tokenizer); if (token != null) { Debug(token.Type + " @ " + token.Position + " Content: " + token.Content); if (token.Type != YSToken.TokenType.WhiteSpace && token.Type != YSToken.TokenType.Newline) { Tokens.Add(token); } break; } } if (Tokenizer.Location == location) { Debug("Unrecognized character " + Tokenizer.Current + " @ " + Tokenizer.Location); break; } } Debug("Finished lexing"); }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { int location = tokenizer.Location; bool dotfound = false; StringBuilder str = new StringBuilder (); //first character of an identity cannot be a number while (!tokenizer.End () && !String.IsNullOrWhiteSpace (tokenizer.Current)) { bool specialcharfound = YSToken.SpecialCharacters.Any (character => character.Word == tokenizer.Current); if (tokenizer.Current == DOT) { if (!dotfound) { dotfound = true; str.Append (tokenizer.Current); tokenizer.Consume(); continue; } else if (dotfound) { break; } } else if (specialcharfound) { break; } int intval = 0; if (int.TryParse (tokenizer.Current, out intval)) { str.Append (tokenizer.Current); tokenizer.Consume (); } else { return null; } } if(str.Length > 0) return new YSToken (location, YSToken.TokenType.NumberData, str.ToString ()); return null; }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { int location = tokenizer.Location; StringBuilder str = new StringBuilder (); //first character of an identity cannot be a number int intval = 0; if(int.TryParse(tokenizer.Current, out intval)) return null; while (!tokenizer.End () && !String.IsNullOrWhiteSpace (tokenizer.Current) && !YSToken.SpecialCharacters.Any (character => character.Word == tokenizer.Current)) { str.Append (tokenizer.Current); tokenizer.Consume (); } if(str.Length > 0) return new YSToken (location, YSToken.TokenType.Identity, str.ToString ()); return null; }
protected override YSToken IsMatchImpl(YSTokenizer tokenizer) { //Console.WriteLine("Attempting to match Comment String"); var str = new StringBuilder(); int location = -1; //Match to comment start foreach (char character in COMMENT_START) { if (character.ToString().Equals(tokenizer.Current)) { tokenizer.Consume(); } else { return(null); } } location = tokenizer.Location; bool ProperEnd = false; while (!tokenizer.End()) { //Console.WriteLine (tokenizer.Current + " " + tokenizer.End()); bool MatchedEnd = true; foreach (char character in COMMENT_END) { if (character.ToString().Equals(tokenizer.Current)) { str.Append(tokenizer.Current); tokenizer.Consume(); } else { MatchedEnd = false; break; } } if (!MatchedEnd) { str.Append(tokenizer.Current); tokenizer.Consume(); } else { str.Remove(str.Length - COMMENT_END.Length, (COMMENT_END.Length)); ProperEnd = true; break; } } if (tokenizer.End()) { return(null); } if (ProperEnd) { return(new YSToken(location, YSToken.TokenType.Comment, str.ToString())); } return(null); }