protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
string compile = "";
int location = tokenizer.Location;
var ctok = tokenizer.Current;
foreach (var character in Word)
{
if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture))
{
tokenizer.Consume();
compile += tokenizer.Current;
}
else
{
return null;
}
}
bool endOfToken, complete;
var next = tokenizer.Current;
endOfToken = String.IsNullOrWhiteSpace(next)
|| YSToken.SpecialCharacters.Any(character => character.Word == next)
|| YSToken.SpecialCharacters.Any(character => character.Word == ctok);
//complete = compile.Equals (Word);
if (endOfToken)
{
return new YSToken(location, Type, Word);
}
return null;
}
public YSLexer(string source)
{
Tokens = new List<YSToken> ();
Tokenizer = new YSTokenizer (() => source.ToCharArray ().Select (i => i.ToString ()).ToList ());
Matchers = new List<YSMatcherBase> ();
Matchers.Add (new YSQuotedStringMatcher ());
Matchers.Add (new YSCommentMatcher ());
Matchers.AddRange (YSToken.SpecialCharacters);
Matchers.AddRange (YSToken.KeywordCharacters);
Matchers.Add (new YSNumberMatcher ());
Matchers.Add (new YSWhiteSpaceMatcher());
Matchers.Add (new YSIdentityMatcher ());
Debug(((YSMatcherBase) Matchers[0]).Identifier());
int safety = 0;
while (!Tokenizer.End () && ++safety < 1000) {
int location = Tokenizer.Location;
foreach (YSMatcherBase Matcher in Matchers) {
YSToken token = Matcher.IsMatch (Tokenizer);
if (token != null) {
Debug (token.Type + " @ " + token.Position + " Content: " + token.Content);
if(token.Type != YSToken.TokenType.WhiteSpace && token.Type != YSToken.TokenType.Newline)
Tokens.Add (token);
break;
}
}
if (Tokenizer.Location == location) {
Debug ("Unrecognized character " + Tokenizer.Current + " @ " + Tokenizer.Location);
break;
}
}
Debug ("Finished lexing");
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
StringBuilder str = new StringBuilder();
//first character of an identity cannot be a number
int intval = 0;
if (int.TryParse(tokenizer.Current, out intval))
{
return(null);
}
while (!tokenizer.End() &&
!String.IsNullOrWhiteSpace(tokenizer.Current) &&
!YSToken.SpecialCharacters.Any(character => character.Word == tokenizer.Current))
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
if (str.Length > 0)
{
return(new YSToken(location, YSToken.TokenType.Identity, str.ToString()));
}
return(null);
}
public YSToken IsMatch(YSTokenizer tokenizer)
{
if (tokenizer.End())
{
return new YSToken(tokenizer.Location, YSToken.TokenType.EOF, null);
}
//YSLexer.Debug ("Current Token: " + tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount);
var match = IsMatchImpl(tokenizer);
//YSLexer.Debug (tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount);
if (match == null)
YSLexer.Debug ("No match for " + Identifier ());
else
YSLexer.Debug (Identifier() + " matched " + match.Content);
if (match == null)
{
tokenizer.Rollback();
}
else
{
tokenizer.Commit();
}
return match;
}
public YSToken IsMatch(YSTokenizer tokenizer)
{
if (tokenizer.End())
{
return(new YSToken(tokenizer.Location, YSToken.TokenType.EOF, null));
}
//YSLexer.Debug ("Current Token: " + tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount);
var match = IsMatchImpl(tokenizer);
//YSLexer.Debug (tokenizer.Current + " Consume Count " + tokenizer.ConsumeCount);
if (match == null)
{
YSLexer.Debug("No match for " + Identifier());
}
else
{
YSLexer.Debug(Identifier() + " matched " + match.Content);
}
if (match == null)
{
tokenizer.Rollback();
}
else
{
tokenizer.Commit();
}
return(match);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
bool dotfound = false;
StringBuilder str = new StringBuilder();
//first character of an identity cannot be a number
while (!tokenizer.End() &&
!String.IsNullOrWhiteSpace(tokenizer.Current))
{
bool specialcharfound = YSToken.SpecialCharacters.Any(character => character.Word == tokenizer.Current);
if (tokenizer.Current == DOT)
{
if (!dotfound)
{
dotfound = true;
str.Append(tokenizer.Current);
tokenizer.Consume();
continue;
}
else if (dotfound)
{
break;
}
}
else if (specialcharfound)
{
break;
}
int intval = 0;
if (int.TryParse(tokenizer.Current, out intval))
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
else
{
return(null);
}
}
if (str.Length > 0)
{
return(new YSToken(location, YSToken.TokenType.NumberData, str.ToString()));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Comment String");
var str = new StringBuilder ();
int location = -1;
//Match to comment start
foreach (char character in COMMENT_START) {
if (character.ToString().Equals (tokenizer.Current)) {
tokenizer.Consume ();
} else {
return null;
}
}
location = tokenizer.Location;
bool ProperEnd = false;
while (!tokenizer.End()) {
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
bool MatchedEnd = true;
foreach (char character in COMMENT_END) {
if (character.ToString().Equals (tokenizer.Current)) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
} else {
MatchedEnd = false;
break;
}
}
if (!MatchedEnd) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
} else {
str.Remove (str.Length - COMMENT_END.Length, (COMMENT_END.Length));
ProperEnd = true;
break;
}
}
if (tokenizer.End ())
return null;
if (ProperEnd) {
return new YSToken(location, YSToken.TokenType.Comment, str.ToString());
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
bool ws = false;
int location = tokenizer.Location;
while (!tokenizer.End () && String.IsNullOrWhiteSpace (tokenizer.Current)) {
ws = true;
tokenizer.Consume ();
}
if (ws) {
return new YSToken (location, YSToken.TokenType.WhiteSpace, null);
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Quoted String");
var str = new StringBuilder();
bool completeString = false;
int location = -1;
//Console.WriteLine (Delims.Contains (tokenizer.Current));
if (Delims.Contains(tokenizer.Current))
{
String current_delim = tokenizer.Current;
location = tokenizer.Location;
tokenizer.Consume();
while (!tokenizer.End() && !tokenizer.Current.Equals(current_delim))
{
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
str.Append(tokenizer.Current);
tokenizer.Consume();
}
//Console.WriteLine (str.ToString ());
if (tokenizer.End())
{
return(null);
}
if (tokenizer.Current.Equals(current_delim))
{
tokenizer.Consume();
completeString = true;
}
}
if (completeString)
{
return(new YSToken(location, YSToken.TokenType.TextData, str.ToString()));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
bool ws = false;
int location = tokenizer.Location;
while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current))
{
ws = true;
tokenizer.Consume();
}
if (ws)
{
return(new YSToken(location, YSToken.TokenType.WhiteSpace, null));
}
return(null);
}
public YSLexer(string source)
{
Tokens = new List <YSToken> ();
Tokenizer = new YSTokenizer(() => source.ToCharArray().Select(i => i.ToString()).ToList());
Matchers = new List <YSMatcherBase> ();
Matchers.Add(new YSQuotedStringMatcher());
Matchers.Add(new YSCommentMatcher());
Matchers.AddRange(YSToken.SpecialCharacters);
Matchers.AddRange(YSToken.KeywordCharacters);
Matchers.Add(new YSNumberMatcher());
Matchers.Add(new YSWhiteSpaceMatcher());
Matchers.Add(new YSIdentityMatcher());
Debug(((YSMatcherBase)Matchers[0]).Identifier());
int safety = 0;
while (!Tokenizer.End() && ++safety < 1000)
{
int location = Tokenizer.Location;
foreach (YSMatcherBase Matcher in Matchers)
{
YSToken token = Matcher.IsMatch(Tokenizer);
if (token != null)
{
Debug(token.Type + " @ " + token.Position + " Content: " + token.Content);
if (token.Type != YSToken.TokenType.WhiteSpace && token.Type != YSToken.TokenType.Newline)
{
Tokens.Add(token);
}
break;
}
}
if (Tokenizer.Location == location)
{
Debug("Unrecognized character " + Tokenizer.Current + " @ " + Tokenizer.Location);
break;
}
}
Debug("Finished lexing");
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
bool dotfound = false;
StringBuilder str = new StringBuilder ();
//first character of an identity cannot be a number
while (!tokenizer.End ()
&& !String.IsNullOrWhiteSpace (tokenizer.Current)) {
bool specialcharfound = YSToken.SpecialCharacters.Any (character => character.Word == tokenizer.Current);
if (tokenizer.Current == DOT) {
if (!dotfound) {
dotfound = true;
str.Append (tokenizer.Current);
tokenizer.Consume();
continue;
} else if (dotfound) {
break;
}
} else if (specialcharfound) {
break;
}
int intval = 0;
if (int.TryParse (tokenizer.Current, out intval)) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
} else {
return null;
}
}
if(str.Length > 0)
return new YSToken (location, YSToken.TokenType.NumberData, str.ToString ());
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
StringBuilder str = new StringBuilder ();
//first character of an identity cannot be a number
int intval = 0;
if(int.TryParse(tokenizer.Current, out intval))
return null;
while (!tokenizer.End ()
&& !String.IsNullOrWhiteSpace (tokenizer.Current)
&& !YSToken.SpecialCharacters.Any (character => character.Word == tokenizer.Current)) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
}
if(str.Length > 0)
return new YSToken (location, YSToken.TokenType.Identity, str.ToString ());
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Quoted String");
var str = new StringBuilder ();
bool completeString = false;
int location = -1;
//Console.WriteLine (Delims.Contains (tokenizer.Current));
if (Delims.Contains (tokenizer.Current)) {
String current_delim = tokenizer.Current;
location = tokenizer.Location;
tokenizer.Consume ();
while (!tokenizer.End() && !tokenizer.Current.Equals (current_delim)) {
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
str.Append (tokenizer.Current);
tokenizer.Consume();
}
//Console.WriteLine (str.ToString ());
if (tokenizer.End ())
return null;
if(tokenizer.Current.Equals(current_delim))
{
tokenizer.Consume();
completeString = true;
}
}
if (completeString) {
return new YSToken(location, YSToken.TokenType.TextData, str.ToString());
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
string compile = "";
int location = tokenizer.Location;
var ctok = tokenizer.Current;
foreach (var character in Word)
{
if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture))
{
tokenizer.Consume();
compile += tokenizer.Current;
}
else
{
return(null);
}
}
bool endOfToken, complete;
var next = tokenizer.Current;
endOfToken = String.IsNullOrWhiteSpace(next) ||
YSToken.SpecialCharacters.Any(character => character.Word == next) ||
YSToken.SpecialCharacters.Any(character => character.Word == ctok);
//complete = compile.Equals (Word);
if (endOfToken)
{
return(new YSToken(location, Type, Word));
}
return(null);
}
protected abstract YSToken IsMatchImpl(YSTokenizer tokenizer);
protected abstract YSToken IsMatchImpl(YSTokenizer tokenizer);
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Comment String");
var str = new StringBuilder();
int location = -1;
//Match to comment start
foreach (char character in COMMENT_START)
{
if (character.ToString().Equals(tokenizer.Current))
{
tokenizer.Consume();
}
else
{
return(null);
}
}
location = tokenizer.Location;
bool ProperEnd = false;
while (!tokenizer.End())
{
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
bool MatchedEnd = true;
foreach (char character in COMMENT_END)
{
if (character.ToString().Equals(tokenizer.Current))
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
else
{
MatchedEnd = false;
break;
}
}
if (!MatchedEnd)
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
else
{
str.Remove(str.Length - COMMENT_END.Length, (COMMENT_END.Length));
ProperEnd = true;
break;
}
}
if (tokenizer.End())
{
return(null);
}
if (ProperEnd)
{
return(new YSToken(location, YSToken.TokenType.Comment, str.ToString()));
}
return(null);
}
