protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
bool dotfound = false;
StringBuilder str = new StringBuilder();
//first character of an identity cannot be a number
while (!tokenizer.End() &&
!String.IsNullOrWhiteSpace(tokenizer.Current))
{
bool specialcharfound = YSToken.SpecialCharacters.Any(character => character.Word == tokenizer.Current);
if (tokenizer.Current == DOT)
{
if (!dotfound)
{
dotfound = true;
str.Append(tokenizer.Current);
tokenizer.Consume();
continue;
}
else if (dotfound)
{
break;
}
}
else if (specialcharfound)
{
break;
}
int intval = 0;
if (int.TryParse(tokenizer.Current, out intval))
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
else
{
return(null);
}
}
if (str.Length > 0)
{
return(new YSToken(location, YSToken.TokenType.NumberData, str.ToString()));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Comment String");
var str = new StringBuilder ();
int location = -1;
//Match to comment start
foreach (char character in COMMENT_START) {
if (character.ToString().Equals (tokenizer.Current)) {
tokenizer.Consume ();
} else {
return null;
}
}
location = tokenizer.Location;
bool ProperEnd = false;
while (!tokenizer.End()) {
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
bool MatchedEnd = true;
foreach (char character in COMMENT_END) {
if (character.ToString().Equals (tokenizer.Current)) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
} else {
MatchedEnd = false;
break;
}
}
if (!MatchedEnd) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
} else {
str.Remove (str.Length - COMMENT_END.Length, (COMMENT_END.Length));
ProperEnd = true;
break;
}
}
if (tokenizer.End ())
return null;
if (ProperEnd) {
return new YSToken(location, YSToken.TokenType.Comment, str.ToString());
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
string compile = "";
int location = tokenizer.Location;
var ctok = tokenizer.Current;
foreach (var character in Word)
{
if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture))
{
tokenizer.Consume();
compile += tokenizer.Current;
}
else
{
return null;
}
}
bool endOfToken, complete;
var next = tokenizer.Current;
endOfToken = String.IsNullOrWhiteSpace(next)
|| YSToken.SpecialCharacters.Any(character => character.Word == next)
|| YSToken.SpecialCharacters.Any(character => character.Word == ctok);
//complete = compile.Equals (Word);
if (endOfToken)
{
return new YSToken(location, Type, Word);
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
StringBuilder str = new StringBuilder();
//first character of an identity cannot be a number
int intval = 0;
if (int.TryParse(tokenizer.Current, out intval))
{
return(null);
}
while (!tokenizer.End() &&
!String.IsNullOrWhiteSpace(tokenizer.Current) &&
!YSToken.SpecialCharacters.Any(character => character.Word == tokenizer.Current))
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
if (str.Length > 0)
{
return(new YSToken(location, YSToken.TokenType.Identity, str.ToString()));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Quoted String");
var str = new StringBuilder();
bool completeString = false;
int location = -1;
//Console.WriteLine (Delims.Contains (tokenizer.Current));
if (Delims.Contains(tokenizer.Current))
{
String current_delim = tokenizer.Current;
location = tokenizer.Location;
tokenizer.Consume();
while (!tokenizer.End() && !tokenizer.Current.Equals(current_delim))
{
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
str.Append(tokenizer.Current);
tokenizer.Consume();
}
//Console.WriteLine (str.ToString ());
if (tokenizer.End())
{
return(null);
}
if (tokenizer.Current.Equals(current_delim))
{
tokenizer.Consume();
completeString = true;
}
}
if (completeString)
{
return(new YSToken(location, YSToken.TokenType.TextData, str.ToString()));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
bool dotfound = false;
StringBuilder str = new StringBuilder ();
//first character of an identity cannot be a number
while (!tokenizer.End ()
&& !String.IsNullOrWhiteSpace (tokenizer.Current)) {
bool specialcharfound = YSToken.SpecialCharacters.Any (character => character.Word == tokenizer.Current);
if (tokenizer.Current == DOT) {
if (!dotfound) {
dotfound = true;
str.Append (tokenizer.Current);
tokenizer.Consume();
continue;
} else if (dotfound) {
break;
}
} else if (specialcharfound) {
break;
}
int intval = 0;
if (int.TryParse (tokenizer.Current, out intval)) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
} else {
return null;
}
}
if(str.Length > 0)
return new YSToken (location, YSToken.TokenType.NumberData, str.ToString ());
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Quoted String");
var str = new StringBuilder ();
bool completeString = false;
int location = -1;
//Console.WriteLine (Delims.Contains (tokenizer.Current));
if (Delims.Contains (tokenizer.Current)) {
String current_delim = tokenizer.Current;
location = tokenizer.Location;
tokenizer.Consume ();
while (!tokenizer.End() && !tokenizer.Current.Equals (current_delim)) {
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
str.Append (tokenizer.Current);
tokenizer.Consume();
}
//Console.WriteLine (str.ToString ());
if (tokenizer.End ())
return null;
if(tokenizer.Current.Equals(current_delim))
{
tokenizer.Consume();
completeString = true;
}
}
if (completeString) {
return new YSToken(location, YSToken.TokenType.TextData, str.ToString());
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
bool ws = false;
int location = tokenizer.Location;
while (!tokenizer.End () && String.IsNullOrWhiteSpace (tokenizer.Current)) {
ws = true;
tokenizer.Consume ();
}
if (ws) {
return new YSToken (location, YSToken.TokenType.WhiteSpace, null);
}
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
bool ws = false;
int location = tokenizer.Location;
while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current))
{
ws = true;
tokenizer.Consume();
}
if (ws)
{
return(new YSToken(location, YSToken.TokenType.WhiteSpace, null));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
int location = tokenizer.Location;
StringBuilder str = new StringBuilder ();
//first character of an identity cannot be a number
int intval = 0;
if(int.TryParse(tokenizer.Current, out intval))
return null;
while (!tokenizer.End ()
&& !String.IsNullOrWhiteSpace (tokenizer.Current)
&& !YSToken.SpecialCharacters.Any (character => character.Word == tokenizer.Current)) {
str.Append (tokenizer.Current);
tokenizer.Consume ();
}
if(str.Length > 0)
return new YSToken (location, YSToken.TokenType.Identity, str.ToString ());
return null;
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
string compile = "";
int location = tokenizer.Location;
var ctok = tokenizer.Current;
foreach (var character in Word)
{
if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture))
{
tokenizer.Consume();
compile += tokenizer.Current;
}
else
{
return(null);
}
}
bool endOfToken, complete;
var next = tokenizer.Current;
endOfToken = String.IsNullOrWhiteSpace(next) ||
YSToken.SpecialCharacters.Any(character => character.Word == next) ||
YSToken.SpecialCharacters.Any(character => character.Word == ctok);
//complete = compile.Equals (Word);
if (endOfToken)
{
return(new YSToken(location, Type, Word));
}
return(null);
}
protected override YSToken IsMatchImpl(YSTokenizer tokenizer)
{
//Console.WriteLine("Attempting to match Comment String");
var str = new StringBuilder();
int location = -1;
//Match to comment start
foreach (char character in COMMENT_START)
{
if (character.ToString().Equals(tokenizer.Current))
{
tokenizer.Consume();
}
else
{
return(null);
}
}
location = tokenizer.Location;
bool ProperEnd = false;
while (!tokenizer.End())
{
//Console.WriteLine (tokenizer.Current + " " + tokenizer.End());
bool MatchedEnd = true;
foreach (char character in COMMENT_END)
{
if (character.ToString().Equals(tokenizer.Current))
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
else
{
MatchedEnd = false;
break;
}
}
if (!MatchedEnd)
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
else
{
str.Remove(str.Length - COMMENT_END.Length, (COMMENT_END.Length));
ProperEnd = true;
break;
}
}
if (tokenizer.End())
{
return(null);
}
if (ProperEnd)
{
return(new YSToken(location, YSToken.TokenType.Comment, str.ToString()));
}
return(null);
}