private int SkipComments(string Text, int StrPtr)
{
if (TokenDefinition.Matches(Text, StrPtr, "("))
{
while (!TokenDefinition.Matches(Text, StrPtr, ")"))
{
if (TokenDefinition.Matches(Text, StrPtr, "\n"))
{
line++;
}
StrPtr++;
}
StrPtr++;
}
return(StrPtr);
}
public List <Token> Tokenise(string Text)
{
List <Token> tokens = new List <Token>();
line = 1;
int StrPtr = 0;
while (true)
{
if (StrPtr >= Text.Length)
{
tokens.Add(new Token("NL", "NL", line));//shhh this is our secret...
tokens.Add(new Token("EOF", "EOF", line));
break;
}
//skip white space
StrPtr = SkipCommentsAndWhiteSpace(Text, StrPtr);
//look for new lines and empty lines
if (TokenDefinition.Matches(Text, StrPtr, "\n"))
{
StrPtr++;
int oldStrPtr = StrPtr;
line++;
StrPtr = SkipCommentsAndWhiteSpace(Text, StrPtr);
if (TokenDefinition.Matches(Text, StrPtr, "\n"))
{
StrPtr++;
line++;
tokens.Add(new Token("NL", "NL"));
tokens.Add(new Token("ELINE", "ELINE"));
continue;
}
StrPtr = oldStrPtr;
tokens.Add(new Token("NL", "NL"));
continue;
}
if (StrPtr >= Text.Length)
{
//loop back around to add EOF
continue;
}
bool found = false;
foreach (TokenDefinition tc in TokDefs)
{
TokenDefinition.TokenResult tr = null;
if (tc is MatchingTokenDefinition)
{
tr = (tc as MatchingTokenDefinition).Match(Text, StrPtr);
}
else if (tc is ConstantDefinition)
{
tr = (tc as ConstantDefinition).Match(Text, StrPtr);
}
else if (tc is PreviousDependantDefinition)
{
tr = (tc as PreviousDependantDefinition).Match(Text, StrPtr, tokens);
}
if (tr != null)
{
tr.T.LineNumber = line;
tokens.Add(tr.T);
StrPtr = tr.NewStrPtr;
found = true;
line += tr.LinesAbsorbed;
break;
}
}
if (!found)
{
throw new TokenException("Invalid token found on line: " + line);
}
}
return(tokens);
}
public Tokeniser()
{
TokenDefinitionList defs = new TokenDefinitionList();
//most of the tokens are defined here, a couple such as EOF and ELINE (empty line) are defined in the Tokenise method
//common variable specifiers
defs.Add("a", "CVARSP", true);
defs.Add("A", "CVARSP", true);
defs.Add("an", "CVARSP", true);
defs.Add("An", "CVARSP", true);
defs.Add("the", "CVARSP", true);
defs.Add("The", "CVARSP", true);
defs.Add("my", "CVARSP", true);
defs.Add("My", "CVARSP", true);
defs.Add("your", "CVARSP", true);
defs.Add("Your", "CVARSP", true);
//last variable/pronouns
defs.Add("it", "LVAR", true);
defs.Add("he", "LVAR", true);
defs.Add("she", "LVAR", true);
defs.Add("him", "LVAR", true);
defs.Add("her", "LVAR", true);
defs.Add("them", "LVAR", true);
defs.Add("they", "LVAR", true);
defs.Add("ze", "LVAR", true);
defs.Add("hir", "LVAR", true);
defs.Add("zie", "LVAR", true);
defs.Add("zir", "LVAR", true);
defs.Add("xe", "LVAR", true);
defs.Add("xem", "LVAR", true);
defs.Add("ve", "LVAR", true);
defs.Add("ver", "LVAR", true);
//assignment
defs.Add("put", "PUT", true);
defs.Add("Put", "PUT", true);
defs.Add("into", "INTO", true);
//poetic literals
//poetic type literals
defs.Add("is", "IS", true);
defs.Add("was", "WAS", true);
defs.Add("were", "WAS", true);
//poetic string literals
defs.Add("says", "SAYS", true);
//arithmetic
defs.Add("plus", "ADD", true);
defs.Add("with", "ADD", true);
defs.Add("minus", "SUB", true);
defs.Add("without", "SUB", true);
defs.Add("times", "MULT", true);
defs.Add("of", "MULT", true);
defs.Add("over", "DIV", true);
//comparison
defs.Add("not", "NOT", true);
defs.Add("ain't", "AINT", true);
defs.Add("aint", "AINT", true);
defs.Add("than", "THAN", true);
defs.Add("higher", "GT", true);
defs.Add("greater", "GT", true);
defs.Add("bigger", "GT", true);
defs.Add("stronger", "GT", true);
defs.Add("lower", "LT", true);
defs.Add("less", "LT", true);
defs.Add("smaller", "LT", true);
defs.Add("weaker", "LT", true);
defs.Add("as", "AS", true);
defs.Add("high", "GTEQ", true);
defs.Add("great", "GTEQ", true);
defs.Add("big", "GTEQ", true);
defs.Add("strong", "GTEQ", true);
defs.Add("low", "LTEQ", true);
defs.Add("little", "LTEQ", true);
defs.Add("small", "LTEQ", true);
defs.Add("weak", "LTEQ", true);
//increment/decrement
defs.Add("build", "BLD", true);
defs.Add("Build", "BLD", true);
defs.Add("up", "UP", true);
defs.Add("knock", "KNK", true);
defs.Add("Knock", "KNK", true);
defs.Add("down", "DWN", true);
//I/O
defs.Add("Say", "SAY", true);
defs.Add("Shout", "SAY", true);
defs.Add("Whisper", "SAY", true);
defs.Add("Listen to", "LSTN", true);
defs.Add("Listen", "LSTN", true);
//loops
defs.Add("While", "WHILE", true);
defs.Add("while", "WHILE", true);
defs.Add("Until", "UNTIL", true);
defs.Add("until", "UNTIL", true);
//types
//undefined
defs.Add("mysterious", "UNDEF", true);
//null
defs.Add("nothing", "NULL", true);
defs.Add("nowhere", "NULL", true);
defs.Add("nobody", "NULL", true);
defs.Add("null", "NULL", true);
defs.Add("empty", "NULL", true);
defs.Add("gone", "NULL", true);
//boolean
defs.Add("true", "TRUE", true);
defs.Add("right", "TRUE", true);
defs.Add("yes", "TRUE", true);
defs.Add("ok", "TRUE", true);
defs.Add("false", "FALSE", true);
defs.Add("wrong", "FALSE", true);
defs.Add("no", "FALSE", true);
defs.Add("lies", "FALSE", true);
//numeric litterals
defs.Add(new MatchingTokenDefinition((Text, StrPtr) =>
{
if (TokenDefinition.InSet(Text, StrPtr, NumberChars))
{
string value = "";
while (TokenDefinition.InSet(Text, StrPtr, NumberChars))
{
value += Text[StrPtr++];
}
//allow a single decimal place
if (TokenDefinition.Matches(Text, StrPtr, "."))
{
value += ".";
StrPtr++;
}
while (TokenDefinition.InSet(Text, StrPtr, NumberChars))
{
value += Text[StrPtr++];
}
return(new TokenDefinition.TokenResult()
{
NewStrPtr = StrPtr, T = new Token("NUM", value)
});
}
return(null);
}));
//string literal
defs.Add(new MatchingTokenDefinition((Text, StrPtr) =>
{
if (TokenDefinition.Matches(Text, StrPtr, "\""))
{
StrPtr++;
string value = "";
char last = ' ';
// I don't know if escape characters exist in RockStar, but they do in this implementation...
while (!(TokenDefinition.Matches(Text, StrPtr, "\"") && !TokenDefinition.Matches(Text, StrPtr - 1, "\\\"")))
{
char c = Text[StrPtr++];
if (last == '\\')
{
if (c == '"')
{
value += "\"";
}
else if (c == 'n')
{
value += "\n";
}
else if (c == 't')
{
value += "\t";
}
else if (c == '\\')
{
value += "\\";
}
else
{
throw new TokenException("Invalid escape character: \\" + c);
}
last = c;
continue;
}
last = c;
if (c == '\\')
{
continue;
}
value += c;
}
StrPtr++;
return(new TokenDefinition.TokenResult()
{
NewStrPtr = StrPtr, T = new Token("STR", value)
});
}
return(null);
}));
//object
//uhh...
//proper variables
defs.Add(new MatchingTokenDefinition((Text, StrPtr) =>
{
string name = "";
if (TokenDefinition.IsBetween(Text, StrPtr, 'A', 'Z'))
{
while (TokenDefinition.IsBetween(Text, StrPtr, 'A', 'Z') || TokenDefinition.IsBetween(Text, StrPtr, 'a', 'z') || (TokenDefinition.Matches(Text, StrPtr, " ") && TokenDefinition.IsBetween(Text, ++StrPtr, 'A', 'Z')))
{
name += Text[StrPtr];
StrPtr++;
}
return(new TokenDefinition.TokenResult()
{
NewStrPtr = StrPtr, T = new Token("PVAR", name)
});
}
return(null);
}));
//'s as is
defs.Add(new PreviousDependantDefinition((Text, StrPtr, Tokens) =>
{
if ((Tokens.Count > 0 && Tokens.Last().Name == "PVAR") || (Tokens.Count > 1 && Tokens[Tokens.Count - 2].Name == "CVARSP") && TokenDefinition.Matches(Text, StrPtr, "'s"))
{
StrPtr += 2;
return(new TokenDefinition.TokenResult()
{
NewStrPtr = StrPtr, T = new Token("IS", "'s")
});
}
return(null);
}));
//words
defs.Add(".", "WORD");
defs.Add(new MatchingTokenDefinition((Text, StrPtr) =>
{
if (TokenDefinition.InSet(Text, StrPtr, WordCharacters))
{
string text = "";
while (TokenDefinition.InSet(Text, StrPtr, WordCharacters))
{
text += Text[StrPtr];
StrPtr++;
}
return(new TokenDefinition.TokenResult()
{
NewStrPtr = StrPtr, T = new Token("WORD", text)
});
}
return(null);
}));
TokDefs = defs;
}
