/// <summary>
/// Is token getter.
/// </summary>
static TokenId IsToken(char c, AllowToken allowToken)
{
for (int i = 0; i < tokenSpecs.Length; i++)
{
if (c == tokenSpecs[i].Token)
{
if (allowToken(tokenSpecs[i].TokenId) != AllowTokenResult.AsIdentifier)
{
return(tokenSpecs[i].TokenId);
}
}
}
return(TokenId.Unknown);
}
/// <summary>
/// Tokenizes the string.
/// </summary>
public static List <Token> Tokenize(string str, AllowToken allowToken)
{
int startIndex = 0;
int currentIndex = 0;
uint lineNumber = 0;
List <Token> tokens = new List <Token>();
// We tokenize.
for (; currentIndex < str.Length; currentIndex++)
{
TokenId id = IsToken(str[currentIndex], allowToken);
if (id != TokenId.Unknown)
{
if (startIndex != currentIndex)
{
tokens.Add(new Token(str.Substring(startIndex, currentIndex - startIndex), lineNumber));
}
tokens.Add(new Token(id, lineNumber));
startIndex = currentIndex + 1;
}
if (str[currentIndex] == '\n')
{
lineNumber++;
}
}
// We add last token.
if (startIndex != currentIndex)
{
tokens.Add(new Token(str.Substring(startIndex, currentIndex - startIndex), lineNumber));
}
// We make some final adjustment, such as grouping numbers.
for (int i = 0; i < tokens.Count; i++)
{
// 1) Numbers are "enlarged", so we support "1.0" to be parsed as one number, or
// "1.0d" to be parsed as number and identifier.
if (allowToken(TokenId.Number) == AllowTokenResult.Allow &&
tokens[i].TokenId == TokenId.Number &&
(i + 2) < tokens.Count && tokens[i + 1].TokenId == TokenId.Dot)
{
if (tokens[i + 2].TokenId == TokenId.Number)
{
tokens[i].identifier = tokens[i].Identifier + "," + tokens[i + 2].Identifier;
tokens.RemoveRange(i + 1, 2);
}
else if (tokens[i + 2].TokenId == TokenId.Identifier)
{
string specifier;
int r = GetNumberWithSpecifier(tokens[i + 2].Identifier, out specifier);
if (r == int.MinValue)
{
continue;
}
// Otherwise, we rearangle.
tokens[i].identifier = tokens[i].Identifier + "," + r.ToString();
tokens[i + 2].identifier = specifier;
tokens.RemoveAt(i + 1);
}
// 2) Numbers with single specifier, "5d"
}
else if (allowToken(TokenId.Number) == AllowTokenResult.Allow &&
tokens[i].TokenId == TokenId.Identifier)
{
string specifier;
int r = GetNumberWithSpecifier(tokens[i].Identifier, out specifier);
if (r == int.MinValue)
{
continue;
}
// Otherwise we rearangle.
tokens[i].identifier = r.ToString();
tokens[i].tokenId = TokenId.Number;
tokens.Insert(i + 1, new Token(specifier, tokens[i].lineNumber));
}
// 4) String grouping.
else if (tokens[i].TokenId == TokenId.Quote &&
allowToken(TokenId.String) == AllowTokenResult.Allow)
{
if (i > 0 && tokens[i - 1].TokenId == TokenId.Backslash)
{
continue;
}
StringBuilder builder = new StringBuilder();
bool found = false;
int j = i;
while (++j < tokens.Count)
{
if (tokens[j].TokenId == TokenId.Quote)
{
found = true;
break;
}
builder.Append(tokens[j].TokenValue);
}
// If not found, we do nothing.
if (!found)
{
continue;
}
// We create range.
tokens[i] = new Token(TokenId.String, builder.ToString(), tokens[i].LineNumber);
tokens.RemoveRange(i + 1, j - i);
}
// 4) Comments
// 5) We support grouppings
else if (i > 0)
{
// a) &&
if (allowToken(TokenId.LogicalAnd) == AllowTokenResult.Allow &&
tokens[i - 1].TokenId == TokenId.And &&
tokens[i].TokenId == TokenId.And)
{
tokens.RemoveAt(i);
tokens[i - 1] = new Token(TokenId.LogicalAnd, tokens[i - 1].LineNumber);
i--;
}
// b) ||
else if (
allowToken(TokenId.LogicalOr) == AllowTokenResult.Allow &&
tokens[i - 1].TokenId == TokenId.Or &&
tokens[i].TokenId == TokenId.Or)
{
tokens.RemoveAt(i);
tokens[i - 1] = new Token(TokenId.LogicalOr, tokens[i - 1].LineNumber);
i--;
}
// c) ->
else if (
allowToken(TokenId.Arrow) == AllowTokenResult.Allow &&
tokens[i - 1].TokenId == TokenId.Minus &&
tokens[i].TokenId == TokenId.Less)
{
tokens.RemoveAt(i);
tokens[i - 1] = new Token(TokenId.Arrow, tokens[i - 1].LineNumber);
i--;
}
// d) ==
else if (
allowToken(TokenId.Equal) == AllowTokenResult.Allow &&
tokens[i - 1].TokenId == TokenId.Assign &&
tokens[i].TokenId == TokenId.Assign)
{
tokens.RemoveAt(i);
tokens[i - 1] = new Token(TokenId.Equal, tokens[i - 1].LineNumber);
i--;
}
// d) !=
else if (
allowToken(TokenId.NotEqual) == AllowTokenResult.Allow &&
tokens[i - 1].TokenId == TokenId.Not &&
tokens[i].TokenId == TokenId.Assign)
{
tokens.RemoveAt(i);
tokens[i - 1] = new Token(TokenId.NotEqual, tokens[i - 1].LineNumber);
i--;
}
}
}
tokens.Add(new Token(TokenId.Terminate, 0));
// We now filter all values.
uint identifierLine = 0;
StringBuilder identifierBuild = new StringBuilder();
List <Token> results = new List <Token>(tokens.Count / 2);
for (int i = 0; i < tokens.Count; i++)
{
AllowTokenResult result = allowToken(tokens[i].TokenId);
// We handle identifiers.
if (result == AllowTokenResult.AsIdentifier)
{
identifierLine = tokens[i].LineNumber;
identifierBuild.Append(tokens[i].TokenValue);
continue;
}
// We now handle non-ignores.
if (identifierBuild.Length != 0)
{
results.Add(new Token(identifierBuild.ToString(), identifierLine));
identifierBuild.Length = 0;
}
if (result != AllowTokenResult.Ignore)
{
results.Add(tokens[i]);
}
}
// We should push them.
return(results);
}
