/// <summary>
/// Reads a punctuation token.
/// </summary>
/// <param name="chars"> The first character of the punctuation token. </param>
/// <returns> A punctuation token. </returns>
private Token ReadPunctuator(int chars)
{
// The most likely case is the the punctuator is a single character and is followed by a space.
var punctuator = PunctuatorToken.FromID(chars);
if (this.reader.Peek() == ' ')
{
return(punctuator);
}
// Otherwise, read characters until we find a string that is not a punctuator.
while (true)
{
int c = this.reader.Peek();
if (c == -1)
{
break;
}
// Try to parse the text as a punctuator.
chars = (chars << 8) | c;
var longPunctuator = PunctuatorToken.FromID(chars);
if (longPunctuator == null)
{
break;
}
punctuator = longPunctuator;
// Advance the input stream.
ReadNextChar();
}
return(punctuator);
}
private static void Add(PunctuatorToken p)
{
string b = p.Text;
int id = 0;
// Create the ID from the string:
for (int i = 0; i < b.Length; i++)
{
id = (id << 8) | (int)b[i];
}
LookupTable[id] = p;
}
public static void Setup()
{
// Called to trigger setting up the operators (because the static constructor might not be called otherwise).
PunctuatorToken.Setup();
}
