/// <summary> /// Reads a punctuation token. /// </summary> /// <param name="chars"> The first character of the punctuation token. </param> /// <returns> A punctuation token. </returns> private Token ReadPunctuator(int chars) { // The most likely case is the the punctuator is a single character and is followed by a space. var punctuator = PunctuatorToken.FromID(chars); if (this.reader.Peek() == ' ') { return(punctuator); } // Otherwise, read characters until we find a string that is not a punctuator. while (true) { int c = this.reader.Peek(); if (c == -1) { break; } // Try to parse the text as a punctuator. chars = (chars << 8) | c; var longPunctuator = PunctuatorToken.FromID(chars); if (longPunctuator == null) { break; } punctuator = longPunctuator; // Advance the input stream. ReadNextChar(); } return(punctuator); }
private static void Add(PunctuatorToken p) { string b = p.Text; int id = 0; // Create the ID from the string: for (int i = 0; i < b.Length; i++) { id = (id << 8) | (int)b[i]; } LookupTable[id] = p; }
public static void Setup() { // Called to trigger setting up the operators (because the static constructor might not be called otherwise). PunctuatorToken.Setup(); }