/// <summary>
/// Build an Array of a particular type from a list of tokens.
/// The Type must be one that can be built with Convert.ChangeType.
/// There are various ways to specify how many elements to parse.
/// WARNING: This will throw an exception if any tokens cannot be
/// converted.
/// </summary>
/// <param name="tokens">The ArrayList of tokens.</param>
/// <param name="i">The starting (and ending) index. This is
/// modified, and left pointing at the last used token.</param>
/// <param name="type">The Type of the array elements.</param>
/// <param name="endToken">An optional end Token to look for.
/// Parsing stops when a token equal to this is found.
/// If this is null, then it is not used.</param>
/// <param name="maxLength">The maximum number of array elements
/// to parse. If this is negative, then it is not used.</param>
/// <param name="log">A Logger to use for messages.</param>
/// <returns>The Array, or null for error.</returns>
public static Array BuildArray(ArrayList tokens, ref int i, Type type,
Token endToken, int maxLength, Logger log)
{
int len = tokens.Count;
if (i >= len)
{
log.Error("BuildArray: Input index too large.");
return(null);
}
// put the objects into an array list first, since we don't
// know length
ArrayList list = new ArrayList();
// allow null endToken specified
if (endToken == null) endToken = new EofToken();
Token token = null;
token = (Token)tokens[i++];
int arrayLength = 0;
while ((!(token is EofToken)) && (token != endToken) && (i < len)
&& ((maxLength < 0) || (arrayLength < maxLength)))
{
Object o = token.ConvertToType(type);
list.Add(o);
arrayLength++;
token = (Token)tokens[i++];
}
i--; // went one past
return(list.ToArray(type));
}
/// <summary>
/// Get the next token. The last token will be an EofToken unless
/// there's an unterminated quote or unterminated block comment
/// and Settings.DoUntermCheck is true, in which case this throws
/// an exception of type StreamTokenizerUntermException or sub-class.
/// </summary>
/// <param name="token">The output token.</param>
/// <returns>bool - true for success, false for failure.</returns>
public bool NextToken(out Token token)
{
if (isPushback)
{
isPushback = false;
if (this.prevToken != null)
{
token = this.prevToken;
return true;
}
else
{
token = null;
return false;
}
}
token = null;
int thisChar = 0; // current character
byte ctype; // type of this character
NextTokenState state = NextTokenState.Start;
int prevChar = 0; // previous character
byte prevCtype = (byte)CharTypeBits.Eof;
// get previous char from nextTokenSb if there
// (nextTokenSb is a StringBuilder containing the characters
// of the next token to be emitted)
if (nextTokenSb.Length > 0)
{
prevChar = nextTokenSb[nextTokenSb.Length - 1];
prevCtype = settings.CharTypes[prevChar];
state = PickNextState(prevCtype, prevChar);
}
// extra state for number parse
int seenDot = 0; // how many .'s in the number
int seenE = 0; // how many e's or E's have we seen in the number
bool seenDigit = false; // seen any digits (numbers can start with -)
// lineNumber can change with each GetNextChar()
// tokenLineNumber is the line on which the token started
int tokenLineNumber = lineNumber;
// State Machine: Produces a single token.
// Enter a state based on a single character.
// Generally, being in a state means we're currently collecting chars
// in that type of token.
// We do state machine until it builds a token (Eof is a token), then
// return that token.
thisChar = prevChar; // for first iteration, since prevChar is set to this
bool done = false; // optimization
while (!done)
{
prevChar = thisChar;
thisChar = GetNextChar();
if (thisChar >= settings.CharTypes.Length)
{
// greater than 7-bit ascii, treat as word character
ctype = (byte)CharTypeBits.Word;
}
else ctype = settings.CharTypes[thisChar];
if (thisChar == Eof)
state = NextTokenState.Eof;
#if DEBUG
log.Debug("Before switch: state = {0}, thisChar = '{1}'", state, (char)thisChar);
#endif
// see if we need to change states, or emit a token
switch (state)
{
case NextTokenState.Start:
// RESET
state = PickNextState(ctype, thisChar);
tokenLineNumber = lineNumber;
break;
case NextTokenState.Char:
token = new CharToken((char)prevChar, tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
break;
case NextTokenState.Word:
if ((!settings.IsCharType(ctype, CharTypeBits.Word))
&& (!settings.IsCharType(ctype, CharTypeBits.Digit)))
{
// end of word, emit
token = new WordToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
break;
case NextTokenState.Whitespace:
if (!settings.IsCharType(ctype, CharTypeBits.Whitespace)
|| (settings.GrabEol && (thisChar == 10)))
{
// end of whitespace, emit
if (settings.GrabWhitespace)
{
token = new WhitespaceToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
else
{
// RESET
nextTokenSb.Length = 0;
tokenLineNumber = lineNumber;
state = PickNextState(ctype, thisChar);
}
}
break;
case NextTokenState.EndQuote:
// we're now 1 char after end of quote
token = new QuoteToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
break;
case NextTokenState.Quote:
// looking for end quote matching char that started the quote
if (thisChar == nextTokenSb[0])
{
// handle escaped backslashes: count the immediately prior backslashes
// - even (including 0) means it's not escaped
// - odd means it is escaped
int backSlashCount = 0;
for (int i = nextTokenSb.Length - 1; i >= 0; i--)
{
if (nextTokenSb[i] == '\\') backSlashCount++;
else break;
}
if ((backSlashCount % 2) == 0)
{
state = NextTokenState.EndQuote;
}
}
if ((state != NextTokenState.EndQuote) && (thisChar == Eof))
{
if (settings.DoUntermCheck)
{
nextTokenSb.Length = 0;
throw new StreamTokenizerUntermQuoteException("Unterminated quote");
}
token = new QuoteToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
break;
case NextTokenState.MaybeComment:
if (thisChar == Eof)
{
token = new CharToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
else
{
// if we get the right char, we're in a comment
if (settings.SlashSlashComments && (thisChar == '/'))
state = NextTokenState.LineComment;
else if (settings.SlashStarComments && (thisChar == '*'))
state = NextTokenState.BlockComment;
else
{
token = new CharToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
}
break;
case NextTokenState.LineComment:
if (thisChar == Eof)
{
if (settings.GrabComments)
{
token = new CommentToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
else
{
// RESET
nextTokenSb.Length = 0;
tokenLineNumber = lineNumber;
state = PickNextState(ctype, thisChar);
}
}
else
{
if (thisChar == '\n')
{
if (settings.GrabComments)
{
token = new CommentToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
else
{
// RESET
nextTokenSb.Length = 0;
tokenLineNumber = lineNumber;
state = PickNextState(ctype, thisChar);
}
}
}
break;
case NextTokenState.BlockComment:
if (thisChar == Eof)
{
if (settings.DoUntermCheck)
{
nextTokenSb.Length = 0;
throw new StreamTokenizerUntermCommentException("Unterminated comment.");
}
if (settings.GrabComments)
{
token = new CommentToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
else
{
// RESET
nextTokenSb.Length = 0;
tokenLineNumber = lineNumber;
state = PickNextState(ctype, thisChar);
}
}
else
{
if ((thisChar == '/') && (prevChar == '*'))
{
state = NextTokenState.EndBlockComment;
}
}
break;
// special case for 2-character token termination
case NextTokenState.EndBlockComment:
if (settings.GrabComments)
{
token = new CommentToken(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
else
{
// RESET
nextTokenSb.Length = 0;
tokenLineNumber = lineNumber;
state = PickNextState(ctype, thisChar);
}
break;
case NextTokenState.MaybeHex:
// previous char was 0
if (thisChar != 'x')
{
// back up and try non-hex
// back up to the 0
nextTokenSb.Append((char)thisChar);
backString.Append(nextTokenSb);
nextTokenSb.Length = 0;
// reset state and don't choose MaybeNumber state.
// pull char from backString
thisChar = backString[0];
backString.Remove(0, 1);
state = PickNextState(settings.CharTypes[thisChar], (int)thisChar,
NextTokenState.MaybeHex);
#if DEBUG
log.Debug("HexGot0x: Next state on '{0}' is {1}", (char)thisChar,
state);
#endif
}
else state = NextTokenState.HexGot0x;
break;
case NextTokenState.HexGot0x:
if (!settings.IsCharType(ctype, CharTypeBits.HexDigit))
{
// got 0x but now a non-hex char
// back up to the 0
nextTokenSb.Append((char)thisChar);
backString.Append(nextTokenSb);
nextTokenSb.Length = 0;
// reset state and don't choose MaybeNumber state.
// pull char from backString
thisChar = backString[0];
backString.Remove(0, 1);
state = PickNextState(settings.CharTypes[thisChar], (int)thisChar,
NextTokenState.MaybeHex);
#if DEBUG
log.Debug("HexGot0x: Next state on '{0}' is {1}", (char)thisChar,
state);
#endif
}
else state = NextTokenState.HexNumber;
break;
case NextTokenState.HexNumber:
if (!settings.IsCharType(ctype, CharTypeBits.HexDigit))
{
// emit the hex number we've collected
#if DEBUG
log.Debug("Emit hex IntToken from string '{0}'", nextTokenSb);
#endif
token = IntToken.ParseHex(nextTokenSb.ToString(), tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
}
break;
case NextTokenState.MaybeNumber:
//
// Determine whether or not to stop collecting characters for
// the number parse. We terminate when it's clear it's not
// a number or no longer a number.
//
bool term = false;
if (settings.IsCharType(ctype, CharTypeBits.Digit)
|| settings.IsCharType(prevChar, CharTypeBits.Digit)) seenDigit = true;
// term conditions
if (thisChar == '.')
{
seenDot++;
if (seenDot > 1) term = true; // more than one dot, it aint a number
}
else if (((thisChar == 'e') || (thisChar == 'E')))
{
seenE++;
if (!seenDigit) term = true; // e before any digits is bad
else if (seenE > 1) term = true; // more than 1 e is bad
else
{
term = true; // done regardless
// scan the exponent, put its characters into
// nextTokenSb, if there are any
char c;
expSb.Clear();
expSb.Append((char)thisChar);
if (GrabInt(expSb, true, out c))
{
// we got a good exponent, tack it on
nextTokenSb.Append(expSb);
thisChar = c; // and continue after the exponent's characters
}
}
}
else if (thisChar == Eof) term = true;
// or a char that can't be in a number
else if ((!settings.IsCharType(ctype, CharTypeBits.Digit)
&& (thisChar != 'e') && (thisChar != 'E')
&& (thisChar != '-') && (thisChar != '.'))
|| ((thisChar == '+') && (seenE == 0)))
{
// it's not a normal number character
term = true;
}
// or a dash not after e
else if ((thisChar == '-') && (!((prevChar == 'e') || (prevChar == 'E')))) term = true;
if (term)
{
// we are terminating a number, or it wasn't a number
if (seenDigit)
{
if ((nextTokenSb.IndexOf('.') >= 0)
|| (nextTokenSb.IndexOf('e') >= 0)
|| (nextTokenSb.IndexOf('E') >= 0)
|| (nextTokenSb.Length >= 19) // probably too large for Int64, use float
)
{
token = new FloatToken(nextTokenSb.ToString(), tokenLineNumber);
#if DEBUG
log.Debug("Emit FloatToken from string '{0}'", nextTokenSb);
#endif
}
else
{
#if DEBUG
log.Debug("Emit IntToken from string '{0}'", nextTokenSb);
#endif
token = new IntToken(nextTokenSb.ToString(), tokenLineNumber);
}
done = true;
nextTokenSb.Length = 0;
}
else
{
// -whatever or -.whatever
// didn't see any digits, must have gotten here by a leading -
// and no digits after it
// back up to -, pick next state excluding numbers
nextTokenSb.Append((char)thisChar);
backString.Append(nextTokenSb);
nextTokenSb.Length = 0;
// restart on the - and don't choose MaybeNumber state
// pull char from backString
thisChar = backString[0];
backString.Remove(0, 1);
state = PickNextState(settings.CharTypes[thisChar], (int)thisChar,
NextTokenState.MaybeNumber);
#if DEBUG
log.Debug("MaybeNumber: Next state on '{0}' is {1}", (char)thisChar,
state);
#endif
}
}
break;
case NextTokenState.Eol:
// tokenLineNumber - 1 because the newline char is on the previous line
token = new EolToken(tokenLineNumber - 1);
done = true;
nextTokenSb.Length = 0;
break;
case NextTokenState.Eof:
token = new EofToken(tokenLineNumber);
done = true;
nextTokenSb.Length = 0;
return (false);
case NextTokenState.Invalid:
default:
// not a good sign, some unrepresented state?
log.Error("NextToken: Hit unrepresented state {0}", state);
return (false);
}
// use a StringBuilder to accumulate characters which are part of this token
if (thisChar != Eof) nextTokenSb.Append((char)thisChar);
#if DEBUG
log.Debug("After switch: state = {0}, nextTokenSb = '{1}', backString = '{2}'",
state, nextTokenSb, backString);
#endif
}
#if DEBUG
log.Debug("Got token {0}", token.ToDebugString());
#endif
return (true);
}
public void PushBack(Token prevToken)
{
this.isPushback = true;
this.prevToken = prevToken;
}
/// <summary>
/// Given a Token[] and a reference int, skip forward
/// in the token array until a WordToken is found,
/// and leave the reference int at that index.
/// </summary>
/// <param name="tokens">The token array.</param>
/// <param name="i">The start index, and the result index.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static bool SkipToWord(Token[] tokens, ref int i)
{
while (!(tokens[i] is WordToken))
{
i++;
if (i >= tokens.Length) return(false);
}
return(true);
}
/// <summary>
/// Given a Token[] and a reference int, skip forward
/// in the token array until a WordToken is found,
/// and leave the reference int at that index.
/// </summary>
/// <param name="tokens">The token array.</param>
/// <param name="dropTokens">The tokens to drop.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static Token[] DropTokens(Token[] tokens,
Token[] dropTokens)
{
ArrayList outputList = new ArrayList();
int i = 0;
for (i = 0; i < tokens.Length; i++)
{
bool dropIt = false;
for (int j = 0; j < dropTokens.Length; j++)
{
if (tokens[i].Equals(dropTokens[j])) dropIt = true;
}
if (!dropIt) outputList.Add(tokens[i]);
}
// copy to array
Token[] outputTokens = new Token[outputList.Count];
i = 0;
foreach(Token t in outputList) outputTokens[i++] = t;
return(outputTokens);
}
/// <summary>
/// Find matching closing character.
/// The matchable pairs of characters are parenthesis (),
/// square brackets [], and curly braces {}.
/// Given a Token[] and a reference int containing the index
/// in the Token[] of a matchable? char, skip forward
/// in the token array until the matching character is found.
/// </summary>
/// <remarks>
/// This implicitly skips matching characters in quotes and
/// comments if they are hidden in the tokens. So if you grab
/// comments and quotes when you tokenize, the characters in those
/// tokens are not looked at by this function.
/// </remarks>
/// <param name="tokens">The token array.</param>
/// <param name="i">The start index, and the result index.</param>
/// <param name="c">The start character whose match is to be found.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static bool FindMatch(Token[] tokens, ref int i,
char c)
{
char endChar;
if (c == '(') endChar = ')';
else if (c == '{') endChar = '}';
else if (c == '[') endChar = ']';
else return(false);
int nestLevel = 1; // count first one
// i'th token must be the start character
if (tokens[i] != c)
{
return(false);
}
i++;
// terminate when we hit an end char and that takes us to
// nest level 0
while (nestLevel > 0)
{
if (tokens[i] == c) nestLevel++;
else if (tokens[i] == endChar) nestLevel--;
i++;
if (i >= tokens.Length) return(false);
}
i--; // went one past
return(true);
}
/// <summary>
/// Given a Token[] and a reference int, skip forward
/// in the token array until a WordToken is found,
/// and leave the reference int at that index.
/// </summary>
/// <param name="tokens">The token array.</param>
/// <param name="i">The start index, and the result index.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static bool SkipWs(Token[] tokens, ref int i)
{
while (tokens[i] is WhitespaceToken)
{
i++;
if (i >= tokens.Length) return(false);
}
return(true);
}
/// <summary>
/// Given a Token[] and a reference int, skip forward
/// in the token array until a WordToken is found,
/// and leave the reference int at that index.
/// </summary>
/// <param name="tokens">The token array.</param>
/// <param name="i">The start index, and the result index.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static bool SkipToEol(Token[] tokens, ref int i)
{
if ((i < 0) || (i >= tokens.Length)) return(false);
while (!(tokens[i] is EolToken))
{
i++;
if (i >= tokens.Length) return(false);
}
return(true);
}
/// <summary>
/// Given a Token[], a reference int and a string, skip forward
/// in the token array until a token matches the string
/// and leave the reference int at that index.
/// </summary>
/// <param name="tokens">The token array.</param>
/// <param name="i">The start index, and the result index.</param>
/// <param name="s">The string to look for.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static bool SkipToStringValue(Token[] tokens, ref int i,
string s)
{
while (tokens[i] != s)
{
i++;
if (i >= tokens.Length) return(false);
}
return(true);
}
/// <summary>
/// Given a Token[] and a reference int, skip forward
/// in the token array until a WordToken is found,
/// and leave the reference int at that index.
/// </summary>
/// <param name="tokens">The token array.</param>
/// <param name="i">The start index, and the result index.</param>
/// <param name="c">The char to look for.</param>
/// <returns>bool - true for success, false for
/// hit the end of the tokens.</returns>
public static bool SkipToChar(Token[] tokens, ref int i,
char c)
{
while(tokens[i] != c)
{
i++;
if (i >= tokens.Length) return(false);
}
return(true);
}
/// <summary> Gets next token, checking for a premature and of line.
///
/// </summary>
/// <param name="tokenizer">the stream tokenizer
/// </param>
/// <exception cref="IOException">if it finds a premature end of line
/// </exception>
protected internal virtual void getNextToken(StreamTokenizer tokenizer,out Token token)
{
tokenizer.NextToken(out token);
if (token is EolToken)
{
errms(tokenizer, "premature end of line");
}
if (token is EofToken)
{
errms(tokenizer, "premature end of file");
}
//else if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"'))
else if ((token is CharToken) && ((token.StringValue == "'") || (token.StringValue == "\"") ))
{
//tokenizer.ttype = SupportClass.StreamTokenizerSupport.TT_WORD;
}
//else if ((tokenizer.ttype == SupportClass.StreamTokenizerSupport.TT_WORD) && (tokenizer.sval.Equals("?")))
else if ((token is CharToken) && (token.StringValue == "?"))
{
//tokenizer.ttype = '?';
}
}
/// <summary> Gets token and checks if its end of line.
///
/// </summary>
/// <param name="tokenizer">the stream tokenizer
/// </param>
/// <exception cref="IOException">if it doesn't find an end of line
/// </exception>
protected internal virtual void getLastToken(StreamTokenizer tokenizer,out Token token , bool endOfFileOk)
{
tokenizer.NextToken(out token);
if ( (!(token is EolToken)) && ( (!(token is EofToken)) || !endOfFileOk))
{
errms(tokenizer, "end of line expected");
}
}
/// <summary> Gets index, checking for a premature and of line.
///
/// </summary>
/// <param name="tokenizer">the stream tokenizer
/// </param>
/// <exception cref="IOException">if it finds a premature end of line
/// </exception>
protected internal virtual void getIndex(StreamTokenizer tokenizer, out Token token)
{
tokenizer.NextToken(out token);
if (token is EolToken)
{
errms(tokenizer, "premature end of line");
}
if (token is EofToken)
{
errms(tokenizer, "premature end of file");
}
}
/// <summary> Gets next token, skipping empty lines.
///
/// </summary>
/// <param name="tokenizer">the stream tokenizer
/// </param>
/// <exception cref="IOException">if reading the next token fails
/// </exception>
protected internal virtual void getFirstToken(StreamTokenizer tokenizer,out Token token)
{
//while (tokenizer.NextToken() == SupportClass.StreamTokenizerSupport.TT_EOL)
tokenizer.NextToken(out token);
while(token is EolToken)
{
tokenizer.NextToken(out token);
} ;
//if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"'))
//if ((token.StringValue == "'") || (token.StringValue == "\"") )
//{
//tokenizer.ttype = SupportClass.StreamTokenizerSupport.TT_WORD;
//}
//else if ((tokenizer.ttype == SupportClass.StreamTokenizerSupport.TT_WORD) && (tokenizer.sval.Equals("?")))
//{
// tokenizer.ttype = '?';
//}
}
/// <summary> Throws error message with line number and last token read.
///
/// </summary>
/// <param name="theMsg">the error message to be thrown
/// </param>
/// <param name="tokenizer">the stream tokenizer
/// </param>
/// <throws> IOExcpetion containing the error message </throws>
protected internal virtual void errms(Token token, System.String theMsg)
{
throw new System.IO.IOException(theMsg + ", read " + token.ToString());
}
