void GetToken() { // eat white space while (_ptr < _len && _expr[_ptr] <= ' ') { _ptr++; } // are we done? if (_ptr >= _len) { _token = new Token(null, TKID.END, TKTYPE.GROUP); return; } // prepare to parse int i = 0; var c = _expr[_ptr]; // operators // this gets called a lot, so it's pretty optimized. // note that operators must start with non-letter/digit characters. var isLetter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); var isDigit = c >= '0' && c <= '9'; if (!isLetter && !isDigit && c != FormatChar.HEX_NUM && c != FormatChar.BIN_NUM) //HEX format will be handled later { // if this is a number starting with a decimal, don't parse as operator var nxt = _ptr + 1 < _len ? _expr[_ptr + 1] : 0; bool isNumber = c == _decimal && nxt >= '0' && nxt <= '9'; if (!isNumber) { // look up localized list separator if (c == _listSep) { _token = new Token(c, TKID.COMMA, TKTYPE.GROUP); _ptr++; return; } // look up single-char tokens on table Token tk; if (_tkTbl.TryGetValue(c, out tk)) { // save token we found _token = tk; _ptr++; // look for double-char tokens (special case) if (_ptr < _len && (c == '>' || c == '<')) { if (_tkTbl.TryGetValue(_expr.Substring(_ptr - 1, 2), out tk)) { _token = tk; _ptr++; } } // found token on the table return; } } } // parse numbers if (isDigit || c == _decimal) { var div = -1.0; // use double, not int (this may get really big) var val = 0.0; int num_fmt = 10; if (c == '0' && _len > 1) { c = _expr[_ptr + 1]; if (c == 'X' || c == 'x') num_fmt = 16; else if (c == 'b' || c == 'B') num_fmt = 2; } //Parse hex value that start with "0x" or "0X" if (16 == num_fmt) { for (i = 2; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (IsHex(c)) val = val * 16 + (double)GetHexValue(c); else break; } } else if (2 == num_fmt) { for (i = 2; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (IsBin(c)) val = val * 2 + (double)GetBinValue(c); else break; } } else { var sci = false; var pct = false; for (i = 0; i + _ptr < _len; i++) { c = _expr[_ptr + i]; // digits always OK if (c >= '0' && c <= '9') { val = val * 10 + (c - '0'); if (div > -1) { div *= 10; } continue; } // one decimal is OK if (c == _decimal && div < 0) { div = 1; continue; } // scientific notation? if ((c == 'E' || c == 'e') && !sci) { if (m_intFmt != IntegerFormat.DEC) { throw new Exception("Not decimal mode, doesn't support scientific notation"); } sci = true; c = _expr[_ptr + i + 1]; if (c == '+' || c == '-') i++; continue; } // percentage? if (c == _percent) { pct = true; i++; break; } // end of literal break; } // end of number, get value if (!sci) { // much faster than ParseDouble if (div > 1) { val /= div; } if (pct) { val /= 100.0; } } else { var lit = _expr.Substring(_ptr, i); val = ParseDouble(lit, _ci); } } // build token object fmtVal = Token.FormatValue(val, Expression.CurCalcMode, Expression.CurIntBits, Expression.CurIntFmt); _token = new Token(fmtVal, TKID.ATOM, TKTYPE.LITERAL); // advance pointer and return _ptr += i; return; } // parse strings if (c == FormatChar.STRING) { // look for end quote, skip double quotes for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (c != FormatChar.STRING) continue; char cNext = i + _ptr < _len - 1 ? _expr[_ptr + i + 1]: ' '; if (cNext != FormatChar.STRING) break; i++; } // check that we got the end of the string if (c != FormatChar.STRING) { Throw("Can't find final quote."); } // end of string var lit = _expr.Substring(_ptr + 1, i - 1); _ptr += i + 1; _token = new Token(lit.Replace("\"\"", "\""), TKID.ATOM, TKTYPE.LITERAL); return; } // parse dates (review) if (c == FormatChar.DATETIME) { // look for end # for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (c == FormatChar.DATETIME) break; } // check that we got the end of the date if (c != FormatChar.DATETIME) { Throw("Can't find final date delimiter ('" + FormatChar.DATETIME + "')."); } // end of date var lit = _expr.Substring(_ptr + 1, i - 1); _ptr += i + 1; _token = new Token(DateTime.Parse(lit, _ci), TKID.ATOM, TKTYPE.LITERAL); return; } // parse HEX format if (c == FormatChar.HEX_NUM) { // look for end # for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (!(c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' || c >= '0' && c <= '9')) break; } // check that we got some chars after @ if (i == 1) { Throw("No characters after HEX format identify '" + FormatChar.HEX_NUM + "'!"); } var lit = _expr.Substring(_ptr + 1, i - 1); _ptr += i; _token = new Token(int.Parse((string)lit, System.Globalization.NumberStyles.HexNumber), TKID.ATOM, TKTYPE.LITERAL); return; } else if (c == FormatChar.BIN_NUM) { // look for end for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (c != '0' && c != '1') break; } // check that we got some chars after @ if (i == 1) { Throw("No characters after BIN format identify '" + FormatChar.BIN_NUM + "'!"); } // end of date var lit = _expr.Substring(_ptr + 1, i - 1); _ptr += i; _token = new Token(Expression.ParseBinStr((string)lit),TKID.ATOM, TKTYPE.LITERAL); return; } // identifiers (functions, objects) must start with alpha or underscore if (!isLetter && c != '_' && (_idChars == null || _idChars.IndexOf(c) < 0)) { Throw("Identifier expected."); } // and must contain only letters/digits/_idChars for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; isLetter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); isDigit = c >= '0' && c <= '9'; if (!isLetter && !isDigit && c != '_' && (_idChars == null || _idChars.IndexOf(c) < 0)) { break; } } // got identifier var id = _expr.Substring(_ptr, i); _ptr += i; _token = new Token(id, TKID.ATOM, TKTYPE.IDENTIFIER); }
// e.g. myfun(a, b, c+2) List<Expression> GetParameters() { // check whether next token is a (, // restore state and bail if it's not var pos = _ptr; var tk = _token; GetToken(); if (_token.ID != TKID.OPEN) { _ptr = pos; _token = tk; return null; } // check for empty Parameter list pos = _ptr; GetToken(); if (_token.ID == TKID.CLOSE) { return null; } _ptr = pos; // get Parameters until we reach the end of the list var parms = new List<Expression>(); var expr = ParseExpression(); parms.Add(expr); while (_token.ID == TKID.COMMA) { expr = ParseExpression(); parms.Add(expr); } // make sure the list was closed correctly if (_token.ID != TKID.CLOSE) { Throw(); } // done return parms; }
void AddToken(object symbol, TKID id, TKTYPE type) { var token = new Token(symbol, id, type); _tkTbl.Add(symbol, token); }
Token GetMember() { // check whether next token is a MEMBER token ('.'), // restore state and bail if it's not var pos = _ptr; var tk = _token; GetToken(); if (_token.ID != TKID.PERIOD) { _ptr = pos; _token = tk; return null; } // skip member token GetToken(); if (_token.Type != TKTYPE.IDENTIFIER) { Throw("Identifier expected"); } return _token; }
void GetToken() { // eat white space while (_ptr < _len && _expr[_ptr] <= ' ') { _ptr++; } // are we done? if (_ptr >= _len) { _token = new Token(null, TKID.END, TKTYPE.GROUP); return; } // prepare to parse int i; var c = _expr[_ptr]; // operators // this gets called a lot, so it's pretty optimized. // note that operators must start with non-letter/digit characters. var isLetter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); var isDigit = c >= '0' && c <= '9'; if (!isLetter && !isDigit) { // if this is a number starting with a decimal, don't parse as operator var nxt = _ptr + 1 < _len ? _expr[_ptr + 1] : 0; bool isNumber = c == _decimal && nxt >= '0' && nxt <= '9'; if (!isNumber) { // look up localized list separator if (c == _listSep) { _token = new Token(c, TKID.COMMA, TKTYPE.GROUP); _ptr++; return; } // look up single-char tokens on table Token tk; if (_tkTbl.TryGetValue(c, out tk)) { // save token we found _token = tk; _ptr++; // look for double-char tokens (special case) if (_ptr < _len && (c == '>' || c == '<')) { if (_tkTbl.TryGetValue(_expr.Substring(_ptr - 1, 2), out tk)) { _token = tk; _ptr++; } } // found token on the table return; } } } // parse numbers if (isDigit || c == _decimal) { var sci = false; var pct = false; var div = -1.0; // use double, not int (this may get really big) var val = 0.0; for (i = 0; i + _ptr < _len; i++) { c = _expr[_ptr + i]; // digits always OK if (c >= '0' && c <= '9') { val = val * 10 + (c - '0'); if (div > -1) { div *= 10; } continue; } // one decimal is OK if (c == _decimal && div < 0) { div = 1; continue; } // scientific notation? if ((c == 'E' || c == 'e') && !sci) { sci = true; c = _expr[_ptr + i + 1]; if (c == '+' || c == '-') i++; continue; } // percentage? if (c == _percent) { pct = true; i++; break; } // end of literal break; } // end of number, get value if (!sci) { // much faster than ParseDouble if (div > 1) { val /= div; } if (pct) { val /= 100.0; } } else { var lit = _expr.Substring(_ptr, i); val = ParseDouble(lit, _ci); } // build token _token = new Token(val, TKID.ATOM, TKTYPE.LITERAL); // advance pointer and return _ptr += i; return; } // parse strings if (c == '\"') { // look for end quote, skip double quotes for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (c != '\"') continue; char cNext = i + _ptr < _len - 1 ? _expr[_ptr + i + 1]: ' '; if (cNext != '\"') break; i++; } // check that we got the end of the string if (c != '\"') { Throw("Can't find final quote."); } // end of string var lit = _expr.Substring(_ptr + 1, i - 1); _ptr += i + 1; _token = new Token(lit.Replace("\"\"", "\""), TKID.ATOM, TKTYPE.LITERAL); return; } // parse dates (review) if (c == '#') { // look for end # for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; if (c == '#') break; } // check that we got the end of the date if (c != '#') { Throw("Can't find final date delimiter ('#')."); } // end of date var lit = _expr.Substring(_ptr + 1, i - 1); _ptr += i + 1; _token = new Token(DateTime.Parse(lit, _ci), TKID.ATOM, TKTYPE.LITERAL); return; } // identifiers (functions, objects) must start with alpha or underscore if (!isLetter && c != '_' && (_idChars == null || _idChars.IndexOf(c) < 0)) { Throw("Identifier expected."); } // and must contain only letters/digits/_idChars for (i = 1; i + _ptr < _len; i++) { c = _expr[_ptr + i]; isLetter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); isDigit = c >= '0' && c <= '9'; if (!isLetter && !isDigit && c != '_' && (_idChars == null || _idChars.IndexOf(c) < 0)) { break; } } // got identifier var id = _expr.Substring(_ptr, i); _ptr += i; _token = new Token(id, TKID.ATOM, TKTYPE.IDENTIFIER); }