public bool TryReadIdentifier(out string token, out StringTokenType type) { token = null; type = StringTokenType.Char; var peek = PeekChar(); if (peek != '_' && !char.IsLetter(peek) && (_nonBreakingIdentifierChars == null || !_nonBreakingIdentifierChars.Contains(peek))) { return(false); } var sb = new StringBuilder(); sb.Append(NextChar()); while (!Eof()) { peek = PeekChar(); if (char.IsLetterOrDigit(peek) || peek == '_') { sb.Append(NextChar()); } else { if (_nonBreakingIdentifierChars == null || !_nonBreakingIdentifierChars.Contains(peek)) { break; } sb.Append(NextChar()); } } type = StringTokenType.Identifier; token = sb.ToString(); return(true); }
public bool TryReadHex(out string token, out StringTokenType type) { type = StringTokenType.Char; token = null; if (PeekChar() == '0') { var peek = PeekChar(1); // try hex if ((peek == 'x' || peek == 'X')) { var sb = new StringBuilder(); sb.Append(NextChar()); sb.Append(NextChar()); while (PeekChar().IsHex()) { sb.Append(NextChar()); } type = StringTokenType.Hexadecimal; token = sb.ToString(); return(true); } } return(false); }
public bool TryReadString(out string token, out StringTokenType type) { type = StringTokenType.Char; token = null; if (PeekChar() != _quote) { return(false); } NextChar(); var sb = new StringBuilder(); while (!Eof()) { var ch = NextChar(); if (ch == _quote) { if (PeekChar() == _quote) { // escaped quote sb.Append(NextChar()); } else { type = StringTokenType.String; token = sb.ToString(); return(true); } } else { sb.Append(ch); } } throw new XPressionException(Source, "unterminated string", Position - sb.Length); }
private void Consume(Func <Char, Boolean> isCheck, StringTokenType type, out StringToken token) { Char c = this.array[this.idx]; int start = this.idx; while (isCheck(c)) { this.idx++; if (this.idx >= this.array.Length) { break; } c = this.array[this.idx]; } token = new StringToken(); token.String = new String(this.array, start, this.idx - start); token.Type = type; }
public bool TryReadComment(out string token, out StringTokenType type) { token = null; type = StringTokenType.Char; if (Peek() != _lineComment) { return(false); } var sb = new StringBuilder(); while (!Eof() && PeekChar() != '\n') { sb.Append(NextChar()); } token = sb.ToString(); type = StringTokenType.Comment; return(true); }
public bool TryReadJson(out string token, out StringTokenType type) { type = StringTokenType.Char; token = null; if (PeekChar() == '{') { var sb = new StringBuilder(); sb.Append(NextChar()); var bracketCount = 1; while (!Eof()) { switch (PeekChar()) { case '{': bracketCount++; break; case '}': bracketCount--; if (bracketCount == 0) { sb.Append(NextChar()); token = sb.ToString(); type = StringTokenType.JSON; return(true); } break; case '\"': case '\'': sb.Append(ReadJsonString(PeekChar())); continue; } sb.Append(NextChar()); } } return(false); }
//{00000000-0000-0000-0000-000000000000} public bool TryReadGuid(out string token, out StringTokenType type) { type = StringTokenType.Char; token = null; if (PeekChar() == '{') { if (PeekChar(9) == '-' && PeekChar(14) == '-' && PeekChar(19) == '-' && PeekChar(24) == '-' && PeekChar(37) == '}') { var current = Position; var guidString = Read(38); Guid guid; if (!Guid.TryParse(guidString, out guid)) { Position = current; return(false); } token = guidString; type = StringTokenType.Guid; return(true); } return(false); } if (PeekChar(8) == '-' && PeekChar(13) == '-' && PeekChar(18) == '-' && PeekChar(23) == '-') { var current = Position; var guidString = Read(36); Guid guid; if (!Guid.TryParse(guidString, out guid)) { Position = current; return(false); } token = guidString; type = StringTokenType.Guid; return(true); } return(false); }
public StringToken(StringTokenType t, int start) { Type = t; Start = start; End = start + 1; }
public StringToken(StringTokenType t, int start, int end) { Type = t; Start = start; End = end; }
public string NextToken(out StringTokenType type) { if (Eof()) { type = StringTokenType.Eof; return(null); } string token; if (TryReadComment(out token, out type)) { return(token); } if (TryReadString(out token, out type)) { return(token); } if (TryReadGuid(out token, out type)) { return(token); } if (PeekIsDigit()) { if (TryReadHex(out token, out type)) { return(token); } if (TryReadDate(out token, out type)) { return(token); } if (TryReadTime(out token, out type)) { return(token); } if (TryReadNumeric(out token, out type)) { return(token); } } if (TryReadDuration(out token, out type)) { return(token); } if (TryReadJson(out token, out type)) { return(token); } if (TryReadIdentifier(out token, out type)) { return(token); } type = StringTokenType.Char; return(Read(1)); }
public bool TryReadDuration(out string token, out StringTokenType type) { token = null; type = StringTokenType.Char; if (PeekChar() != 'P') { return(false); } var position = Position; var sb = new StringBuilder(); sb.Append(NextChar()); if (PeekIsDigit()) { while (PeekIsDigit()) { sb.Append(NextChar()); } if (!PeekAnyOf('Y', 'M', 'D')) { Seek(position); return(false); } sb.Append(NextChar()); if (PeekIsDigit()) { while (PeekIsDigit()) { sb.Append(NextChar()); } if (!PeekAnyOf('M', 'D')) { Seek(position); return(false); } sb.Append(NextChar()); if (PeekIsDigit()) { while (PeekIsDigit()) { sb.Append(NextChar()); } if (PeekChar() != 'D') { Seek(position); return(false); } sb.Append(NextChar()); } } if (PeekChar() != 'T') { token = sb.ToString(); type = StringTokenType.Duration; return(true); } sb.Append(NextChar()); } else { if (PeekChar() != 'T') { Seek(position); return(false); } sb.Append(NextChar()); } // read time part while (PeekIsDigit()) { sb.Append(NextChar()); } if (!PeekAnyOf('H', 'M', 'S')) { Seek(position); return(false); } sb.Append(NextChar()); if (PeekIsDigit()) { while (PeekIsDigit()) { sb.Append(NextChar()); } if (!PeekAnyOf('M', 'S')) { Seek(position); return(false); } sb.Append(NextChar()); if (PeekIsDigit()) { while (PeekIsDigit()) { sb.Append(NextChar()); } if (PeekChar() != 'S') { Seek(position); return(false); } sb.Append(NextChar()); } } token = sb.ToString(); type = StringTokenType.Duration; return(true); }
public bool TryReadTime(out string token, out StringTokenType type) { token = null; type = StringTokenType.Char; if (PeekChar(2) != ':' || PeekChar(5) != ':') { return(false); } var position = Position; var sb = new StringBuilder(); for (var i = 0; i < 8; i++) { if (i != 2 && i != 5 && !PeekIsDigit()) { Seek(position); return(false); } sb.Append(NextChar()); } if (PeekChar() == '.') { // read thousands sb.Append(NextChar()); while (PeekIsDigit()) { sb.Append(NextChar()); } } switch (PeekChar()) { case 'Z': // UTC Datetime : 2002-09-24T06:00:00.000Z sb.Append(NextChar()); type = StringTokenType.TimeUtc; token = sb.ToString(); return(true); case '+': case '-': position = Position; type = StringTokenType.Time; token = sb.ToString(); // Timeoffset : 2002-09-24T06:00:00.000+06:00 2002-09-24T06:00:00.000-06:00 sb.Append(NextChar()); for (var i = 0; i < 5; i++) { // read offset if (i == 2) { if (PeekChar() != ':') { Seek(position); return(true); } } else if (!PeekIsDigit()) { Seek(position); return(true); } sb.Append(NextChar()); } type = StringTokenType.TimeOffset; token = sb.ToString(); return(true); default: type = StringTokenType.Time; token = sb.ToString(); return(true); } }
public bool TryReadDate(out string token, out StringTokenType type) { token = null; type = StringTokenType.Char; if (PeekChar(4) != '-' || PeekChar(7) != '-') { return(false); } var position = Position; var sb = new StringBuilder(); for (var i = 0; i < 10; i++) { if (i != 4 && i != 7 && !PeekIsDigit()) { Seek(position); return(false); } sb.Append(NextChar()); } switch (PeekChar()) { case 'Z': // UTC Date : 2002-09-24Z sb.Append(NextChar()); type = StringTokenType.DateUtc; token = sb.ToString(); return(true); case '+': case '-': position = Position; token = sb.ToString(); type = StringTokenType.Date; // Date offset : 2002-09-24+06:00 2002-09-24+06:00 sb.Append(NextChar()); for (var i = 0; i < 5; i++) { // read offset if (i == 2) { if (PeekChar() != ':') { Seek(position); return(true); } } else if (!PeekIsDigit()) { Seek(position); return(true); } sb.Append(NextChar()); } type = StringTokenType.DateOffset; token = sb.ToString(); return(true); case 'T': // DateTime : 2002-09-24T06:00:00 2002-09-24T06:00:00.100 // Or DateTimeOffset : sb.Append(NextChar()); string timeToken; StringTokenType timeType; if (!TryReadTime(out timeToken, out timeType)) { Seek(position); return(false); } sb.Append(timeToken); switch (timeType) { case StringTokenType.Time: type = StringTokenType.DateTime; break; case StringTokenType.TimeUtc: type = StringTokenType.DateTimeUtc; break; case StringTokenType.TimeOffset: type = StringTokenType.DateTimeOffset; break; default: Seek(position); return(false); } token = sb.ToString(); return(true); default: type = StringTokenType.Date; token = sb.ToString(); return(true); } }
public bool TryReadNumeric(out string token, out StringTokenType type) { token = null; type = StringTokenType.Char; if (!PeekIsDigit()) { return(false); } var dotCount = 0; var expCount = 0; var sb = new StringBuilder(); while (true) { var peek = PeekChar(); if (char.IsDigit(peek)) { sb.Append(NextChar()); continue; } switch (peek) { case 'e': case 'E': if (expCount++ > 0) { token = sb.ToString(); type = StringTokenType.Double; return(true); } sb.Append(NextChar()); if (PeekChar() == '-') { sb.Append(NextChar()); } continue; case '.': if (expCount > 0 || dotCount++ > 0) { token = sb.ToString(); type = StringTokenType.Double; return(true); } sb.Append(NextChar()); continue; case 'm': case 'M': NextChar(); token = sb.ToString(); type = StringTokenType.Decimal; return(true); case 'f': case 'F': // no protocol NextChar(); token = sb.ToString(); type = StringTokenType.Single; return(true); case 'd': case 'D': // no protocol NextChar(); token = sb.ToString(); type = StringTokenType.Double; return(true); case 'L': NextChar(); token = sb.ToString(); type = StringTokenType.Int64; return(true); default: token = sb.ToString(); type = dotCount > 0 || expCount > 0 ? StringTokenType.Double : StringTokenType.Int32; return(true); } } }