public static bool TryReadInt64(ref ReadOnlySpan <byte> remaining, out long result, char standardFormat) { result = default; if (standardFormat != '\0') { if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadInt64(ref bytes, out result, standardFormat)); } switch (GetTokenType(ref remaining)) { case EtfTokenType.SmallInteger: { //remaining = remaining.Slice(1); result = remaining[1]; remaining = remaining.Slice(2); return(true); } case EtfTokenType.Integer: { remaining = remaining.Slice(1); result = BinaryPrimitives.ReadInt32BigEndian(remaining); remaining = remaining.Slice(4); return(true); } case EtfTokenType.SmallBig: { //remaining = remaining.Slice(1); byte bytes = remaining[1]; bool isPositive = remaining[2] == 0; remaining = remaining.Slice(3); return(TryReadSignedBigNumber(bytes, isPositive, ref remaining, out result)); } case EtfTokenType.LargeBig: { remaining = remaining.Slice(1); if (!BinaryPrimitives.TryReadUInt32BigEndian(remaining, out uint bytes)) { return(false); } if (bytes > int.MaxValue) { return(false); // TODO: Spans dont allow uint accessors } bool isPositive = remaining[4] == 0; remaining = remaining.Slice(5); return(TryReadSignedBigNumber((int)bytes, isPositive, ref remaining, out result)); } default: return(false); } }
public static bool TryReadUInt8(ref ReadOnlySpan <byte> remaining, out byte result, char standardFormat) { result = default; if (standardFormat != '\0') { if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadUInt8(ref bytes, out result, standardFormat)); } result = default; if (!TryReadUInt64(ref remaining, out ulong longResult, standardFormat)) { return(false); } if (longResult > byte.MaxValue) { return(false); } result = (byte)longResult; return(true); }
public static bool TryReadInt32(ref ReadOnlySpan <byte> remaining, out int result, char standardFormat) { result = default; if (standardFormat != '\0') { if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadInt32(ref bytes, out result, standardFormat)); } result = default; if (!TryReadInt64(ref remaining, out long longResult, standardFormat)) { return(false); } if (longResult > int.MaxValue || longResult < int.MinValue) { return(false); } result = (int)longResult; return(true); }
public static bool TryReadUInt64(ref ReadOnlySpan <byte> remaining, out ulong result, char standardFormat) { result = default; switch (GetTokenType(ref remaining)) { case JsonTokenType.Number: if (!Utf8Reader.TryReadUInt64(ref remaining, out result, JsonSerializer.IntFormat.Symbol)) { return(false); } return(true); case JsonTokenType.String: remaining = remaining.Slice(1); if (!Utf8Reader.TryReadUInt64(ref remaining, out result, standardFormat)) { return(false); } if (remaining.Length == 0 || remaining[0] != '"') { return(false); } remaining = remaining.Slice(1); return(true); } return(false); }
// Detects a Unicode encoding private int DetectUnicodeEncoding() { int mode = this.mode; int c1 = this.stream.ReadByte(); int c2; if (c1 < 0) { return(-1); } Utf8Reader utf8reader; switch (mode) { case 0: // UTF-8 only utf8reader = new Utf8Reader(this.stream, this.errorThrow); this.reader = utf8reader; utf8reader.Unget(c1); c1 = utf8reader.ReadChar(); if (c1 == 0xfeff && !this.dontSkipUtf8Bom) { // Skip BOM c1 = utf8reader.ReadChar(); } return(c1); case 1: case 3: c2 = this.DetectUtf8OrUtf16(c1); if (c2 >= -1) { return(c2); } break; case 2: case 4: // UTF-8, UTF-16, or UTF-32 c2 = this.DetectUtf8Or16Or32(c1); if (c2 >= -1) { return(c2); } break; } // Default case: assume UTF-8 utf8reader = new Utf8Reader(this.stream, this.errorThrow); this.reader = utf8reader; utf8reader.Unget(c1); c1 = utf8reader.ReadChar(); if (!this.dontSkipUtf8Bom && c1 == 0xfeff) { // Skip BOM c1 = utf8reader.ReadChar(); } return(c1); }
public static bool TryReadChar(ref ReadOnlySpan <byte> remaining, out char result) { result = default; if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadChar(ref bytes, out result)); }
public static bool TryReadDecimal(ref ReadOnlySpan <byte> remaining, out decimal result, char standardFormat) { result = default; if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadDecimal(ref bytes, out result, standardFormat)); }
// Detects a Unicode encoding private int DetectUnicodeEncoding() { int mode = this.mode; int c1 = this.stream.ReadByte(); int c2; if (c1 < 0) { return(-1); } Utf8Reader utf8reader; if (mode == 0) { // UTF-8 only utf8reader = new Utf8Reader(this.stream, this.errorThrow); this.reader = utf8reader; c1 = utf8reader.ReadChar(); if (c1 == 0xfeff) { // Skip BOM c1 = utf8reader.ReadChar(); } return(c1); } else if (mode == 1 || mode == 3) { c2 = this.DetectUtf8OrUtf16(c1); if (c2 >= -1) { return(c2); } } else if (mode == 2 || mode == 4) { // UTF-8, UTF-16, or UTF-32 c2 = this.DetectUtf8Or16Or32(c1); if (c2 >= -1) { return(c2); } } // Default case: assume UTF-8 utf8reader = new Utf8Reader(this.stream, this.errorThrow); this.reader = utf8reader; utf8reader.Unget(c1); c1 = utf8reader.ReadChar(); if (!this.dontSkipUtf8Bom && c1 == 0xfeff) { // Skip BOM c1 = utf8reader.ReadChar(); } return(c1); }
public static bool TryReadSingle(ref ReadOnlySpan <byte> remaining, out float result, char standardFormat) { result = default; if (standardFormat != '\0') { if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadSingle(ref bytes, out result, standardFormat)); } switch (GetTokenType(ref remaining)) { case EtfTokenType.Float: // TODO: Untested var bytes = remaining.Slice(1, 31); remaining.Slice(32); return(Utf8Reader.TryReadSingle(ref bytes, out result, 'g')); case EtfTokenType.NewFloat: { // TODO: Untested, does Discord have any endpoints that accept floats? if (remaining.Length < 8) { return(false); } remaining = remaining.Slice(1); Span <double> dst = stackalloc double[1]; var dstBytes = MemoryMarshal.AsBytes(dst); // Swap endian dstBytes[0] = remaining[7]; dstBytes[1] = remaining[6]; dstBytes[2] = remaining[5]; dstBytes[3] = remaining[4]; dstBytes[4] = remaining[3]; dstBytes[5] = remaining[2]; dstBytes[6] = remaining[1]; dstBytes[7] = remaining[0]; result = (float)dst[0]; remaining = remaining.Slice(8); return(true); } default: return(false); } }
public static bool TryReadDateTime(ref ReadOnlySpan <byte> remaining, out DateTime result, char standardFormat) { result = default; switch (GetTokenType(ref remaining)) { case JsonTokenType.String: remaining = remaining.Slice(1); if (!Utf8Reader.TryReadDateTime(ref remaining, out result, standardFormat)) { return(false); } if (remaining.Length == 0 || remaining[0] != '"') { return(false); } remaining = remaining.Slice(1); return(true); } return(false); }
public override bool isDecodable(Env env, StringValue str) { if (str.isUnicode()) { return(true); } Utf8Reader reader = new Utf8Reader(str); int ch; while ((ch = reader.read()) >= 0) { if (ch == ERROR_CHARACTER) { return(false); } } return(true); }
public static bool TryReadBoolean(ref ReadOnlySpan <byte> remaining, out bool result, char standardFormat) { result = default; switch (GetTokenType(ref remaining)) { case JsonTokenType.True: if (remaining.Length < 4) { return(false); } result = true; remaining = remaining.Slice(4); return(true); case JsonTokenType.False: if (remaining.Length < 5) { return(false); } result = false; remaining = remaining.Slice(5); return(true); case JsonTokenType.String: remaining = remaining.Slice(1); if (!Utf8Reader.TryReadBoolean(ref remaining, out result, standardFormat)) { return(false); } if (remaining.Length == 0 || remaining[0] != '"') { return(false); } remaining = remaining.Slice(1); return(true); } return(false); }
// Detects a Unicode encoding private int DetectUnicodeEncoding() { int mode = this.mode; int c1 = this.stream.ReadByte(); int c2; if (c1 < 0) { return -1; } Utf8Reader utf8reader; if (mode == 0) { // UTF-8 only utf8reader = new Utf8Reader(this.stream, this.errorThrow); this.reader = utf8reader; c1 = utf8reader.ReadChar(); if (c1 == 0xfeff) { // Skip BOM c1 = utf8reader.ReadChar(); } return c1; } else if (mode == 1 || mode == 3) { c2 = this.DetectUtf8OrUtf16(c1); if (c2 >= -1) { return c2; } } else if (mode == 2 || mode == 4) { // UTF-8, UTF-16, or UTF-32 c2 = this.DetectUtf8Or16Or32(c1); if (c2 >= -1) { return c2; } } // Default case: assume UTF-8 utf8reader = new Utf8Reader(this.stream, this.errorThrow); this.reader = utf8reader; utf8reader.Unget(c1); c1 = utf8reader.ReadChar(); if (!this.dontSkipUtf8Bom && c1 == 0xfeff) { // Skip BOM c1 = utf8reader.ReadChar(); } return c1; }
private int DetectUtf8OrUtf16(int c1) { int mode = this.mode; int c2; if (c1 == 0xff || c1 == 0xfe) { c2 = this.stream.ReadByte(); bool bigEndian = c1 == 0xfe; int otherbyte = bigEndian ? 0xff : 0xfe; if (c2 == otherbyte) { var newReader = new Utf16Reader( this.stream, bigEndian, this.errorThrow); this.reader = newReader; return(newReader.ReadChar()); } // Assume UTF-8 here, so the 0xff or 0xfe is invalid if (this.errorThrow) { throw new InvalidOperationException("Invalid Unicode stream"); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return(0xfffd); } } else if (mode == 1) { if (c1 >= 0x01 && c1 <= 0x7f) { // Nonzero ASCII character c2 = this.stream.ReadByte(); if (c2 == 0) { // NZA 0, so UTF-16LE var newReader = new Utf16Reader( this.stream, false, this.errorThrow); this.reader = newReader; } else { // NZA NZ var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; } return(c1); } else if (c1 == 0) { // Zero c2 = this.stream.ReadByte(); if (c2 >= 0x01 && c2 <= 0x7f) { // 0 NZA, so UTF-16BE var newReader = new Utf16Reader(this.stream, true, this.errorThrow); this.reader = newReader; return(c2); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return(c1); } } } // Use default of UTF-8 return(-2); }
private int DetectUtf8Or16Or32(int c1) { int c2, c3, c4; if (c1 == 0xff || c1 == 0xfe) { // Start of a possible byte-order mark // FF FE 0 0 --> UTF-32LE // FF FE ... --> UTF-16LE // FE FF --> UTF-16BE c2 = this.stream.ReadByte(); bool bigEndian = c1 == 0xfe; int otherbyte = bigEndian ? 0xff : 0xfe; if (c2 == otherbyte) { c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (!bigEndian && c3 == 0 && c4 == 0) { this.reader = new Utf32Reader(this.stream, false, this.errorThrow); return(this.reader.ReadChar()); } else { var newReader = new Utf16Reader( this.stream, bigEndian, this.errorThrow); newReader.Unget(c3, c4); this.reader = newReader; return(newReader.ReadChar()); } } // Assume UTF-8 here, so the 0xff or 0xfe is invalid if (this.errorThrow) { throw new InvalidOperationException("Invalid Unicode stream"); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return(0xfffd); } } else if (c1 == 0 && this.mode == 4) { // Here, the relevant cases are: // 0 0 0 NZA --> UTF-32BE (if mode is 4) // 0 0 FE FF --> UTF-32BE // Anything else is treated as UTF-8 c2 = this.stream.ReadByte(); c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (c2 == 0 && ((c3 == 0xfe && c4 == 0xff) || (c3 == 0 && c4 >= 0x01 && c4 <= 0x7f))) { this.reader = new Utf32Reader(this.stream, true, this.errorThrow); return(c3 == 0 ? c4 : this.reader.ReadChar()); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.UngetThree(c2, c3, c4); this.reader = utf8reader; return(c1); } } else if (this.mode == 2) { if (c1 >= 0x01 && c1 <= 0x7f) { // Nonzero ASCII character c2 = this.stream.ReadByte(); if (c2 == 0) { // NZA 0, so UTF-16LE or UTF-32LE c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (c3 == 0 && c4 == 0) { this.reader = new Utf32Reader( this.stream, false, this.errorThrow); return(c1); } else { var newReader = new Utf16Reader( this.stream, false, this.errorThrow); newReader.Unget(c3, c4); this.reader = newReader; return(c1); } } else { // NZA NZ, so UTF-8 var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return(c1); } } else if (c1 == 0) { // Zero c2 = this.stream.ReadByte(); if (c2 >= 0x01 && c2 <= 0x7f) { // 0 NZA, so UTF-16BE var newReader = new Utf16Reader(this.stream, true, this.errorThrow); this.reader = newReader; return(c2); } else if (c2 == 0) { // 0 0, so maybe UTF-32BE c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (c3 == 0 && c4 >= 0x01 && c4 <= 0x7f) { // 0 0 0 NZA this.reader = new Utf32Reader(this.stream, true, this.errorThrow); return(c4); } else if (c3 == 0xfe && c4 == 0xff) { // 0 0 FE FF this.reader = new Utf32Reader(this.stream, true, this.errorThrow); return(this.reader.ReadChar()); } else { // 0 0 ... var newReader = new Utf8Reader(this.stream, this.errorThrow); newReader.UngetThree(c2, c3, c4); this.reader = newReader; return(c1); } } else { // 0 NonAscii, so UTF-8 var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return(c1); } } } // Use default of UTF-8 return(-2); }
public static bool TryReadBoolean(ref ReadOnlySpan <byte> remaining, out bool result, char standardFormat) { result = default; if (standardFormat != '\0') { if (!TryReadUtf8Bytes(ref remaining, out var bytes)) { return(false); } return(Utf8Reader.TryReadBoolean(ref bytes, out result, standardFormat)); } switch (GetTokenType(ref remaining)) { case EtfTokenType.SmallAtom: case EtfTokenType.SmallAtomUtf8: { if (remaining.Length < 2) { return(false); } //remaining = remaining.Slice(1); byte length = remaining[1]; //remaining = remaining.Slice(1); switch (length) { case 4: remaining = remaining.Slice(6); result = true; return(true); case 5: remaining = remaining.Slice(7); result = false; return(true); default: return(false); } } case EtfTokenType.Atom: case EtfTokenType.AtomUtf8: { if (remaining.Length < 3) { return(false); } remaining = remaining.Slice(1); ushort length = BinaryPrimitives.ReadUInt16BigEndian(remaining); //remaining = remaining.Slice(2); switch (length) { case 4: remaining = remaining.Slice(6); result = true; return(true); case 5: remaining = remaining.Slice(7); result = false; return(true); default: return(false); } } default: return(false); } }
private int DetectUtf8Or16Or32(int c1) { int c2, c3, c4; if (c1 == 0xff || c1 == 0xfe) { // Start of a possible byte-order mark // FF FE 0 0 --> UTF-32LE // FF FE ... --> UTF-16LE // FE FF --> UTF-16BE c2 = this.stream.ReadByte(); bool bigEndian = c1 == 0xfe; int otherbyte = bigEndian ? 0xff : 0xfe; if (c2 == otherbyte) { c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (!bigEndian && c3 == 0 && c4 == 0) { this.reader = new Utf32Reader(this.stream, false, this.errorThrow); return this.reader.ReadChar(); } else { var newReader = new Utf16Reader( this.stream, bigEndian, this.errorThrow); newReader.Unget(c3, c4); this.reader = newReader; return newReader.ReadChar(); } } // Assume UTF-8 here, so the 0xff or 0xfe is invalid if (this.errorThrow) { throw new InvalidOperationException("Invalid Unicode stream"); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return 0xfffd; } } else if (c1 == 0 && this.mode == 4) { // Here, the relevant cases are: // 0 0 0 NZA --> UTF-32BE (if mode is 4) // 0 0 FE FF --> UTF-32BE // Anything else is treated as UTF-8 c2 = this.stream.ReadByte(); c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (c2 == 0 && ((c3 == 0xfe && c4 == 0xff) || (c3 == 0 && c4 >= 0x01 && c4 <= 0x7f))) { this.reader = new Utf32Reader(this.stream, true, this.errorThrow); return c3 == 0 ? c4 : this.reader.ReadChar(); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.UngetThree(c2, c3, c4); this.reader = utf8reader; return c1; } } else if (this.mode == 2) { if (c1 >= 0x01 && c1 <= 0x7f) { // Nonzero ASCII character c2 = this.stream.ReadByte(); if (c2 == 0) { // NZA 0, so UTF-16LE or UTF-32LE c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (c3 == 0 && c4 == 0) { this.reader = new Utf32Reader( this.stream, false, this.errorThrow); return c1; } else { var newReader = new Utf16Reader( this.stream, false, this.errorThrow); newReader.Unget(c3, c4); this.reader = newReader; return c1; } } else { // NZA NZ, so UTF-8 var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return c1; } } else if (c1 == 0) { // Zero c2 = this.stream.ReadByte(); if (c2 >= 0x01 && c2 <= 0x7f) { // 0 NZA, so UTF-16BE var newReader = new Utf16Reader(this.stream, true, this.errorThrow); this.reader = newReader; return c2; } else if (c2 == 0) { // 0 0, so maybe UTF-32BE c3 = this.stream.ReadByte(); c4 = this.stream.ReadByte(); if (c3 == 0 && c4 >= 0x01 && c4 <= 0x7f) { // 0 0 0 NZA this.reader = new Utf32Reader(this.stream, true, this.errorThrow); return c4; } else if (c3 == 0xfe && c4 == 0xff) { // 0 0 FE FF this.reader = new Utf32Reader(this.stream, true, this.errorThrow); return this.reader.ReadChar(); } else { // 0 0 ... var newReader = new Utf8Reader(this.stream, this.errorThrow); newReader.UngetThree(c2, c3, c4); this.reader = newReader; return c1; } } else { // 0 NonAscii, so UTF-8 var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return c1; } } } // Use default of UTF-8 return -2; }
private int DetectUtf8OrUtf16(int c1) { int mode = this.mode; int c2; if (c1 == 0xff || c1 == 0xfe) { c2 = this.stream.ReadByte(); bool bigEndian = c1 == 0xfe; int otherbyte = bigEndian ? 0xff : 0xfe; if (c2 == otherbyte) { var newReader = new Utf16Reader( this.stream, bigEndian, this.errorThrow); this.reader = newReader; return newReader.ReadChar(); } // Assume UTF-8 here, so the 0xff or 0xfe is invalid if (this.errorThrow) { throw new InvalidOperationException("Invalid Unicode stream"); } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return 0xfffd; } } else if (mode == 1) { if (c1 >= 0x01 && c1 <= 0x7f) { // Nonzero ASCII character c2 = this.stream.ReadByte(); if (c2 == 0) { // NZA 0, so UTF-16LE var newReader = new Utf16Reader( this.stream, false, this.errorThrow); this.reader = newReader; } else { // NZA NZ var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; } return c1; } else if (c1 == 0) { // Zero c2 = this.stream.ReadByte(); if (c2 >= 0x01 && c2 <= 0x7f) { // 0 NZA, so UTF-16BE var newReader = new Utf16Reader(this.stream, true, this.errorThrow); this.reader = newReader; return c2; } else { var utf8reader = new Utf8Reader(this.stream, this.errorThrow); utf8reader.Unget(c2); this.reader = utf8reader; return c1; } } } // Use default of UTF-8 return -2; }