public void EncodingPerformanceTestUsingCorefxlab(int charLength, int minCodePoint, int maxCodePoint) { string unicodeString = GenerateString(charLength, minCodePoint, maxCodePoint); ReadOnlySpan <char> characters = unicodeString.Slice(); int consumed; int encodedBytes; Assert.True(Utf8Encoder.TryComputeEncodedBytes(characters, out encodedBytes)); byte[] utf8Buffer = new byte[encodedBytes]; Span <byte> span = new Span <byte>(utf8Buffer); foreach (var iteration in Benchmark.Iterations) { using (iteration.StartMeasurement()) if (!Utf8Encoder.TryEncode(characters, span, out consumed, out encodedBytes)) { throw new Exception(); // this should not happen } } }
private static void AppendStringInternal(this IWriter writer, string value, int startIndex, int endIndex) { for (var i = startIndex; i <= endIndex;) { UnicodeCodePoint codePoint; int encodedChars; var success = Utf16LittleEndianEncoder.TryDecodeCodePointFromString(value, i, out codePoint, out encodedChars); if (!success) { throw new ArgumentException(); } i += encodedChars; int encodedBytes; success = Utf8Encoder.TryEncodeCodePoint( codePoint, writer.GetFreeBuffer(MaxUtf8CodePointBytes).ToSpan(), out encodedBytes); Debug.Assert(success); writer.CommitBytes(encodedBytes); } }
public Utf8EncoderTests() { _stringData = "random_string"; _byteData = _stringData.GetBytes(); encoder = new Utf8Encoder(Protocol.MsgPack); }
public override bool TryDecode(ReadOnlySpan <byte> encodedBytes, Span <byte> utf8, out int bytesConsumed, out int bytesWritten) { var utf16 = encodedBytes.NonPortableCast <byte, char>(); return(Utf8Encoder.TryEncode(utf16, utf8, out bytesConsumed, out bytesWritten)); }
public override bool TryDecode(ReadOnlySpan <byte> encodedBytes, Span <uint> utf32, out int bytesConsumed, out int charactersWritten) { return(Utf8Encoder.TryDecode(encodedBytes, utf32, out bytesConsumed, out charactersWritten)); }
public override bool TryEncode(ReadOnlySpan <uint> utf32, Span <byte> data, out int charactersConsumed, out int bytesWritten) { return(Utf8Encoder.TryEncode(utf32, data, out charactersConsumed, out bytesWritten)); }
public Utf8EncoderTests() { _stringData = "random_string"; _byteData = _stringData.GetBytes(); encoder = new Utf8Encoder(Defaults.Protocol); }
public override bool TryComputeEncodedBytes(ReadOnlySpan <uint> utf32, out int bytesNeeded) => Utf8Encoder.TryComputeEncodedBytes(utf32, out bytesNeeded);
public static StringValue decodeEncode(Env env, StringValue str, string inCharset, string outCharset, string replacement, bool isIgnoreErrors) { bool isStartUtf8 = false; bool isEndUtf8 = false; if (inCharset.equalsIgnoreCase("utf8") || inCharset.equalsIgnoreCase("utf-8")) { isStartUtf8 = true; } if (outCharset.equalsIgnoreCase("utf8") || outCharset.equalsIgnoreCase("utf-8")) { isEndUtf8 = true; } if (isStartUtf8 && isEndUtf8) { return(UnicodeUtility.utf8Clean(env, str, null, isIgnoreErrors)); } // decode phase CharSequence unicodeStr; Decoder decoder; if (isStartUtf8) { decoder = new Utf8Decoder(inCharset); } else { decoder = new GenericDecoder(inCharset); } decoder.setIgnoreErrors(isIgnoreErrors); unicodeStr = decoder.decode(env, str); // encode phase Encoder encoder; if (isEndUtf8) { encoder = new Utf8Encoder(); } else { encoder = Encoder.create(outCharset); } encoder.setIgnoreErrors(isIgnoreErrors); StringValue sb = env.createBinaryBuilder(); return(encoder.encode(sb, unicodeStr)); }
protected Utf8EncoderTests() { _stringData = "random_string"; _byteData = _stringData.GetBytes(); _encoder = new Utf8Encoder(); }
public override bool TryEncode(ReadOnlySpan <uint> utf32, Span <byte> encodedBytes, out int codeUnitsConsumed, out int bytesWritten) { return(Utf8Encoder.TryEncode(utf32, encodedBytes, out codeUnitsConsumed, out bytesWritten)); }
public override bool TryEncode(ReadOnlySpan <char> utf16, Span <byte> encodedBytes, out int charactersConsumed, out int bytesWritten) { return(Utf8Encoder.TryEncode(utf16, encodedBytes, out charactersConsumed, out bytesWritten)); }
public static bool TryFormat(this string value, Span <byte> buffer, Format.Parsed format, FormattingData formattingData, out int bytesWritten) { if (formattingData.IsUtf16) { var valueBytes = value.Length << 1; if (valueBytes > buffer.Length) { bytesWritten = 0; return(false); } unsafe { fixed(char *pCharacters = value) { byte *pBytes = (byte *)pCharacters; buffer.Set(pBytes, valueBytes); } } bytesWritten = valueBytes; return(true); } GCHandle handle; var byteSpan = buffer.Pin(out handle); try { var avaliableBytes = byteSpan.Length; bytesWritten = 0; for (int i = 0; i < value.Length; i++) { var c = value[i]; var codepoint = (ushort)c; if (codepoint <= 0x7f) // this if block just optimizes for ascii { if (bytesWritten + 1 > avaliableBytes) { bytesWritten = 0; return(false); } byteSpan[bytesWritten++] = (byte)codepoint; } else { var encoded = new FourBytes(); var bytes = Utf8Encoder.CharToUtf8(c, ref encoded); if (bytesWritten + bytes > avaliableBytes) { bytesWritten = 0; return(false); } byteSpan[bytesWritten] = encoded.B0; if (bytes > 1) { byteSpan[bytesWritten + 1] = encoded.B1; if (bytes > 2) { byteSpan[bytesWritten + 2] = encoded.B2; if (bytes > 3) { byteSpan[bytesWritten + 3] = encoded.B3; } } } bytesWritten += bytes; } } return(true); } finally { handle.Free(); } }
public void Name_ReturnsUtf8() { var encoder = new Utf8Encoder(); encoder.Name.Should().Be("UTF8"); }