public void EncodingPerformanceTestUsingCorefxlab(int charLength, int minCodePoint, int maxCodePoint)
{
string unicodeString = GenerateString(charLength, minCodePoint, maxCodePoint);
ReadOnlySpan <char> characters = unicodeString.Slice();
int consumed;
int encodedBytes;
Assert.True(Utf8Encoder.TryComputeEncodedBytes(characters, out encodedBytes));
byte[] utf8Buffer = new byte[encodedBytes];
Span <byte> span = new Span <byte>(utf8Buffer);
foreach (var iteration in Benchmark.Iterations)
{
using (iteration.StartMeasurement())
if (!Utf8Encoder.TryEncode(characters, span, out consumed, out encodedBytes))
{
throw new Exception(); // this should not happen
}
}
}
private static void AppendStringInternal(this IWriter writer, string value, int startIndex, int endIndex)
{
for (var i = startIndex; i <= endIndex;)
{
UnicodeCodePoint codePoint;
int encodedChars;
var success = Utf16LittleEndianEncoder.TryDecodeCodePointFromString(value, i, out codePoint, out encodedChars);
if (!success)
{
throw new ArgumentException();
}
i += encodedChars;
int encodedBytes;
success = Utf8Encoder.TryEncodeCodePoint(
codePoint,
writer.GetFreeBuffer(MaxUtf8CodePointBytes).ToSpan(),
out encodedBytes);
Debug.Assert(success);
writer.CommitBytes(encodedBytes);
}
}
public Utf8EncoderTests()
{
_stringData = "random_string";
_byteData = _stringData.GetBytes();
encoder = new Utf8Encoder(Protocol.MsgPack);
}
public override bool TryDecode(ReadOnlySpan <byte> encodedBytes, Span <byte> utf8, out int bytesConsumed, out int bytesWritten)
{
var utf16 = encodedBytes.NonPortableCast <byte, char>();
return(Utf8Encoder.TryEncode(utf16, utf8, out bytesConsumed, out bytesWritten));
}
public override bool TryDecode(ReadOnlySpan <byte> encodedBytes, Span <uint> utf32, out int bytesConsumed, out int charactersWritten)
{
return(Utf8Encoder.TryDecode(encodedBytes, utf32, out bytesConsumed, out charactersWritten));
}
public override bool TryEncode(ReadOnlySpan <uint> utf32, Span <byte> data, out int charactersConsumed, out int bytesWritten)
{
return(Utf8Encoder.TryEncode(utf32, data, out charactersConsumed, out bytesWritten));
}
public Utf8EncoderTests()
{
_stringData = "random_string";
_byteData = _stringData.GetBytes();
encoder = new Utf8Encoder(Defaults.Protocol);
}
public override bool TryComputeEncodedBytes(ReadOnlySpan <uint> utf32, out int bytesNeeded) => Utf8Encoder.TryComputeEncodedBytes(utf32, out bytesNeeded);
public static StringValue decodeEncode(Env env,
StringValue str,
string inCharset,
string outCharset,
string replacement,
bool isIgnoreErrors)
{
bool isStartUtf8 = false;
bool isEndUtf8 = false;
if (inCharset.equalsIgnoreCase("utf8") ||
inCharset.equalsIgnoreCase("utf-8"))
{
isStartUtf8 = true;
}
if (outCharset.equalsIgnoreCase("utf8") ||
outCharset.equalsIgnoreCase("utf-8"))
{
isEndUtf8 = true;
}
if (isStartUtf8 && isEndUtf8)
{
return(UnicodeUtility.utf8Clean(env, str, null, isIgnoreErrors));
}
// decode phase
CharSequence unicodeStr;
Decoder decoder;
if (isStartUtf8)
{
decoder = new Utf8Decoder(inCharset);
}
else
{
decoder = new GenericDecoder(inCharset);
}
decoder.setIgnoreErrors(isIgnoreErrors);
unicodeStr = decoder.decode(env, str);
// encode phase
Encoder encoder;
if (isEndUtf8)
{
encoder = new Utf8Encoder();
}
else
{
encoder = Encoder.create(outCharset);
}
encoder.setIgnoreErrors(isIgnoreErrors);
StringValue sb = env.createBinaryBuilder();
return(encoder.encode(sb, unicodeStr));
}
protected Utf8EncoderTests()
{
_stringData = "random_string";
_byteData = _stringData.GetBytes();
_encoder = new Utf8Encoder();
}
public override bool TryEncode(ReadOnlySpan <uint> utf32, Span <byte> encodedBytes, out int codeUnitsConsumed, out int bytesWritten)
{
return(Utf8Encoder.TryEncode(utf32, encodedBytes, out codeUnitsConsumed, out bytesWritten));
}
public override bool TryEncode(ReadOnlySpan <char> utf16, Span <byte> encodedBytes, out int charactersConsumed, out int bytesWritten)
{
return(Utf8Encoder.TryEncode(utf16, encodedBytes, out charactersConsumed, out bytesWritten));
}
public static bool TryFormat(this string value, Span <byte> buffer, Format.Parsed format, FormattingData formattingData, out int bytesWritten)
{
if (formattingData.IsUtf16)
{
var valueBytes = value.Length << 1;
if (valueBytes > buffer.Length)
{
bytesWritten = 0;
return(false);
}
unsafe
{
fixed(char *pCharacters = value)
{
byte *pBytes = (byte *)pCharacters;
buffer.Set(pBytes, valueBytes);
}
}
bytesWritten = valueBytes;
return(true);
}
GCHandle handle;
var byteSpan = buffer.Pin(out handle);
try {
var avaliableBytes = byteSpan.Length;
bytesWritten = 0;
for (int i = 0; i < value.Length; i++)
{
var c = value[i];
var codepoint = (ushort)c;
if (codepoint <= 0x7f) // this if block just optimizes for ascii
{
if (bytesWritten + 1 > avaliableBytes)
{
bytesWritten = 0;
return(false);
}
byteSpan[bytesWritten++] = (byte)codepoint;
}
else
{
var encoded = new FourBytes();
var bytes = Utf8Encoder.CharToUtf8(c, ref encoded);
if (bytesWritten + bytes > avaliableBytes)
{
bytesWritten = 0;
return(false);
}
byteSpan[bytesWritten] = encoded.B0;
if (bytes > 1)
{
byteSpan[bytesWritten + 1] = encoded.B1;
if (bytes > 2)
{
byteSpan[bytesWritten + 2] = encoded.B2;
if (bytes > 3)
{
byteSpan[bytesWritten + 3] = encoded.B3;
}
}
}
bytesWritten += bytes;
}
}
return(true);
}
finally
{
handle.Free();
}
}
public void Name_ReturnsUtf8()
{
var encoder = new Utf8Encoder();
encoder.Name.Should().Be("UTF8");
}
