/// <summary>
/// Initializes a new EncodingData instance for the specified encoding.
/// </summary>
/// <param name="encoding">Encoding to encapsulate.</param>
/// <returns>A new EncodingData instance.</returns>
private static EncodingData ForEncoding(Encoding encoding)
{
Debug.Assert(encoding != null, "encoding != null");
EncodingData result = new EncodingData();
result.encoding = encoding;
result.name = encoding.WebName;
return result;
}
public unsafe static bool TryParseSingle(byte* text, int index, int length, EncodingData encoding,
TextFormat format, out float value, out int bytesConsumed)
{
// Precondition replacement
if (length < 1 || index < 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
value = 0f;
bytesConsumed = 0;
if (encoding.IsInvariantUtf8)
{
string floatString = "";
bool decimalPlace = false, e = false, signed = false, digitLast = false, eLast = false;
if ((length) >= 3 && text[index] == 'N' && text[index + 1] == 'a' && text[index + 2] == 'N')
{
value = float.NaN;
bytesConsumed = 3;
return true;
}
if (text[index] == '-' || text[index] == '+')
{
signed = true;
floatString += (char)text[index];
index++;
bytesConsumed++;
}
if ((length - index) >= 8 && text[index] == 'I' && text[index + 1] == 'n' &&
text[index + 2] == 'f' && text[index + 3] == 'i' && text[index + 4] == 'n' &&
text[index + 5] == 'i' && text[index + 6] == 't' && text[index + 7] == 'y')
{
if (signed && text[index - 1] == '-')
{
value = float.NegativeInfinity;
}
else
{
value = float.PositiveInfinity;
}
bytesConsumed += 8;
return true;
}
for (int byteIndex = index; byteIndex < length; byteIndex++)
{
byte nextByte = text[byteIndex];
byte nextByteVal = (byte)(nextByte - '0');
if (nextByteVal > 9)
{
if (!decimalPlace && nextByte == '.')
{
if (digitLast)
{
digitLast = false;
}
if (eLast)
{
eLast = false;
}
bytesConsumed++;
decimalPlace = true;
floatString += (char)nextByte;
}
else if (!e && nextByte == 'e' || nextByte == 'E')
{
e = true;
eLast = true;
bytesConsumed++;
floatString += (char)nextByte;
}
else if (eLast && nextByte == '+' || nextByte == '-')
{
eLast = false;
bytesConsumed++;
floatString += (char)nextByte;
}
else if ((decimalPlace && signed && bytesConsumed == 2) || ((signed || decimalPlace) && bytesConsumed == 1))
{
value = 0;
bytesConsumed = 0;
return false;
}
else
{
if (float.TryParse(floatString, out value))
{
return true;
}
else
{
bytesConsumed = 0;
return false;
}
}
}
else
{
if (eLast)
eLast = false;
if (!digitLast)
digitLast = true;
bytesConsumed++;
floatString += (char)nextByte;
}
}
if ((decimalPlace && signed && bytesConsumed == 2) || ((signed || decimalPlace) && bytesConsumed == 1))
{
value = 0;
bytesConsumed = 0;
return false;
}
else
{
if (float.TryParse(floatString, out value))
{
return true;
}
else
{
bytesConsumed = 0;
return false;
}
}
}
return false;
}
/// <summary>
/// Gets the encoding of the recent file.
/// </summary>
/// <param name="recentFile">The <see cref="RecentFile"/> instance.</param>
public static Encoding GetEncoding(this RecentFile recentFile)
{
return(EncodingData.EncodingFromString(recentFile.EncodingAsString) ?? Encoding.UTF8);
}
public HttpHeaderBuffer(Span <byte> bytes, EncodingData encoding)
{
_bytes = bytes;
_encoding = encoding;
}
public static bool TryParseBoolean(ReadOnlySpan<byte> text, EncodingData encoding, TextFormat format, out bool value, out int bytesConsumed)
{
bytesConsumed = 0;
value = default(bool);
if (text.Length < 1) {
return false;
}
if (encoding.IsInvariantUtf8) {
byte firstCodeUnit = text[0];
if (firstCodeUnit == '1') {
bytesConsumed = 1;
value = true;
return true;
}
else if (firstCodeUnit == '0') {
bytesConsumed = 1;
value = false;
return true;
}
else if (PrimitiveParser.IsTrue(text)) {
bytesConsumed = 4;
value = true;
return true;
}
else if (PrimitiveParser.IsFalse(text)) {
bytesConsumed = 5;
value = false;
return true;
}
else {
return false;
}
}
return false;
}
public void EncodingFromAcceptCharsetTest()
{
// It takes over a minute to run all combinations.
CombinatorialEngine engine = CombinatorialEngine.FromDimensions(
new Dimension("EncodingData", EncodingData.Values),
new Dimension("StringData", StringData.Values),
new Dimension("SerializationFormatData", SerializationFormatData.StructuredValues),
new Dimension("WebServerLocation", new object[] { WebServerLocation.InProcess }));
engine.Mode = CombinatorialEngineMode.EveryElement;
TestUtil.RunCombinatorialEngineFail(engine, delegate(Hashtable table)
{
EncodingData encodingData = (EncodingData)table["EncodingData"];
StringData stringData = (StringData)table["StringData"];
WebServerLocation location = (WebServerLocation)table["WebServerLocation"];
SerializationFormatData format = (SerializationFormatData)table["SerializationFormatData"];
if (encodingData.Encoding == null)
{
return;
}
if (!EncodingHandlesString(encodingData.Encoding, "<>#&;\r\n"))
{
return;
}
// Transliteration of ISCII characters and Unicode is possible, but round-tripping from
// Unicode will not work because all phonetic sounds share an ISCII value, but have
// distinct Unicode points depending on the language.
if (encodingData.Name.StartsWith("x-iscii") && stringData.TextScript != null && stringData.TextScript.SupportsIscii)
{
return;
}
using (CustomDataContext.CreateChangeScope())
using (TestWebRequest request = TestWebRequest.CreateForLocation(location))
{
request.DataServiceType = typeof(CustomDataContext);
request.HttpMethod = "POST";
request.RequestUriString = "/Customers";
request.RequestContentType = UnitTestsUtil.JsonLightMimeType;
request.SetRequestStreamAsText("{ " +
"@odata.type : \"AstoriaUnitTests.Stubs.Customer\" ," +
"ID: 100," +
"Name: " +
System.Data.Test.Astoria.Util.JsonPrimitiveTypesUtil.PrimitiveToString(stringData.Value, typeof(string)) +
" }");
request.SendRequest();
request.HttpMethod = "GET";
request.AcceptCharset = encodingData.Name;
request.Accept = format.MimeTypes[0];
request.RequestUriString = "/Customers(100)";
bool encoderCanHandleData = EncodingHandlesString(encodingData.Encoding, stringData.Value);
Trace.WriteLine("Encoding handles string: " + encoderCanHandleData);
Exception exception = TestUtil.RunCatching(request.SendRequest);
XmlDocument document = null;
Stream byteStream = new MemoryStream();
if (exception == null)
{
using (Stream stream = request.GetResponseStream())
{
IOUtil.CopyStream(stream, byteStream);
}
byteStream.Position = 0;
Trace.WriteLine(TestUtil.BuildHexDump(byteStream));
byteStream.Position = 0;
if (format == SerializationFormatData.Atom)
{
document = new XmlDocument(TestUtil.TestNameTable);
using (StreamReader reader = new StreamReader(byteStream, encodingData.Encoding))
{
document.Load(reader);
}
TestUtil.TraceXml(document);
XmlElement nameElement = TestUtil.AssertSelectSingleElement(document, "//ads:Name");
if (stringData.Value == null)
{
Assert.IsTrue(UnitTestsUtil.HasElementNullValue(nameElement,
new System.Net.Mime.ContentType(request.ResponseContentType).MediaType));
}
else
{
Assert.AreEqual(stringData.Value, nameElement.InnerText);
}
}
}
else
{
TestUtil.AssertExceptionExpected(exception, !encoderCanHandleData);
}
}
});
}
public static SequenceFormatter <TSequence> CreateFormatter <TSequence>(this TSequence sequence, EncodingData encoding = default(EncodingData)) where TSequence : ISequence <Memory <byte> >
{
return(new SequenceFormatter <TSequence>(sequence, encoding));
}
public static bool TryParseUInt32(ReadOnlySpan<byte> text, EncodingData encoding, TextFormat numericFormat,
out uint value, out int bytesConsumed)
{
// Precondition replacement
if (text.Length < 1)
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = default(uint);
bytesConsumed = 0;
if (encoding.IsInvariantUtf8)
{
for (int byteIndex = 0; byteIndex < text.Length; byteIndex++) // loop through the byte array
{
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9) // if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
return true; // otherwise return true
}
}
else if (value > UInt32.MaxValue / 10)
{
value = default(uint);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 6 because the max values for unsigned types all end in 5s.
else if (value == UInt32.MaxValue / 10 && nextByteVal >= 6) // overflow
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = (uint)(value * 10 + nextByteVal); // left shift the value and add the nextByte
bytesConsumed++;
}
return true;
}
else if (encoding.IsInvariantUtf16)
{
for (int byteIndex = 0; byteIndex < text.Length - 1; byteIndex += 2) // loop through the byte array two bytes at a time for UTF-16
{
byte byteAfterNext = text[byteIndex + 1];
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9 || byteAfterNext != 0) // if the second byte isn't zero, this isn't an ASCII-equivalent code unit and we can quit here
// if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
return true; // otherwise return true
}
}
else if (value > UInt32.MaxValue / 10)
{
value = default(uint);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 6 because the max values for unsigned types all end in 5s.
else if (value == UInt32.MaxValue / 10 && nextByteVal >= 6) // overflow
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = (uint)(value * 10 + nextByteVal); // left shift the value and add the nextByte
bytesConsumed += 2;
}
return true;
}
else
{
int byteIndex = 0;
while (byteIndex < text.Length)
{
uint result;
int consumed;
bool success = encoding.TryParseSymbol(text.Slice(byteIndex), out result, out consumed);
if (!success || result > 9)
{
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
return true; // otherwise return true
}
}
else if (value > UInt32.MaxValue / 10)
{
value = default(uint);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 6 because the max values for unsigned types all end in 5s.
else if (value == UInt32.MaxValue / 10 && result >= 6) // overflow
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = (uint)(value * 10 + result); // left shift the value and add the nextByte
bytesConsumed += consumed;
}
return true;
}
}
public PipelineTextOutput(IPipeWriter writer, EncodingData encoding)
{
_writer = writer;
Encoding = encoding;
}
public static PipelineTextOutput AsTextOutput(this IPipeWriter writer, EncodingData formattingData)
{
return(new PipelineTextOutput(writer, formattingData));
}
/// <summary>Initializes a new dummy EncodingData instance.</summary>
/// <param name="name">Encoding name.</param>
/// <returns>A new EncodingData instance.</returns>
private static EncodingData ForDummy(string name)
{
EncodingData result = new EncodingData();
result.name = name;
return result;
}
public static void Append <TFormatter>(this TFormatter formatter, char value, EncodingData encoding) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, encoding))
{
formatter.Enlarge();
}
}
public static bool TryAppend <TFormatter>(this TFormatter formatter, ReadOnlySpan <char> value, EncodingData encoding) where TFormatter : IOutput
{
int bytesWritten;
if (!encoding.TextEncoder.TryEncodeFromUtf16(value, formatter.Buffer, out bytesWritten))
{
return(false);
}
formatter.Advance(bytesWritten);
return(true);
}
public static bool TryParseBoolean(byte[] text, int index, TextFormat format, EncodingData encoding,
out bool value, out int bytesConsumed)
{
bytesConsumed = 0;
value = default(bool);
if (text.Length < 1 || index < 0 || index >= text.Length)
{
return(false);
}
if (encoding.IsInvariantUtf8)
{
byte firstByte = text[index];
if (firstByte == '1')
{
bytesConsumed = 1;
value = true;
return(true);
}
else if (firstByte == '0')
{
bytesConsumed = 1;
value = false;
return(true);
}
else if (IsTrue(text, index))
{
bytesConsumed = 4;
value = true;
return(true);
}
else if (IsFalse(text, index))
{
bytesConsumed = 5;
value = false;
return(true);
}
else
{
return(false);
}
}
return(false);
}
public static void Append <TFormatter>(this TFormatter formatter, double value, EncodingData encoding, TextFormat format = default(TextFormat)) where TFormatter : IOutput
{
while (!formatter.TryAppend(value, encoding, format))
{
formatter.Enlarge();
}
}
public bool TryFormat(Span <byte> buffer, out int bytesWritten, TextFormat format, EncodingData encoding)
{
if (!PrimitiveFormatter.TryFormat(_age, buffer, out bytesWritten, format, encoding))
{
return(false);
}
char symbol = _inMonths ? 'm' : 'y';
int symbolBytes;
if (!encoding.TextEncoder.TryEncodeChar(symbol, buffer.Slice(bytesWritten), out symbolBytes))
{
return(false);
}
bytesWritten += symbolBytes;
return(true);
}
public static bool TryAppend <TFormatter>(this TFormatter formatter, double value, EncodingData encoding, TextFormat format = default(TextFormat)) where TFormatter : IOutput
{
int bytesWritten;
if (!value.TryFormat(formatter.Buffer, format, encoding, out bytesWritten))
{
return(false);
}
formatter.Advance(bytesWritten);
return(true);
}
public unsafe static bool TryParseSingle(byte *text, int index, int length, TextFormat format, EncodingData encoding,
out float value, out int bytesConsumed)
{
// Precondition replacement
if (length < 1 || index < 0)
{
value = 0;
bytesConsumed = 0;
return(false);
}
value = 0f;
bytesConsumed = 0;
if (encoding.IsInvariantUtf8)
{
string floatString = "";
bool decimalPlace = false, e = false, signed = false, digitLast = false, eLast = false;
if ((length) >= 3 && text[index] == 'N' && text[index + 1] == 'a' && text[index + 2] == 'N')
{
value = float.NaN;
bytesConsumed = 3;
return(true);
}
if (text[index] == '-' || text[index] == '+')
{
signed = true;
floatString += (char)text[index];
index++;
bytesConsumed++;
}
if ((length - index) >= 8 && text[index] == 'I' && text[index + 1] == 'n' &&
text[index + 2] == 'f' && text[index + 3] == 'i' && text[index + 4] == 'n' &&
text[index + 5] == 'i' && text[index + 6] == 't' && text[index + 7] == 'y')
{
if (signed && text[index - 1] == '-')
{
value = float.NegativeInfinity;
}
else
{
value = float.PositiveInfinity;
}
bytesConsumed += 8;
return(true);
}
for (int byteIndex = index; byteIndex < length; byteIndex++)
{
byte nextByte = text[byteIndex];
byte nextByteVal = (byte)(nextByte - '0');
if (nextByteVal > 9)
{
if (!decimalPlace && nextByte == '.')
{
if (digitLast)
{
digitLast = false;
}
if (eLast)
{
eLast = false;
}
bytesConsumed++;
decimalPlace = true;
floatString += (char)nextByte;
}
else if (!e && nextByte == 'e' || nextByte == 'E')
{
e = true;
eLast = true;
bytesConsumed++;
floatString += (char)nextByte;
}
else if (eLast && nextByte == '+' || nextByte == '-')
{
eLast = false;
bytesConsumed++;
floatString += (char)nextByte;
}
else if ((decimalPlace && signed && bytesConsumed == 2) || ((signed || decimalPlace) && bytesConsumed == 1))
{
value = 0;
bytesConsumed = 0;
return(false);
}
else
{
if (float.TryParse(floatString, out value))
{
return(true);
}
else
{
bytesConsumed = 0;
return(false);
}
}
}
else
{
if (eLast)
{
eLast = false;
}
if (!digitLast)
{
digitLast = true;
}
bytesConsumed++;
floatString += (char)nextByte;
}
}
if ((decimalPlace && signed && bytesConsumed == 2) || ((signed || decimalPlace) && bytesConsumed == 1))
{
value = 0;
bytesConsumed = 0;
return(false);
}
else
{
if (float.TryParse(floatString, out value))
{
return(true);
}
else
{
bytesConsumed = 0;
return(false);
}
}
}
return(false);
}
public static unsafe bool TryParseInt64(byte* text, int index, int length, EncodingData encoding, TextFormat numericFormat,
out long value, out int bytesConsumed)
{
// Precondition replacement
if (length < 1 || index < 0)
{
value = default(long);
bytesConsumed = 0;
return false;
}
value = default(long);
bytesConsumed = 0;
bool negative = false;
bool signed = false;
if (encoding.IsInvariantUtf8)
{
if (text[index] == '-')
{
negative = true;
signed = true;
index++;
bytesConsumed++;
}
else if (text[index] == '+')
{
signed = true;
index++;
bytesConsumed++;
}
for (int byteIndex = index; byteIndex < length + index; byteIndex++)
{
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9) // if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 1 && signed) // if the first character happened to be a '-', we reset the byte counter so logic proceeds as normal.
{
bytesConsumed = 0;
}
if (bytesConsumed == 0)
{
return false;
}
else
{
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (long)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true;
}
}
else if (value > Int64.MaxValue / 10) // overflow
{
value = default(long);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 8 because the max values for unsigned types all end in 7s.
// The min values all end in 8s, which is why that addition exists.
else if (value == Int64.MaxValue / 10 && nextByteVal >= 8 + (negative ? 1 : 0) ) // overflow
{
value = default(long);
bytesConsumed = 0;
return false;
}
value = (long)(value * 10 + nextByteVal); // parse the current digit to a long and add it to the left-shifted value
bytesConsumed++; // increment the number of bytes consumed, then loop
}
if (negative && value != Int64.MinValue) // We check if the value is negative at the very end to save on comp time
{
value = (long)-value;
}
return true;
}
else if (encoding.IsInvariantUtf16)
{
if (text[index] == '-' && text[index + 1] == 0)
{
negative = true;
signed = true;
index += 2;
bytesConsumed += 2;
}
else if (text[index] == '+' && text[index + 1] == 0)
{
signed = true;
index += 2;
bytesConsumed += 2;
}
for (int byteIndex = index; byteIndex < length + index - 1; byteIndex += 2) // loop through the byte array two bytes at a time for UTF-16
{
byte byteAfterNext = text[byteIndex + 1];
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9 || byteAfterNext != 0) // if the second byte isn't zero, this isn't an ASCII-equivalent code unit and we can quit here
// if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 2 && signed) // if the first character happened to be a '-' or a '+', we reset the byte counter so logic proceeds as normal
{
bytesConsumed = 0;
}
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (long)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true; // otherwise return true
}
}
else if (value > Int64.MaxValue / 10) // overflow
{
value = default(long);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 8 because the max values for unsigned types all end in 7s.
// The min values all end in 8s, which is why that addition exists.
else if (value == Int64.MaxValue / 10 && nextByteVal >= 8 + (negative ? 1 : 0) ) // overflow
{
value = default(long);
bytesConsumed = 0;
return false;
}
value = (long)(value * 10 + nextByteVal); // parse the current digit to a long and add it to the left-shifted value
bytesConsumed += 2; // increment the number of bytes consumed, then loop
}
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (long)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true;
}
return false;
}
public static bool TryAppend <TFormatter>(this TFormatter formatter, string value, EncodingData encoding) where TFormatter : IOutput
{
int bytesWritten;
if (!encoding.TextEncoder.TryEncodeString(value, formatter.Buffer, out bytesWritten))
{
return(false);
}
formatter.Advance(bytesWritten);
return(true);
}
public static bool TryParseInt16(byte[] text, int index, EncodingData encoding, TextFormat numericFormat,
out short value, out int bytesConsumed)
{
// Precondition replacement
if (text.Length < 1 || index < 0 || index >= text.Length)
{
value = default(short);
bytesConsumed = 0;
return false;
}
value = default(short);
bytesConsumed = 0;
bool negative = false;
bool signed = false;
if (encoding.IsInvariantUtf8)
{
if (text[index] == '-')
{
negative = true;
signed = true;
index++;
bytesConsumed++;
}
else if (text[index] == '+')
{
signed = true;
index++;
bytesConsumed++;
}
for (int byteIndex = index; byteIndex < text.Length; byteIndex++) // loop through the byte array
{
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9) // if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 1 && signed) // if the first character happened to be a '-' or a '+', we reset the byte counter so logic proceeds as normal.
{
bytesConsumed = 0;
}
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (short)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true; // otherwise return true
}
}
else if (value > Int16.MaxValue / 10) // overflow
{
value = default(short);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 8 because the max values for unsigned types all end in 7s.
// The min values all end in 8s, which is why that addition exists.
else if (value == Int16.MaxValue / 10 && nextByteVal >= 8 + (negative ? 1 : 0) ) // overflow
{
value = default(short);
bytesConsumed = 0;
return false;
}
value = (short)(value * 10 + nextByteVal); // parse the current digit to a short and add it to the left-shifted value
bytesConsumed++; // increment the number of bytes consumed, then loop
}
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (short)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true;
}
else if (encoding.IsInvariantUtf16)
{
if (text[index] == '-' && text[index + 1] == 0)
{
negative = true;
signed = true;
index += 2;
bytesConsumed += 2;
}
else if (text[index] == '+' && text[index + 1] == 0)
{
signed = true;
index += 2;
bytesConsumed += 2;
}
for (int byteIndex = index; byteIndex < text.Length - 1; byteIndex += 2) // loop through the byte array two bytes at a time for UTF-16
{
byte byteAfterNext = text[byteIndex + 1];
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9 || byteAfterNext != 0) // if the second byte isn't zero, this isn't an ASCII-equivalent code unit and we can quit here
// if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 2 && signed) // if the first character happened to be a '-' or a '+', we reset the byte counter so logic proceeds as normal
{
bytesConsumed = 0;
}
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (short)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true; // otherwise return true
}
}
else if (value > Int16.MaxValue / 10) // overflow
{
value = default(short);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 8 because the max values for unsigned types all end in 7s.
// The min values all end in 8s, which is why that addition exists.
else if (value == Int16.MaxValue / 10 && nextByteVal >= 8 + (negative ? 1 : 0) ) // overflow
{
value = default(short);
bytesConsumed = 0;
return false;
}
value = (short)(value * 10 + nextByteVal); // parse the current digit to a short and add it to the left-shifted value
bytesConsumed += 2; // increment the number of bytes consumed, then loop
}
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (short)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true;
}
else
{
int codeUnitsConsumed = 0;
while (bytesConsumed + index < text.Length)
{
uint result;
int consumed;
bool success = encoding.TryParseSymbol(text.Slice(bytesConsumed + index), out result, out consumed);
if (!success || result > 9)
{
if (bytesConsumed == 0 && result == (int)EncodingData.Symbol.MinusSign)
{
negative = true;
signed = true;
bytesConsumed += consumed;
codeUnitsConsumed++;
}
else if (bytesConsumed == 0 && result == (int)EncodingData.Symbol.PlusSign)
{
negative = true;
signed = true;
bytesConsumed += consumed;
}
else if (codeUnitsConsumed == 1 && signed) // if the first character happened to be a '-' or a '+', we reset the byte counter so logic proceeds as normal.
{
bytesConsumed = 0;
return false;
}
else if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (short)-value;
if (value > 0)
{
value = 0;
bytesConsumed = 0;
return false;
}
}
return true; // otherwise return true
}
}
else if (value > Int16.MaxValue / 10)
{
value = default(short);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 8 because the max values for unsigned types all end in 7s.
// The min values all end in 8s, which is why that addition exists.
else if (value == Int16.MaxValue / 10 && result >= 8 + (negative ? 1 : 0) ) // overflow
{
value = default(short);
bytesConsumed = 0;
return false;
}
else
{
value = (short)(value * 10 + result); // left shift the value and add the nextByte
bytesConsumed += consumed;
codeUnitsConsumed++;
}
}
if (negative) // We check if the value is negative at the very end to save on comp time
{
value = (short)-value;
if (value > 0)
{
value = default(short);
bytesConsumed = 0;
return false;
}
}
return true; // otherwise return true
}
}
public static bool TryParseUInt32 <TSequence>(this TSequence bytes, EncodingData encoding, out uint value, out int consumed) where TSequence : ISpanSequence <byte>
{
Position position = Position.First;
Span <byte> first;
if (!bytes.TryGet(ref position, out first, advance: true))
{
throw new ArgumentException("bytes cannot be empty");
}
if (!PrimitiveParser.TryParseUInt32(first, EncodingData.Encoding.Utf8, out value, out consumed))
{
return(false); // TODO: maybe we should continue in some cases, e.g. if the first span ends in a decimal separator
// ... cont, maybe consumed could be set even if TryParse returns false
}
if (position.Equals(Position.AfterLast) || first.Length > consumed)
{
return(true);
}
Span <byte> second;
if (!bytes.TryGet(ref position, out second, advance: true))
{
throw new ArgumentException("bytes cannot be empty");
}
Span <byte> temp;
int numberOfBytesFromSecond = second.Length;
if (numberOfBytesFromSecond > 64)
{
numberOfBytesFromSecond = 64;
}
var tempBufferLength = first.Length + numberOfBytesFromSecond;
if (tempBufferLength > 128)
{
temp = new byte[tempBufferLength];
}
else
{
unsafe
{
byte *data = stackalloc byte[tempBufferLength];
temp = new Span <byte>(data, tempBufferLength);
}
}
first.CopyTo(temp);
second.Slice(0, numberOfBytesFromSecond).CopyTo(temp.Slice(first.Length));
if (!PrimitiveParser.TryParseUInt32(temp, EncodingData.Encoding.Utf8, out value, out consumed))
{
return(false);
}
if (position.Equals(Position.AfterLast) || temp.Length > consumed)
{
return(true);
}
throw new NotImplementedException();
}
public OutputFormatter(TOutput output, EncodingData encoding)
{
_output = output;
_encoding = encoding;
}
public static unsafe bool TryParseBoolean(byte* text, int index, int length, EncodingData encoding,
TextFormat numericFormat, out bool value, out int bytesConsumed)
{
bytesConsumed = 0;
value = default(bool);
if (length < 1 || index < 0)
{
return false;
}
if (encoding.IsInvariantUtf8)
{
byte firstByte = text[index];
if (firstByte == '1')
{
bytesConsumed = 1;
value = true;
return true;
}
else if (firstByte == '0')
{
bytesConsumed = 1;
value = false;
return true;
}
else if (IsTrue(text, index, length))
{
bytesConsumed = 4;
value = true;
return true;
}
else if (IsFalse(text, index, length))
{
bytesConsumed = 5;
value = false;
return true;
}
else
{
return false;
}
}
return false;
}
public bool TryFormat(Span <byte> buffer, out int written, TextFormat format, EncodingData formattingData)
{
written = 0;
int justWritten;
if (!'{'.TryEncode(buffer, out justWritten, formattingData.TextEncoding))
{
return(false);
}
written += justWritten;
bool firstProperty = true;
foreach (var property in _properties)
{
if (property.Key.Object != this)
{
continue;
}
if (firstProperty)
{
firstProperty = false;
}
else
{
if (!','.TryEncode(buffer.Slice(written), out justWritten, formattingData.TextEncoding))
{
return(false);
}
written += justWritten;
}
if (!property.Key.TryFormat(buffer.Slice(written), out justWritten, format, formattingData))
{
written = 0; return(false);
}
written += justWritten;
if (!':'.TryEncode(buffer.Slice(written), out justWritten, formattingData.TextEncoding))
{
return(false);
}
written += justWritten;
if (!property.Value.TryFormat(buffer.Slice(written), out justWritten, format, formattingData))
{
written = 0; return(false);
}
written += justWritten;
}
if (!'}'.TryEncode(buffer.Slice(written), out justWritten, formattingData.TextEncoding))
{
written = 0; return(false);
}
written += justWritten;
return(true);
}
public WritableChannelFormatter(IWritableChannel channel, EncodingData encoding)
{
_channel = channel;
Encoding = encoding;
}
public bool TryFormat(Span <byte> buffer, out int written, TextFormat format, EncodingData formattingData)
{
switch (_type)
{
case JsonReader.JsonValueType.String:
return(_value.TryFormatQuotedString(buffer, format, formattingData, out written));
case JsonReader.JsonValueType.Number:
return(_value.TryFormat(buffer, format, formattingData, out written));
case JsonReader.JsonValueType.Object:
return(_object.TryFormat(buffer, out written, format, formattingData));
case JsonReader.JsonValueType.Null:
return("null".TryEncode(buffer, out written, formattingData.TextEncoding));
case JsonReader.JsonValueType.True:
return("true".TryEncode(buffer, out written, formattingData.TextEncoding));
case JsonReader.JsonValueType.False:
return("false".TryEncode(buffer, out written, formattingData.TextEncoding));
default:
throw new NotImplementedException();
}
}
/// <summary>
/// Gets the encoding of the recent file.
/// </summary>
/// <param name="recentFile">The <see cref="RecentFile"/> instance.</param>
/// <param name="value">The encoding to set for the recent file.</param>
public static void SetEncoding(this RecentFile recentFile, Encoding value)
{
recentFile.EncodingAsString = EncodingData.EncodingToString(value);
}
public bool TryFormat(Span <byte> buffer, out int written, TextFormat format, EncodingData formattingData)
{
return(_name.TryFormatQuotedString(buffer, format, formattingData, out written));
}
public static bool TryParseBoolean(ReadOnlySpan <byte> text, TextFormat format, EncodingData encoding, out bool value, out int bytesConsumed)
{
bytesConsumed = 0;
value = default(bool);
if (text.Length < 1)
{
return(false);
}
if (encoding.IsInvariantUtf8)
{
byte firstCodeUnit = text[0];
if (firstCodeUnit == '1')
{
bytesConsumed = 1;
value = true;
return(true);
}
else if (firstCodeUnit == '0')
{
bytesConsumed = 1;
value = false;
return(true);
}
else if (IsTrue(text))
{
bytesConsumed = 4;
value = true;
return(true);
}
else if (IsFalse(text))
{
bytesConsumed = 5;
value = false;
return(true);
}
else
{
return(false);
}
}
return(false);
}
// TODO: this should be properly implemented
// currently it handles formatting to UTF8 only.
public static bool TryFormat(this Utf8String str, Span <byte> buffer, TextFormat format, EncodingData formattingData, out int written)
{
written = 0;
if (buffer.Length < str.Length)
{
return(false);
}
foreach (var cp in str)
{
var b = cp;
buffer[written++] = cp;
}
return(true);
}
private void ProcessSourceBook()
{
BookNodeStack nodeStack = new BookNodeStack(TextFormatMode.Structured);
bool allowWhitespace = false;
bool wordBegin = true;
bool binaryElement = false;
// Read the source book and write formatted content into a buffer.
using (XmlTextReader reader = new XmlTextReader(sourceFileName))
{
try
{
while (reader.Read())
{
switch (reader.NodeType)
{
case XmlNodeType.XmlDeclaration:
sourceEncodingName = reader["encoding"];
targetEncoding = new EncodingData(Encoding.GetEncoding(sourceEncodingName));
break;
case XmlNodeType.Element:
string elementName = reader.Name;
bool elementEmpty = reader.IsEmptyElement;
WriteElementOpeningTag(elementName, nodeStack.FormatMode, nodeStack.Count);
while (reader.MoveToNextAttribute())
{
WriteElementAttribute(reader.Name, reader.Value);
if (!elementEmpty &&
references != null &&
elementName == "a" && reader.Name == "l:href")
{
currentReference = FindReference(reader.Value);
}
}
if (elementEmpty)
{
WriteEmptyElementCloser();
}
else
{
WriteElementCloser();
}
if (currentReference != null)
{
currentReferenceTextStart = output.Length;
}
allowWhitespace = allowWhitespace && nodeStack.FormatMode != TextFormatMode.Structured;
wordBegin = wordBegin || nodeStack.FormatMode != TextFormatMode.Inline;
if (!elementEmpty)
{
nodeStack.AddNode(elementName);
}
binaryElement = (elementName == "binary" && !elementEmpty);
break;
case XmlNodeType.EndElement:
TextFormatMode insideFormatMode = nodeStack.FormatMode;
nodeStack.DeleteNode(reader.Name);
TextFormatMode outsideFormatMode = nodeStack.FormatMode;
if (insideFormatMode == TextFormatMode.Inline &&
outsideFormatMode == TextFormatMode.Structured)
{
DeleteTrailingWhitespace();
}
if (reader.Name == "a" && currentReference != null)
{
SetReferenceName(currentReference, output.ToString(currentReferenceTextStart, output.Length - currentReferenceTextStart));
currentReference = null;
}
WriteElementClosingTag(reader.Name, insideFormatMode, nodeStack.Count);
allowWhitespace = allowWhitespace && nodeStack.FormatMode != TextFormatMode.Structured;
wordBegin = wordBegin || nodeStack.FormatMode != TextFormatMode.Inline;
binaryElement = false;
break;
case XmlNodeType.Whitespace:
case XmlNodeType.SignificantWhitespace:
if (nodeStack.FormatMode != TextFormatMode.Structured)
{
WriteText(reader.Value, nodeStack.FormatMode, ref allowWhitespace, ref wordBegin);
}
break;
case XmlNodeType.Text:
if (binaryElement)
{
WriteBinary(reader.Value, nodeStack.Count);
}
else
{
WriteText(reader.Value, nodeStack.FormatMode, ref allowWhitespace, ref wordBegin);
}
break;
case XmlNodeType.Comment:
WriteComment(reader.Value, nodeStack.FormatMode, nodeStack.Count);
allowWhitespace = true;
wordBegin = true;
break;
}
}
}
catch (Exception ex)
{
string message = string.Format(errXmlParseError, sourceFileName, reader.LineNumber, reader.LinePosition, ex.Message);
throw new Exception(message, ex);
}
}
}
public static bool TryFormatQuotedString(this Utf8String str, Span <byte> buffer, TextFormat format, EncodingData formattingData, out int written)
{
written = 0;
int justWritten;
if (!'"'.TryEncode(buffer, out justWritten, formattingData.TextEncoding))
{
return(false);
}
written += justWritten;
if (!str.TryFormat(buffer.Slice(written), format, formattingData, out justWritten))
{
return(false);
}
written += justWritten;
if (!'"'.TryEncode(buffer.Slice(written), out justWritten, formattingData.TextEncoding))
{
return(false);
}
written += justWritten;
return(true);
}
public static bool TryParseInt64(ReadOnlySpan <byte> text, out long value, out int bytesConsumed, EncodingData encoding = default(EncodingData), TextFormat format = default(TextFormat))
{
if (format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}
if (encoding.IsInvariantUtf8)
{
if (format.IsHexadecimal)
{
return(InvariantUtf8.Hex.TryParseInt64(text, out value, out bytesConsumed));
}
else
{
return(InvariantUtf8.TryParseInt64(text, out value, out bytesConsumed));
}
}
else if (encoding.IsInvariantUtf16)
{
ReadOnlySpan <char> utf16Text = text.Cast <byte, char>();
int charsConsumed;
bool result;
if (format.IsHexadecimal)
{
result = InvariantUtf16.Hex.TryParseInt64(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseInt64(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return(result);
}
if (format.IsHexadecimal)
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}
if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}
uint nextSymbol;
int thisSymbolConsumed;
if (!encoding.TryParseSymbol(text, out nextSymbol, out thisSymbolConsumed))
{
value = default(long);
bytesConsumed = 0;
return(false);
}
int sign = 1;
if ((EncodingData.Symbol)nextSymbol == EncodingData.Symbol.MinusSign)
{
sign = -1;
}
int signConsumed = 0;
if ((EncodingData.Symbol)nextSymbol == EncodingData.Symbol.PlusSign || (EncodingData.Symbol)nextSymbol == EncodingData.Symbol.MinusSign)
{
signConsumed = thisSymbolConsumed;
if (!encoding.TryParseSymbol(text.Slice(signConsumed), out nextSymbol, out thisSymbolConsumed))
{
value = default(long);
bytesConsumed = 0;
return(false);
}
}
if (nextSymbol > 9)
{
value = default(long);
bytesConsumed = 0;
return(false);
}
long parsedValue = (long)nextSymbol;
int index = signConsumed + thisSymbolConsumed;
while (index < text.Length)
{
bool success = encoding.TryParseSymbol(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > 9)
{
bytesConsumed = index;
value = (long)(parsedValue * sign);
return(true);
}
// If parsedValue > (long.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (long.MaxValue / 10), any nextDigit greater than 7 or 8 (depending on sign) implies overflow.
bool positive = sign > 0;
bool nextDigitTooLarge = nextSymbol > 8 || (positive && nextSymbol > 7);
if (parsedValue > long.MaxValue / 10 || (parsedValue == long.MaxValue / 10 && nextDigitTooLarge))
{
bytesConsumed = 0;
value = default(long);
return(false);
}
index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + (long)nextSymbol;
}
bytesConsumed = text.Length;
value = (long)(parsedValue * sign);
return(true);
}
public static bool TryParseUInt64(ReadOnlySpan <byte> text, out ulong value, out int bytesConsumed, TextFormat format = default(TextFormat), EncodingData encoding = default(EncodingData))
{
if (encoding == default(EncodingData))
{
encoding = EncodingData.InvariantUtf8;
}
if (!format.IsDefault && format.HasPrecision)
{
throw new NotImplementedException("Format with precision not supported.");
}
if (encoding.IsInvariantUtf8)
{
if (format.IsHexadecimal)
{
return(InvariantUtf8.Hex.TryParseUInt64(text, out value, out bytesConsumed));
}
else
{
return(InvariantUtf8.TryParseUInt64(text, out value, out bytesConsumed));
}
}
else if (encoding.IsInvariantUtf16)
{
ReadOnlySpan <char> utf16Text = text.Cast <byte, char>();
int charsConsumed;
bool result;
if (format.IsHexadecimal)
{
result = InvariantUtf16.Hex.TryParseUInt64(utf16Text, out value, out charsConsumed);
}
else
{
result = InvariantUtf16.TryParseUInt64(utf16Text, out value, out charsConsumed);
}
bytesConsumed = charsConsumed * sizeof(char);
return(result);
}
if (format.IsHexadecimal)
{
throw new NotImplementedException("The only supported encodings for hexadecimal parsing are InvariantUtf8 and InvariantUtf16.");
}
if (!(format.IsDefault || format.Symbol == 'G' || format.Symbol == 'g'))
{
throw new NotImplementedException(String.Format("Format '{0}' not supported.", format.Symbol));
}
uint nextSymbol;
int thisSymbolConsumed;
if (!encoding.TryParseSymbol(text, out nextSymbol, out thisSymbolConsumed))
{
value = default(ulong);
bytesConsumed = 0;
return(false);
}
if (nextSymbol > 9)
{
value = default(ulong);
bytesConsumed = 0;
return(false);
}
ulong parsedValue = nextSymbol;
int index = thisSymbolConsumed;
while (index < text.Length)
{
bool success = encoding.TryParseSymbol(text.Slice(index), out nextSymbol, out thisSymbolConsumed);
if (!success || nextSymbol > 9)
{
bytesConsumed = index;
value = (ulong)parsedValue;
return(true);
}
// If parsedValue > (ulong.MaxValue / 10), any more appended digits will cause overflow.
// if parsedValue == (ulong.MaxValue / 10), any nextDigit greater than 5 implies overflow.
if (parsedValue > ulong.MaxValue / 10 || (parsedValue == ulong.MaxValue / 10 && nextSymbol > 5))
{
bytesConsumed = 0;
value = default(ulong);
return(false);
}
index += thisSymbolConsumed;
parsedValue = parsedValue * 10 + nextSymbol;
}
bytesConsumed = text.Length;
value = (ulong)parsedValue;
return(true);
}
public static unsafe bool TryParseUInt32(byte* text, int index, int length, EncodingData encoding, TextFormat numericFormat,
out uint value, out int bytesConsumed)
{
// Precondition replacement
if (length < 1 || index < 0)
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = default(uint);
bytesConsumed = 0;
if (encoding.IsInvariantUtf8)
{
for (int byteIndex = index; byteIndex < length + index; byteIndex++)
{
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9) // if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 0)
{
return false;
}
else
{
return true;
}
}
else if (value > UInt32.MaxValue / 10) // overflow
{
value = default(uint);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 6 because the max values for unsigned types all end in 5s.
else if (value == UInt32.MaxValue / 10 && nextByteVal >= 6) // overflow
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = (uint)(value * 10 + nextByteVal); // left shift the value and add the nextByte
bytesConsumed++; // increment the number of bytes consumed, then loop
}
return true;
}
else if (encoding.IsInvariantUtf16)
{
for (int byteIndex = index; byteIndex < length + index - 1; byteIndex += 2) // loop through the byte array two bytes at a time for UTF-16
{
byte byteAfterNext = text[byteIndex + 1];
byte nextByteVal = (byte)(text[byteIndex] - '0');
if (nextByteVal > 9 || byteAfterNext != 0) // if the second byte isn't zero, this isn't an ASCII-equivalent code unit and we can quit here
// if nextByteVal > 9, we know it is not a digit because any value less than '0' will overflow
// to greater than 9 since byte is an unsigned type.
{
if (bytesConsumed == 0) // check to see if we've processed any digits at all
{
return false;
}
else
{
return true; // otherwise return true
}
}
else if (value > UInt32.MaxValue / 10)
{
value = default(uint);
bytesConsumed = 0;
return false;
}
// This next check uses a hardcoded 6 because the max values for unsigned types all end in 5s.
else if (value == UInt32.MaxValue / 10 && nextByteVal >= 6) // overflow
{
value = default(uint);
bytesConsumed = 0;
return false;
}
value = (uint)(value * 10 + nextByteVal); // left shift the value and add the nextByte
bytesConsumed += 2;
}
return true;
}
return false;
}
public BytesReader(IReadOnlyMemoryList <byte> bytes, EncodingData encoding) : this(new ReadOnlyBytes(bytes))
{
}
public SequenceFormatter(TSequence buffers, EncodingData encoding)
{
_encoding = encoding;
_buffers = buffers;
_previousWrittenBytes = -1;
}
public ArrayFormatter(int capacity, EncodingData encoding, ArrayPool<byte> pool = null)
{
_pool = pool != null ? pool : ArrayPool<byte>.Shared;
_encoding = encoding;
_buffer = new ResizableArray<byte>(_pool.Rent(capacity));
}
public SpanFormatter(Span <byte> buffer, EncodingData encoding)
{
_encoding = encoding;
_count = 0;
_buffer = buffer;
}
public SequenceFormatter(ISpanSequence<byte> buffers, EncodingData encoding)
{
_encoding = encoding;
_buffers = buffers;
_previousWrittenBytes = -1;
}
