private void Convert(byte[] data, int dataLength, EncodingType encoding, bool hasBom, byte[] outputData, out int outputLength) { int outputIndex = 0; if (hasBom) { CopyBomTo(outputData, encoding, out outputIndex); } int numBytesPerUnit = encoding.GetNumBytesPerUnit(); int startDataIndex = outputIndex; for (int dataIndex = startDataIndex; dataIndex <= dataLength - numBytesPerUnit; dataIndex += numBytesPerUnit) { var currentUnit = GetUnit(data, dataIndex, numBytesPerUnit); var nextUnit = GetUnit(data, dataIndex + numBytesPerUnit, numBytesPerUnit); ProcessUnit(outputData, currentUnit, encoding, ref outputIndex); if (currentUnit.IsCR(encoding) && nextUnit.IsLF(encoding)) { // If CRLF then the do not process the next LF unit because the end of line has been inserted while processing the CR dataIndex += numBytesPerUnit; } } outputLength = outputIndex; }
private bool IsFromEncoding( EncodingDetectionContext context, EncodingType encoding, Func <EncodingDetectionContext, int, bool> IsFromEndiannes) { int numBytesPerUnit = encoding.GetNumBytesPerUnit(); // If there is a entire null unit within the data then can't be this encoding if (ContainsNullBytesUnit(context.Data, context.DataLength, numBytesPerUnit)) { return(false); } return(IsFromEndiannes(context, numBytesPerUnit)); }
private static byte[] ToUnit(this byte byteChar, EncodingType encoding) { var nullBytes = Enumerable.Range(0, encoding.GetNumBytesPerUnit() - 1) .Select(i => ByteCode.Null); var byteCharArray = new byte[1] { byteChar }; if (encoding.IsBigEndian()) { return(nullBytes.Concat(byteCharArray).ToArray()); } return(byteCharArray.Concat(nullBytes).ToArray()); }
public void GetEncoding_WhenDataEndsByEolButDataIsNotCompletelyFilled(byte[] eolBytes, EncodingType expectedEncoding) { // Arrange int bufferLength = 10 * expectedEncoding.GetNumBytesPerUnit(); byte[] data = new byte[bufferLength]; int dataLength = bufferLength / 2; FillDataWithDummyValues(data, dataLength); eolBytes.CopyTo(data, dataLength - eolBytes.Length); CreateSut(); // Act var encoding = sut.GetEncoding(data, dataLength).encoding; // Assert Assert.Equal(expectedEncoding, encoding); }