public Bytes compress([NotNull] IBufferProtocol data)
{
using var buffer = data.GetBuffer();
byte[] input = buffer.AsUnsafeArray() ?? buffer.ToArray();
byte[] output = new byte[ZlibModule.DEFAULTALLOC];
long start_total_out = zst.total_out;
zst.next_in = input;
zst.next_in_index = 0;
zst.avail_in = input.Length;
zst.next_out = output;
zst.next_out_index = 0;
zst.avail_out = output.Length;
int err = zst.deflate(Z_NO_FLUSH);
while (err == Z_OK && zst.avail_out == 0)
{
int length = output.Length;
Array.Resize(ref output, output.Length * 2);
zst.next_out = output;
zst.avail_out = length;
err = zst.deflate(Z_NO_FLUSH);
}
if (err != Z_OK && err != Z_BUF_ERROR)
{
throw ZlibModule.zlib_error(this.zst, err, "while compressing");
}
return(GetBytes(output, 0, (int)(zst.total_out - start_total_out)));
}
internal static IPythonBuffer?GetBufferNoThrow(this IBufferProtocol bufferProtocol, BufferFlags flags = BufferFlags.Simple)
{
try {
return(bufferProtocol.GetBuffer(flags));
} catch (BufferException) {
return(null);
}
}
public static unsafe int send(int handle, [NotNone] IBufferProtocol data)
{
using var buffer = data.GetBuffer();
var span = buffer.AsReadOnlySpan();
fixed(byte *ptr = &MemoryMarshal.GetReference(span))
return(send(new IntPtr(handle), new IntPtr(ptr), span.Length, 0));
}
public void update([NotNone] IBufferProtocol data)
{
using var buffer = data.GetBuffer();
byte[] bytes = buffer.ToArray();
lock (_hasher) {
_hasher.TransformBlock(bytes, 0, bytes.Length, bytes, 0);
}
}
public static Bytes decompress([NotNone] IBufferProtocol data,
int wbits = MAX_WBITS,
int bufsize = DEFAULTALLOC)
{
using var buffer = data.GetBuffer();
var bytes = Decompress(buffer.AsUnsafeArray() ?? buffer.ToArray(), wbits, bufsize);
return(Bytes.Make(bytes));
}
public Bytes compress([NotNull] IBufferProtocol data)
{
using var buffer = data.GetBuffer();
byte[] bytes = buffer.AsUnsafeArray() ?? buffer.ToArray();
this.bz2Output.Write(bytes, 0, bytes.Length);
return(Bytes.Make(this.GetLatestData()));
}
public Bytes decompress([NotNone] IBufferProtocol data, int max_length = 0)
{
if (max_length < 0)
{
throw new ArgumentException("max_length must be greater than zero");
}
using var buffer = data.GetBuffer();
byte[] input = buffer.AsUnsafeArray() ?? buffer.ToArray();
byte[] output = new byte[max_length > 0 && ZlibModule.DEFAULTALLOC > max_length ? max_length : ZlibModule.DEFAULTALLOC];
long start_total_out = zst.total_out;
zst.next_in = input;
zst.next_in_index = 0;
zst.avail_in = input.Length;
zst.next_out = output;
zst.next_out_index = 0;
zst.avail_out = output.Length;
int err = zst.inflate(FlushStrategy.Z_SYNC_FLUSH);
while (err == Z_OK && zst.avail_out == 0)
{
if (max_length > 0 && output.Length >= max_length)
{
break;
}
int old_length = output.Length;
Array.Resize(ref output, output.Length * 2);
zst.next_out = output;
zst.avail_out = old_length;
err = zst.inflate(FlushStrategy.Z_SYNC_FLUSH);
}
if (max_length > 0)
{
unconsumed_tail = GetBytes(zst.next_in, zst.next_in_index, zst.avail_in);
}
if (err == Z_STREAM_END)
{
unused_data += GetBytes(zst.next_in, zst.next_in_index, zst.avail_in);
eof = true;
}
else if (err != Z_OK && err != Z_BUF_ERROR)
{
throw ZlibModule.zlib_error(this.zst, err, "while decompressing");
}
return(GetBytes(output, 0, (int)(zst.total_out - start_total_out)));
}
public static Bytes compress([NotNone] IBufferProtocol data,
int level = Z_DEFAULT_COMPRESSION)
{
using var buffer = data.GetBuffer();
byte[] input = buffer.AsUnsafeArray() ?? buffer.ToArray();
byte[] output = new byte[input.Length + input.Length / 1000 + 12 + 1];
ZStream zst = new ZStream();
zst.next_in = input;
zst.avail_in = input.Length;
zst.next_out = output;
zst.avail_out = output.Length;
int err = zst.DeflateInit(level);
switch (err)
{
case (Z_OK):
break;
case (Z_STREAM_ERROR):
throw PythonOps.CreateThrowable(error,
"Bad compression level");
default:
zst.deflateEnd();
zlib_error(zst, err, "while compressing data");
return(null);
}
err = zst.deflate(FlushStrategy.Z_FINISH);
if (err != Z_STREAM_END)
{
zst.deflateEnd();
throw zlib_error(zst, err, "while compressing data");
}
err = zst.deflateEnd();
if (err == Z_OK)
{
var res = new byte[(int)zst.total_out];
Array.Copy(output, res, res.Length);
return(Bytes.Make(res));
}
throw zlib_error(zst, err, "while finishing compression");
}
private int DoWrite(IBufferProtocol bufferProtocol)
{
using var buffer = bufferProtocol.GetBuffer();
var bytes = buffer.AsReadOnlySpan();
if (bytes.Length == 0)
{
return(0);
}
EnsureSizeSetLength(_pos + bytes.Length);
bytes.CopyTo(_data.AsSpan(_pos, bytes.Length));
_pos += bytes.Length;
return(bytes.Length);
}
public MemoryView([NotNone] MemoryView @object)
{
_exporter = @object._exporter;
_flags = BufferFlags.RecordsRO;
_buffer = _exporter.GetBuffer(_flags);
_offset = @object._offset;
_isReadOnly = @object._isReadOnly;
_numDims = @object._numDims;
_format = @object._format;
_itemSize = @object._itemSize;
_shape = @object._shape;
_strides = @object._strides;
_isCContig = @object._isCContig;
_isFContig = @object._isFContig;
_numItems = @object._numItems;
}
public Bytes decompress([NotNull] IBufferProtocol data)
{
if (_finished)
{
throw PythonOps.EofError("End of stream was already found");
}
using var buffer = data.GetBuffer();
byte[] bytes = buffer.AsUnsafeArray() ?? buffer.ToArray();
if (!InitializeMemoryStream(bytes))
{
AddData(bytes);
}
List <byte> output = new List <byte>();
if (InitializeBZ2Stream())
{
long memoryPosition = this.input.Position;
object state = this.bz2Input.DumpState();
try {
// this is the same as what Read() does, so it's unlikely to be
// any slower. However, using blocks would require fewer state saves,
// which would probably be faster.
int b;
while ((b = this.bz2Input.ReadByte()) != -1)
{
output.Add((byte)b);
memoryPosition = this.input.Position;
state = this.bz2Input.DumpState();
}
this.lastSuccessfulPosition = this.input.Position;
this._finished = true;
} catch (IOException) {
// rewind the decompressor and the memory buffer to try again when
// more data arrives
this.input.Position = memoryPosition;
this.bz2Input.RestoreState(state);
}
}
return(new Bytes(output));
}
public static void PlaySound(CodeContext /*!*/ context, [NotNone] IBufferProtocol sound, int flags)
{
if (((flags & SND_ASYNC) == SND_ASYNC) && ((flags & SND_MEMORY) == SND_MEMORY))
{
throw PythonOps.RuntimeError("Cannot play asynchronously from memory");
}
if ((flags & SND_MEMORY) == 0)
{
throw PythonOps.TypeError($"'{nameof(sound)}' must be str or None, not '{PythonOps.GetPythonTypeName(sound)}'");
}
using var buffer = sound.GetBuffer();
if (!PlaySound(buffer.ToArray(), IntPtr.Zero, flags))
{
throw PythonOps.RuntimeError("Failed to play sound");
}
}
public static Bytes byteswap(CodeContext /*!*/ context, [NotNone] IBufferProtocol fragment, int width)
{
if (width < 1 || width > 4)
{
throw PythonExceptions.CreateThrowable(error(context), "Size should be 1, 2, 3 or 4");
}
using var buffer = fragment.GetBuffer();
if (buffer.NumBytes() % width != 0)
{
throw PythonExceptions.CreateThrowable(error(context), "not a whole number of frames");
}
var array = buffer.ToArray();
if (width == 2)
{
for (var i = 0; i < array.Length; i += width)
{
array.ByteSwap(i, i + 1);
}
}
else if (width == 3)
{
for (var i = 0; i < array.Length; i += width)
{
array.ByteSwap(i, i + 2);
}
}
else if (width == 4)
{
for (var i = 0; i < array.Length; i += width)
{
array.ByteSwap(i, i + 3);
array.ByteSwap(i + 1, i + 2);
}
}
return(Bytes.Make(array));
}
private static IList <byte> ConvertFromBufferProtocolToByteListHelper(IBufferProtocol bp)
{
using var buf = bp.GetBuffer(BufferFlags.Simple);
return(buf.AsReadOnlySpan().ToArray());
}
public Bytes([NotNull] IBufferProtocol source)
{
using IPythonBuffer buffer = source.GetBuffer(BufferFlags.FullRO);
_bytes = buffer.ToArray();
}
public MemoryView([NotNone] IBufferProtocol @object)
{
_exporter = @object;
// MemoryView should support all possible buffer exports (BufferFlags.FullRO)
// but handling of suboffsets (BufferFlags.Indirect) is not implemented yet.
// Hence the request is for BufferFlags.RecordsRO
_flags = BufferFlags.RecordsRO;
_buffer = @object.GetBuffer(_flags);
// doublecheck that we don't have to deal with suboffsets
if (_buffer.SubOffsets != null)
{
throw PythonOps.NotImplementedError("memoryview: indirect buffers are not supported");
}
ReadOnlySpan <byte> memblock = _buffer.AsReadOnlySpan();
if ((_buffer.ItemCount != 0 && !VerifyStructure(memblock.Length, _buffer.ItemSize, _buffer.NumOfDims, _buffer.Shape, _buffer.Strides, _buffer.Offset)) ||
(_buffer.Shape == null && (_buffer.Offset != 0 || _buffer.NumBytes() != memblock.Length))
)
{
throw PythonOps.BufferError("memoryview: invalid buffer exported from object of type {0}", PythonOps.GetPythonTypeName(@object));
}
_offset = _buffer.Offset;
_isReadOnly = _buffer.IsReadOnly;
_numDims = _buffer.NumOfDims;
// in flags we requested format be provided, check that the exporter complied
if (_buffer.Format == null)
{
throw PythonOps.BufferError("memoryview: object of type {0} did not report its format", PythonOps.GetPythonTypeName(@object));
}
_format = _buffer.Format;
_itemSize = _buffer.ItemSize;
// for convenience _shape and _strides are never null, even if _numDims == 0 or _flags indicate no _shape or _strides
_shape = _buffer.Shape ?? (_numDims > 0 ? new int[] { _buffer.ItemCount } : Array.Empty <int>());
if (_shape.Count == 0)
{
_strides = _shape; // TODO: use a static singleton
_isCContig = true;
}
else if (_buffer.Strides != null)
{
_strides = _buffer.Strides;
_isCContig = true;
for (int i = _strides.Count - 1, curStride = _itemSize; i >= 0 && _isCContig; i--)
{
_isCContig &= _strides[i] == curStride;
curStride *= _shape[i];
}
}
else
{
_strides = GetContiguousStrides(_shape, _itemSize);
_isCContig = true;
}
// invariants
_numItems = _buffer.ItemCount;
_isFContig = _isCContig && _numDims <= 1; // TODO: support for ND Fortran arrays not implemented
// sanity check
Debug.Assert(_numItems == 0 || VerifyStructure(memblock.Length, _itemSize, _numDims, _numDims > 0 ? _shape : null, _numDims > 0 ? _strides : null, _offset));
}
public static int adler32([NotNone] IBufferProtocol data, long value = 1L)
{
using var buffer = data.GetBuffer();
return((int)Adler32.GetAdler32Checksum(value, buffer.AsUnsafeArray() ?? buffer.ToArray(), 0, buffer.NumBytes()));
}
private object RFindWorker(IBufferProtocol bufferProtocol, long start, long end)
{
using var pythonBuffer = bufferProtocol.GetBuffer();
var s = pythonBuffer.AsReadOnlySpan();
start = PythonOps.FixSliceIndex(start, _view.Capacity);
end = PythonOps.FixSliceIndex(end, _view.Capacity);
if (s.Length == 0)
{
return(start <= end?ReturnLong(start) : -1);
}
long findLength = end - start;
if (s.Length > findLength)
{
return(-1);
}
int index = -1;
int bufferLength = Math.Max(s.Length, PAGESIZE);
CompareInfo c = CultureInfo.InvariantCulture.CompareInfo;
if (findLength <= bufferLength * 2)
{
// In this case, the search area is not significantly larger than s, so we only need to
// allocate a single string to search through.
byte[] buffer = new byte[findLength];
findLength = _view.ReadArray(start, buffer, 0, (int)findLength);
index = buffer.AsSpan().LastIndexOf(s);
}
else
{
// We're matching s against a significantly larger file, so we partition the stream into
// sections twice the length of s and search each segment. Because a match could exist on a
// boundary, sections must overlap by s.Length. Data is saved in 2 buffers to avoid
// reading the same parts of the stream twice.
byte[] buffer0 = new byte[bufferLength];
byte[] buffer1 = new byte[bufferLength];
int remainder = (int)((end - start) % bufferLength);
if (remainder == 0)
{
remainder = bufferLength;
}
start = end - bufferLength - remainder;
findLength -= bufferLength + remainder;
_view.ReadArray(start, buffer0, 0, bufferLength);
int bytesRead = _view.ReadArray(start + bufferLength, buffer1, 0, remainder);
while (findLength >= 0)
{
var combinedBuffer = CombineBytes(buffer0, buffer1, bytesRead);
index = combinedBuffer.AsSpan().LastIndexOf(s);
if (index != -1)
{
return(ReturnLong(index + start));
}
byte[] temp = buffer0;
buffer0 = buffer1;
buffer1 = temp;
start -= bufferLength;
bytesRead = _view.ReadArray(start, buffer0, 0, bufferLength);
findLength -= bytesRead;
}
}
return(index == -1 ? -1 : ReturnLong(index + start));
}