private async ValueTask ValidateMagicAsync()
{
var startingPosition = BaseStream.Position;
var magicLength = ArrowFileConstants.Magic.Length;
try
{
await Buffers.RentReturnAsync(magicLength, async (buffer) =>
{
// Seek to the beginning of the stream
BaseStream.Position = 0;
// Read beginning of stream
await BaseStream.ReadAsync(buffer).ConfigureAwait(false);
VerifyMagic(buffer);
// Move stream position to magic-length bytes away from the end of the stream
BaseStream.Position = BaseStream.Length - magicLength;
// Read the end of the stream
await BaseStream.ReadAsync(buffer).ConfigureAwait(false);
VerifyMagic(buffer);
}).ConfigureAwait(false);
}
finally
{
BaseStream.Position = startingPosition;
}
}
public async Task <byte[]> ReadBytesAsync(int count, CancellationToken cancellationToken)
{
var b = new byte[count];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return(b);
}
private void ContinuousRead()
{
byte[] buffer = new byte[4096];
Action kickoffRead = null;
kickoffRead = async() =>
{
try
{
int count = await BaseStream.ReadAsync(buffer, 0, buffer.Length);
byte[] dst = new byte[count];
Buffer.BlockCopy(buffer, 0, dst, 0, count);
OnDataReceived(dst);
}
catch (Exception exception)
{
Console.WriteLine(exception.Message);
Console.WriteLine(exception.StackTrace);
}
kickoffRead();
};
kickoffRead();
}
public async Task <uint> ReadUInt32Async(CancellationToken cancellationToken)
{
var b = new byte[sizeof(uint)];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return(BitConverter.ToUInt32(b, 0));
}
public async Task <double> ReadDoubleAsync(CancellationToken cancellationToken)
{
var b = new byte[sizeof(double)];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return(BitConverter.ToDouble(b, 0));
}
public async Task <long> ReadInt64Async(CancellationToken cancellationToken)
{
var b = new byte[sizeof(long)];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return(BitConverter.ToInt64(b, 0));
}
public async Task <sbyte> ReadSByteAsync(CancellationToken cancellationToken)
{
var b = new byte[sizeof(sbyte)];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return((sbyte)b[0]);
}
public async Task<byte[]> ReadBytesAsync(int count, CancellationToken cancellationToken)
{
var b = new byte[count];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken)
.ConfigureAwait(false);
return b;
}
private async Task <byte[]> ReadBufferAsync(int size, CancellationToken cancellationToken)
{
var buffer = Buffer;
int offset = 0;
Stream stream = BaseStream;
if (stream == null)
{
FileNotOpen();
}
do
{
int read = await BaseStream.ReadAsync(buffer, offset, size, cancellationToken).ConfigureAwait(false);
if (read == 0)
{
EndOfStream();
}
offset += read;
size -= read;
} while (size > 0);
return(buffer);
}
/// <summary>
/// Asynchronously reads a sequence of bytes from the stream and advances the position
/// within the stream by the number of bytes read.
/// </summary>
/// <remarks>
/// Reads data from the <see cref="BaseStream"/>, not allowing it to
/// read beyond the <see cref="EndBoundary"/>.
/// </remarks>
/// <returns>The total number of bytes read into the buffer. This can be less than the number of bytes requested if
/// that many bytes are not currently available, or zero (0) if the end of the stream has been reached.</returns>
/// <param name="buffer">The buffer to read data into.</param>
/// <param name="offset">The offset into the buffer to start reading data.</param>
/// <param name="count">The number of bytes to read.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <exception cref="System.ArgumentNullException">
/// <paramref name="buffer"/> is <c>null</c>.
/// </exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// <para><paramref name="offset"/> is less than zero or greater than the length of <paramref name="buffer"/>.</para>
/// <para>-or-</para>
/// <para>The <paramref name="buffer"/> is not large enough to contain <paramref name="count"/> bytes starting
/// at the specified <paramref name="offset"/>.</para>
/// </exception>
/// <exception cref="System.ObjectDisposedException">
/// The stream has been disposed.
/// </exception>
/// <exception cref="System.NotSupportedException">
/// The stream does not support reading.
/// </exception>
/// <exception cref="System.OperationCanceledException">
/// The operation was canceled via the cancellation token.
/// </exception>
/// <exception cref="System.IO.IOException">
/// An I/O error occurred.
/// </exception>
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
CheckDisposed();
CheckCanRead();
ValidateArguments(buffer, offset, count);
// if we are at the end of the stream, we cannot read anymore data
if (EndBoundary != -1 && StartBoundary + position >= EndBoundary)
{
eos = true;
return(0);
}
// make sure that the source stream is in the expected position
if (BaseStream.Position != StartBoundary + position)
{
BaseStream.Seek(StartBoundary + position, SeekOrigin.Begin);
}
int n = EndBoundary != -1 ? (int)Math.Min(EndBoundary - (StartBoundary + position), count) : count;
int nread = await BaseStream.ReadAsync(buffer, offset, n, cancellationToken).ConfigureAwait(false);
if (nread > 0)
{
position += nread;
}
else if (nread == 0)
{
eos = true;
}
return(nread);
}
public async Task <short> ReadInt16Async(CancellationToken cancellationToken)
{
var b = new byte[sizeof(short)];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return(BitConverter.ToInt16(b, 0));
}
public async Task <float> ReadSingleAsync(CancellationToken cancellationToken)
{
var b = new byte[sizeof(float)];
await BaseStream.ReadAsync(b, 0, b.Length, cancellationToken);
return(BitConverter.ToSingle(b, 0));
}
public async Task <IMessage> ReadAsync()
{
IMessage message = null;
MessageBuffer buffer = new MessageBuffer(Encoding);
while (message == null)
{
byte[] chunk = new byte[_BufferSize];
int result = await BaseStream.ReadAsync(chunk, 0, _BufferSize);
if (result > 0)
{
buffer.Append(chunk.Take(result).ToArray());
}
else
{
break;
}
if (!buffer.Valid)
{
continue;
}
string content = buffer.TryReadContent();
message = MessageSerializer.Deserialize(content);
buffer = new MessageBuffer(Encoding);
}
return(message);
}
/************************ Construction ***********************************/
/************************ Methods ****************************************/
/// <summary>
/// Asynchronously reads data from the stream firing the events
/// when data is received
/// </summary>
/// <returns>
/// Total number of bytes that were read from the entire
/// read sequence
/// </returns>
private async Task <int> ReadDataAsync()
{
int retval = -1;
byte[] buffer = new byte[BufferSize];
int bytesRead = 0;
try
{
// While we are still able to read data from the stream,
// do so and fire events informing listeners we received
// data.
while ((bytesRead = await BaseStream.ReadAsync(buffer,
0,
buffer.Length,
CancelTokenSource.Token)) > 0)
{
retval += bytesRead;
OnDataReceived(buffer, bytesRead);
} // end of while - stream is still available and reading bytes
} // end of try - to read bytes from stream
catch (Exception exc)
{
Console.WriteLine($"Exception in reading data: ${exc}");
if (exc is TaskCanceledException || exc is OperationCanceledException)
{
Debug.WriteLine("Cancel exception encountered, will ignore");
} // end of if - cancel exception
else // we will rethrow to inform of the exception
{
throw;
} // end of else - rethrow
} // end of catch - cancel exception
return(retval);
} // end of function - ReadDataAsync
public override async Task <int> ReadAsync(byte[] buffer, CancellationToken cancellationToken = default)
{
int length = await BaseStream.ReadAsync(buffer, cancellationToken);
var decrypted = DecryptCipher.ProcessBytes(buffer, 0, length);
Array.Copy(decrypted, 0, buffer, 0, decrypted.Length);
return(length);
}
protected override async Task <long> WriteStreamInternal(Stream stream, long start, long?stop)
{
long total = 0;
int readed;
byte[] buffer = new byte[ReadBufferLength];
do
{
readed = await BaseStream.ReadAsync(
buffer,
0,
(int)Math.Min(buffer.Length, start - total)
).ConfigureAwait(false);
total += readed;
}while (total < start && readed > 0);
if (readed == 0 && start > 0)
{
return(0);
}
var ascii = Encoding.ASCII;
var nl = ascii.GetBytes("\r\n");
do
{
var read = stop == null
? buffer.Length
: (int)Math.Min(buffer.Length, stop.Value - total);
readed = await BaseStream.ReadAsync(buffer, 0, read).ConfigureAwait(false);
if (readed <= 0)
{
return(total - start);
}
var length = ascii.GetBytes(readed.ToString("X"));
try
{
await stream.WriteAsync(length, 0, length.Length).ConfigureAwait(false);
await stream.WriteAsync(nl, 0, nl.Length).ConfigureAwait(false);
await stream.WriteAsync(buffer, 0, readed).ConfigureAwait(false);
await stream.WriteAsync(nl, 0, nl.Length).ConfigureAwait(false);
total += readed;
await stream.FlushAsync().ConfigureAwait(false);
}
catch (IOException)
{
WebServerLog.Add(ServerLogType.Information, GetType(), "write", "connection closed");
return(total);
}
}while (readed > 0);
return(total - start);
}
/// <inheritdoc />
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
var readCount = await BaseStream.ReadAsync(buffer, offset, count, cancellationToken);
if (readCount > 0)
{
StreamMonitor.OnRead(new ArraySegment <byte>(buffer, offset, readCount));
}
return(readCount);
}
/// <summary>
/// Reads data from the stream
/// </summary>
/// <param name="buffer">Buffer to read into</param>
/// <param name="offset">Where in the buffer to start</param>
/// <param name="count">Number of bytes to be read</param>
/// <param name="token">The <see cref="CancellationToken"/> for this task</param>
/// <returns>The amount of bytes read from the stream</returns>
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken token)
{
if (BaseStream == null)
{
return(0);
}
m_lastActivity = DateTime.Now;
return(await BaseStream.ReadAsync(buffer, offset, count, token));
}
protected override async Task <Schema> ReadSchemaAsync()
{
if (HasReadSchema)
{
return(Schema);
}
await ValidateFileAsync();
var bytesRead = 0;
var footerLength = 0;
await Buffers.RentReturnAsync(4, async (buffer) =>
{
BaseStream.Position = BaseStream.Length - ArrowFileConstants.Magic.Length - 4;
bytesRead = await BaseStream.ReadAsync(buffer, 0, 4);
footerLength = BinaryPrimitives.ReadInt32LittleEndian(buffer);
if (bytesRead != 4)
{
throw new InvalidDataException(
$"Failed to read footer length. Read <{bytesRead}>, expected 4.");
}
if (footerLength <= 0)
{
throw new InvalidDataException(
$"Footer length has invalid size <{footerLength}>");
}
});
await Buffers.RentReturnAsync(footerLength, async (buffer) =>
{
_footerStartPostion = (int)BaseStream.Length - footerLength - ArrowFileConstants.Magic.Length - 4;
BaseStream.Position = _footerStartPostion;
bytesRead = await BaseStream.ReadAsync(buffer, 0, footerLength);
if (bytesRead != footerLength)
{
throw new InvalidDataException(
$"Failed to read footer. Read <{bytesRead}> bytes, expected <{footerLength}>.");
}
// Deserialize the footer from the footer flatbuffer
_footer = new ArrowFooter(Flatbuf.Footer.GetRootAsFooter(new ByteBuffer(buffer)));
Schema = _footer.Schema;
});
return(Schema);
}
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
int bytesRead = await BaseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
if (this.OnRead != null)
{
this.OnRead(bytesRead);
}
return(bytesRead);
}
/// <summary>
/// 从当前流异步读取字节序列，将流中的位置向前移动读取的字节数，并监控取消请求。
/// </summary>
/// <returns>
/// 表示异步读取操作的任务。
/// </returns>
/// <param name="buffer">数据写入的缓冲区。</param><param name="offset"><paramref name="buffer"/> 中的字节偏移量，从该偏移量开始写入从流中读取的数据。</param><param name="count">最多读取的字节数。</param><param name="cancellationToken">针对取消请求监视的标记。 默认值为 <see cref="P:System.Threading.CancellationToken.None"/>。</param><exception cref="T:System.ArgumentNullException"><paramref name="buffer"/> 为 null。</exception><exception cref="T:System.ArgumentOutOfRangeException"><paramref name="offset"/> 或 <paramref name="count"/> 为负。</exception><exception cref="T:System.ArgumentException"><paramref name="offset"/> 与 <paramref name="count"/> 的和大于缓冲区长度。</exception><exception cref="T:System.NotSupportedException">流不支持读取。</exception><exception cref="T:System.ObjectDisposedException">流已被释放。</exception><exception cref="T:System.InvalidOperationException">该流正在由其前一次读取操作使用。</exception>
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
var readcount = await BaseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
if (_enableClone)
{
await MirrorStream.WriteAsync(buffer, offset, readcount, cancellationToken).ConfigureAwait(true);
}
_position += readcount;
OnProgressChanged(new DataProgressEventArgs(_streamLength, _position));
return(readcount);
}
/// <summary>
/// Attempt to read sufficient data for a whole chunk from the wrapped stream,
/// returning the number of bytes successfully read to be processed into a chunk
/// </summary>
private async Task <int> FillInputBufferAsync(CancellationToken cancellationToken)
{
if (_wrappedStreamConsumed)
{
return(0);
}
var inputBufferPos = 0;
if (_readStrategy == ReadStrategy.ReadDirect)
{
while (inputBufferPos < _inputBuffer.Length && !_wrappedStreamConsumed)
{
// chunk buffer size may not align exactly with underlying buffer size
var chunkBufferRemaining = _inputBuffer.Length - inputBufferPos;
if (chunkBufferRemaining > _wrappedStreamBufferSize)
{
chunkBufferRemaining = _wrappedStreamBufferSize;
}
var bytesRead = await BaseStream.ReadAsync(_inputBuffer, inputBufferPos, chunkBufferRemaining, cancellationToken).ConfigureAwait(false);
if (bytesRead == 0)
{
_wrappedStreamConsumed = true;
}
else
{
inputBufferPos += bytesRead;
}
}
}
else
{
var readBuffer = new byte[_wrappedStreamBufferSize];
while (inputBufferPos < _inputBuffer.Length && !_wrappedStreamConsumed)
{
var bytesRead = await BaseStream.ReadAsync(readBuffer, 0, _wrappedStreamBufferSize, cancellationToken).ConfigureAwait(false);
if (bytesRead == 0)
{
_wrappedStreamConsumed = true;
}
else
{
Buffer.BlockCopy(readBuffer, 0, _inputBuffer, inputBufferPos, bytesRead);
inputBufferPos += bytesRead;
}
}
}
return(inputBufferPos);
}
/// <summary>
/// Asynchronously reads a sequence of decrypted bytes from the current stream, advances
/// the position within the stream by the number of bytes read, and monitors
/// cancellation requests.
/// </summary>
/// <param name="buffer">
/// An array of bytes. When this method returns, the buffer contains the specified
/// byte array with the values between offset and (offset + count - 1) replaced
/// by the bytes read from the current source.
/// </param>
/// <param name="offset">
/// The zero-based byte offset in buffer at which to begin storing the data read
/// from the current stream.
/// </param>
/// <param name="count">
/// The maximum number of bytes to be read from the current stream.
/// </param>
/// <param name="cancellationToken">
/// The token to monitor for cancellation requests. The default value is
/// System.Threading.CancellationToken.None.
/// </param>
/// <returns>
/// A task that represents the asynchronous read operation. The value of the TResult
/// parameter contains the total number of bytes read into the buffer. This can be
/// less than the number of bytes requested if that many bytes are not currently
/// available, or zero (0) if the end of the stream has been reached.
/// </returns>
/// <exception cref="AmazonCryptoException">
/// Underlying crypto exception wrapped in Amazon exception
/// </exception>
public override async System.Threading.Tasks.Task <int> ReadAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken)
{
try
{
var readBytes = await BaseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
return(readBytes);
}
catch (CryptoException cryptoException)
{
throw new AmazonCryptoException($"Failed to decrypt: {cryptoException.Message}", cryptoException);
}
}
/// <summary>
/// Read the raw network message
/// </summary>
/// <returns>byte buffer</returns>
protected async Task <byte[]> ReadBytesAsync()
{
// establish function to find out if there's data available, based on type of stream
Func <bool> availableFunc;
if (BaseStream is NetworkStream)
{
availableFunc = () => { return(((NetworkStream)BaseStream).DataAvailable); }
}
;
else if (BaseStream is PipeStream)
{
availableFunc = () => { return(true); }
}
;
else if (BaseStream.CanSeek)
{
availableFunc = () => { return(BaseStream.Position < BaseStream.Length); }
}
;
else
{
throw new InvalidOperationException("IpcStream.BaseStream needs to be seekable or derived from a known type.");
}
// wait until data is available
await Extensions.WaitUntil(availableFunc, 25, Server.ReadTimeOut).ConfigureAwait(false);
// read message length
byte[] buffer = new byte[2];
if (await BaseStream.ReadAsync(buffer, 0, 2).ConfigureAwait(false) != 2)
{
throw new EndOfStreamException("Insufficient bytes read from network stream");
}
// calculate message length
int length = buffer[0] * 256;
length += buffer[1];
// read message
buffer = new byte[length];
int read = 0;
while (read < length)
{
read += await BaseStream.ReadAsync(buffer, read, length - read).ConfigureAwait(false);
}
return(buffer);
}
async Task <int> ReadAsync(byte[] buffer, int offset, int count, bool doAsync, CancellationToken cancellationToken)
{
CheckDisposed();
ValidateArguments(buffer, offset, count);
if (count == 0)
{
return(0);
}
zIn.next_out = buffer;
zIn.next_out_index = offset;
zIn.avail_out = count;
do
{
if (zIn.avail_in == 0 && !eos)
{
if (doAsync)
{
zIn.avail_in = await BaseStream.ReadAsync(zIn.next_in, 0, zIn.next_in.Length, cancellationToken).ConfigureAwait(false);
}
else
{
zIn.avail_in = BaseStream.Read(zIn.next_in, 0, zIn.next_in.Length);
}
eos = zIn.avail_in == 0;
zIn.next_in_index = 0;
}
int retval = zIn.inflate(JZlib.Z_FULL_FLUSH);
if (retval == JZlib.Z_STREAM_END)
{
break;
}
if (eos && retval == JZlib.Z_BUF_ERROR)
{
return(0);
}
if (retval != JZlib.Z_OK)
{
throw new IOException("Error inflating: " + zIn.msg);
}
} while (zIn.avail_out == count);
return(count - zIn.avail_out);
}
private void BeginRead()
{
if (!available)
{
return;
}
var buffer = new byte[2048];
BaseStream.ReadAsync(buffer, 0, buffer.Length).ContinueWith(t =>
{
NativeStream->DataReceived(NativeStream, buffer, t.Result);
BeginRead();
}, cancellationToken.Token);
}
private async Task FillBuff(int totalBytes)
{
var bytesRead = 0;
do
{
var n = await BaseStream.ReadAsync(buff, bytesRead, totalBytes - bytesRead).ConfigureAwait(false);
if (n == 0)
{
throw new IOException("Unable to read next byte(s).");
}
bytesRead += n;
} while (bytesRead < totalBytes);
}
private async Task ValidateMagicAsync()
{
var startingPosition = BaseStream.Position;
var magicLength = ArrowFileConstants.Magic.Length;
try
{
await Buffers.RentReturnAsync(magicLength, async (buffer) =>
{
// Seek to the beginning of the stream
BaseStream.Position = 0;
// Read beginning of stream
await BaseStream.ReadAsync(buffer, 0, magicLength);
if (!ArrowFileConstants.Magic.SequenceEqual(buffer.Take(magicLength)))
{
throw new InvalidDataException(
$"Invalid magic at offset <{BaseStream.Position}>");
}
// Move stream position to magic-length bytes away from the end of the stream
BaseStream.Position = BaseStream.Length - magicLength;
// Read the end of the stream
await BaseStream.ReadAsync(buffer, 0, magicLength);
if (!ArrowFileConstants.Magic.SequenceEqual(buffer.Take(magicLength)))
{
throw new InvalidDataException(
$"Invalid magic at offset <{BaseStream.Position}>");
}
});
}
finally
{
BaseStream.Position = startingPosition;
}
}
public async Task <Data> ReadDataAsync(int length)
{
var buffer = new byte[length];
var countRead = await BaseStream.ReadAsync(buffer, 0, length);
if (countRead == 0)
{
return(null);
}
if (countRead == length)
{
return(buffer);
}
var result = buffer.Take(countRead).ToArray();
return(result);
}
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
byte[] array = new byte[64];
int length = await base.ReadAsync(buffer, offset, count, cancellationToken);
if (length == 0)
{
while (true)
{
var len = await BaseStream.ReadAsync(array, 0, array.Length);
if (len == 0)
{
break;
}
}
}
return(length);
}
