public int ReadTile(IStreamReader inputStream, byte[] inputBuffer, int index)
{
if (PointCount == 0)
{
return(0);
}
if (index + StorageSize > inputBuffer.Length)
{
throw new ArgumentException("Tile data is larger than available buffer", "inputBuffer");
}
// seek if necessary (hopefully this is minimized)
var position = TileSet.TileSource.PointDataOffset + (PointOffset * TileSet.TileSource.PointSizeBytes);
if (inputStream.Position != position)
{
inputStream.Seek(position);
}
// read available points from main tile area and get low-res points from source
var localStorageSize = (PointCount - LowResCount) * TileSet.TileSource.PointSizeBytes;
var bytesRead = inputStream.Read(inputBuffer, index, localStorageSize);
//bytesRead += TileSet.TileSource.ReadLowResTile(this, inputBuffer, index + bytesRead);
var lowResPosition = TileSet.TileSource.PointDataOffset + ((TileSet.PointCount - TileSet.LowResCount + LowResOffset) * TileSet.TileSource.PointSizeBytes);
inputStream.Seek(lowResPosition);
var lowResStorageSize = LowResCount * TileSet.TileSource.PointSizeBytes;
bytesRead += inputStream.Read(inputBuffer, index + bytesRead, lowResStorageSize);
return(bytesRead);
}
public T Read(int stride = 1)
{
var value = FDataReader.Length > 0 ? FDataReader.Read(stride) : default(T);
FPosition += stride;
return(value);
}
public static int Read <T>(IStreamReader <T> reader, T[] buffer, int index, int length, int stride = 1)
{
int slicesToRead = GetNumSlicesAhead(reader, index, length, stride);
switch (stride)
{
case 0:
if (index == 0 && slicesToRead == buffer.Length)
{
buffer.Init(reader.Read(stride)); // Slightly faster
}
else
{
buffer.Fill(index, slicesToRead, reader.Read(stride));
}
break;
default:
for (int i = index; i < index + slicesToRead; i++)
{
buffer[i] = reader.Read(stride);
}
break;
}
return(slicesToRead);
}
public T Read(int stride = 1)
{
if (Eos)
{
throw new InvalidOperationException();
}
FPosition += stride;
return(FSourceReader.Read(stride));
}
public T Read(int stride = 1)
{
var result = FReader.Read(stride);
if (FReader.Eos)
{
FReader.Position %= FReader.Length;
}
return(result);
}
public T Read(int stride = 1)
{
if (Eos)
{
throw new InvalidOperationException();
}
Position += stride;
var result = FSourceReader.Read(stride);
Position += stride;
return(result);
}
public bool MoveNext()
{
// check for cancel
if (m_current != null && m_process != null && !m_process.Update(m_current))
{
return(false);
}
if (m_stream.Position < m_endPosition)
{
int bytesRead = m_stream.Read(m_buffer.Data, 0, m_usableBytesPerBuffer);
if (bytesRead == 0)
{
throw new Exception("I did something wrong");
}
if (m_stream.Position > m_endPosition)
{
bytesRead -= (int)(m_stream.Position - m_endPosition);
}
int index = (m_current != null) ? m_current.Index + 1 : 0;
m_current = new PointCloudBinarySourceEnumeratorChunk(index, m_buffer, bytesRead, m_source.PointSizeBytes, (float)(m_stream.Position - m_source.PointDataOffset) / (m_endPosition - m_source.PointDataOffset));
return(true);
}
return(false);
}
protected override void Sync(int i, int offsetIntoDataStream, int binSize, IStreamReader <T> dataReader)
{
var spread = this[i];
var builder = spread.ToBuilder();
builder.Count = binSize;
var slicesToRead = binSize;
using (var buffer = MemoryPool <T> .GetBuffer())
{
var j = 0;
dataReader.Position = offsetIntoDataStream;
while (slicesToRead > 0)
{
var slicesRead = dataReader.Read(buffer, 0, Math.Min(slicesToRead, buffer.Length));
if (slicesRead > 0)
{
for (int k = 0; k < slicesRead; k++)
{
builder[j++] = buffer[k];
}
slicesToRead -= slicesRead;
}
else
{
break;
}
}
}
this[i] = builder.ToSpread();
}
public int Read(T[] buffer, int offset, int count)
{
var totalCopiedItems = 0;
while (count > 0 && reader != null)
{
var itemsCopied = reader.Read(buffer, offset, count);
if (itemsCopied > 0)
{
count -= itemsCopied;
offset += itemsCopied;
totalCopiedItems += itemsCopied;
}
else
{
try{
reader.Dispose();
}finally{
reader = null;
}
reader = GetNextReader();
}
}
return(totalCopiedItems);
}
private static byte[] ReadBytes(IStreamReader reader)
{
var bLen = reader.ReadInt32(); // Byte length
if (bLen < 0)
{
return(null);
}
if (bLen == NullLengthFlag)
{
return(null);
}
if (bLen == 0)
{
return(new byte[0]);
}
var result = new byte[bLen];
var readLen = reader.Read(result, 0, bLen);
if (readLen != bLen)
{
throw new ArgumentOutOfRangeException(nameof(readLen));
}
return(result);
}
//called when data for any output pin is requested
public void Evaluate(int SpreadMax)
{
/// Configure the output count
if (FAnalogIns.IsChanged)
{
FAnalogIns.SliceCount = FAnalogInputCount[0];
}
if (FDigitalIns.IsChanged)
{
FDigitalIns.SliceCount = FDigitalInputCount[0];
}
/// If there is nothing new to read, there is nothing to parse
if (!FirmataIn.IsChanged)
{
return;
}
/// Read in the stream
try {
using (IStreamReader <Stream> InputReader = FirmataIn.GetReader())
{
while (!InputReader.Eos)
{
HandleStream(InputReader.Read());
}
}
} catch (Exception e) {
return;
}
}
protected void PrintText(StreamWriter sw, IStreamReader workbookReader)
{
BiffHeader bh;
try
{
while (workbookReader.BaseStream.Position < workbookReader.BaseStream.Length)
{
bh.id = (RecordType)workbookReader.ReadUInt16();
bh.length = workbookReader.ReadUInt16();
byte[] buffer = new byte[bh.length];
if (bh.length != workbookReader.Read(buffer, bh.length))
{
sw.WriteLine("EOF");
}
sw.Write("BIFF {0}\t{1}\t", bh.id, bh.length);
//Dump(buffer);
int count = 0;
foreach (byte b in buffer)
{
sw.Write("{0:X02} ", b);
count++;
if (count % 16 == 0 && count < buffer.Length)
{
sw.Write("\n\t\t\t");
}
}
sw.Write("\n");
if (_backgroundWorker != null)
{
int progress = 100;
if (sw.BaseStream.Length != 0)
{
progress = (int)(100 * workbookReader.BaseStream.Position / workbookReader.BaseStream.Length);
}
_backgroundWorker.ReportProgress(progress);
if (_backgroundWorker.CancellationPending)
{
_isCancelled = true;
break;
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
//called when data for any output pin is requested
public void Evaluate(int SpreadMax)
{
/// Configure the output count
if (FAnalogIns.IsChanged)
{
FAnalogIns.SliceCount = FAnalogInputCount[0];
}
if (FDigitalIns.IsChanged)
{
FDigitalIns.SliceCount = FDigitalInputCount[0];
}
StringOut.SliceCount = 0;
/// If there is nothing new to read, there is nothing to parse
if (!FirmataIn.IsChanged)
{
return;
}
/// Read in the stream
try {
using (IStreamReader <Stream> InputReader = FirmataIn.GetReader())
{
while (!InputReader.Eos)
{
Stream InStream = InputReader.Read();
if (InStream == null || InStream.Length == 0 || !InStream.CanRead)
{
continue;
}
// Read the incoming bytes to the internal stream buffer
for (int i = InStream.ReadByte(); i != -1; i = InStream.ReadByte())
{
Decode((byte)i);
}
}
}
} catch (Exception e) {
return;
}
}
public static WireMessageId Read(IStreamReader reader)
{
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
var buffer = new byte[BufferSize];
if (reader.Read(buffer, 0, BufferSize) != BufferSize)
{
throw new ArgumentOutOfRangeException(nameof(buffer));
}
return(Convert(buffer.ToInt(BufferOffsets[0]),
buffer.ToInt(BufferOffsets[1]),
buffer.ToInt(BufferOffsets[2]),
buffer.ToInt(BufferOffsets[3]),
buffer.ToInt(BufferOffsets[4])));
}
public void Copy(Stream source, Stream target)
{
using (var readQueue = new BoundedBlockingQueue <byte[]>(100))
{
scheduler.StartNew(() =>
{
reader.Read(source, readQueue);
readQueue.CompleteAdding();
});
using (var processedQueue = new BoundedBlockingQueue <IndexedBuffer>(100))
{
scheduler.StartNew(() =>
{
processor.Process(readQueue, processedQueue);
processedQueue.CompleteAdding();
});
var writeWaitHandle = new ManualResetEvent(false);
scheduler.StartNew(() => writer.Write(processedQueue, target), writeWaitHandle);
writeWaitHandle.WaitOne();
}
}
}
/// <summary>
/// Reads a sequence of items from the current stream and advances the position
/// within the stream by the number of items read.
/// Use the stride parameter to control advancing of the position after one item has been read.
/// </summary>
/// <param name="reader">The reader to use from reading.</param>
/// <param name="segment">The array segment to read into.</param>
/// <param name="stride">The stride by which the position is advanced after reading one item.</param>
/// <returns>
/// The total number of items read into the buffer.
/// This can be less than the number of items requested if that many items are not currently available,
/// or zero (0) if the end of the stream has been reached.
/// </returns>
public static int Read <T>(this IStreamReader <T> reader, ArraySegment <T> segment, int stride = 1)
{
return(reader.Read(segment.Array, segment.Offset, segment.Count, stride));
}
private string ReadString(IStreamReader reader)
{
var bLen = reader.ReadInt32(); // Byte array length
if (bLen < 0)
{
return(null);
}
if (bLen == 0)
{
return(String.Empty);
}
if (bLen == 1)
{
return(new String((char)reader.ReadByte(), 1));
}
var sLen = reader.ReadInt32(); // String length
if (sLen > bLen || bLen > 4 * sLen)
{
throw new Exception(SerializationErrors.StreamNotContainingValidWireMessage);
}
if (sLen > MaxAllowedStringLength)
{
throw new Exception(SerializationErrors.StringLengthExceededAllowedLimit);
}
var sb = (StringBuilder)null;
var bytes = StringBytesCache.Acquire();
try
{
var chars = StringCharsCache.Acquire();
try
{
var readLen = 0;
var bytesOffset = 0;
var bytesUsed = 0;
var charsUsed = 0;
var completed = false;
var remaining = bLen;
do
{
readLen = Math.Min(StringBufferSize - bytesOffset, remaining);
readLen = reader.Read(bytes, bytesOffset, readLen);
if (readLen == bLen)
{
return(Encoding.UTF8.GetString(bytes, 0, readLen));
}
if (readLen == 0)
{
throw new ArgumentOutOfRangeException(nameof(readLen));
}
bytesOffset += readLen;
UTF8Decoder.Convert(bytes, 0, bytesOffset, chars, 0, StringBufferSize,
false, out bytesUsed, out charsUsed, out completed);
if (charsUsed > 0)
{
if (sb == null)
{
sb = new StringBuilder(sLen);
}
sb.Append(chars, 0, charsUsed);
bytesOffset -= bytesUsed;
if (bytesOffset > 0)
{
Array.Copy(bytes, bytesUsed, bytes, 0, bytesOffset);
}
}
}while ((remaining -= readLen) > 0);
}
finally
{
StringCharsCache.Release(chars);
}
}
finally
{
StringBytesCache.Release(bytes);
}
return(sb?.ToString());
}
private WireMessage Read(IStreamReader reader)
{
var dataTypeCd = reader.ReadByte() - NotNullFlag;
if (dataTypeCd > 0)
{
var message = new WireMessage();
var buffer = StringBytesCache.Acquire();
try
{
var sizeOfDataTypeCd = WireMessageFieldSizeOf.DataTypeCd;
reader.Read(buffer, 0, WireMessageSizeOf.ConstantFields - sizeOfDataTypeCd);
message.MessageType = (MessageType)buffer[WireMessageBufferOffsetOf.MessageType - sizeOfDataTypeCd];
message.State = (WireMessageState)buffer[WireMessageBufferOffsetOf.State - sizeOfDataTypeCd];
var timeoutMSec = buffer.ToInt(WireMessageBufferOffsetOf.TimeoutMSec - sizeOfDataTypeCd);
message.TimeoutMSec = timeoutMSec != int.MinValue ? timeoutMSec : (int?)null;
message.Id = new WireMessageId(
buffer.ToInt(WireMessageBufferOffsetOf.IdMajor - sizeOfDataTypeCd),
buffer.ToInt(WireMessageBufferOffsetOf.IdMajorRevision - sizeOfDataTypeCd),
buffer.ToInt(WireMessageBufferOffsetOf.IdMinor - sizeOfDataTypeCd),
buffer.ToInt(WireMessageBufferOffsetOf.IdMinorRevision - sizeOfDataTypeCd),
buffer.ToInt(WireMessageBufferOffsetOf.IdProcessId - sizeOfDataTypeCd)
);
}
finally
{
StringBytesCache.Release(buffer);
}
message.From = Aid.Parse(ReadString(reader));
message.To = Aid.Parse(ReadString(reader));
var isNullByte = reader.ReadByte();
if (isNullByte != NullFlag)
{
var header = new Dictionary <string, string>();
message.Header = header;
var count = reader.ReadInt32();
if (count > 0)
{
string key;
for (var i = 0; i < count; i++)
{
key = ReadString(reader);
if (key != null)
{
header[key] = ReadString(reader);
}
}
}
}
isNullByte = reader.ReadByte();
if (isNullByte != NullFlag)
{
var len = reader.ReadInt32();
if (len > 0)
{
using (var tempStream = new ChunkedStream())
{
var chunkSize = tempStream.ChunkSize;
while (len > 0)
{
var bytes = reader.ReadBytes(chunkSize);
var readLen = bytes?.Length ?? 0;
if (readLen == 0)
{
throw new Exception(SerializationErrors.StreamNotContainingValidWireMessage);
}
len -= readLen;
tempStream.Write(bytes, 0, readLen);
}
tempStream.Position = 0;
message.Exception = GetSerializer().Deserialize <Exception>(tempStream);
}
}
}
ReadData(dataTypeCd, message, reader);
return(message);
}
return(null);
}
public int ReadTile(IStreamReader inputStream, byte[] inputBuffer, int index)
{
if (PointCount == 0)
return 0;
if (index + StorageSize > inputBuffer.Length)
throw new ArgumentException("Tile data is larger than available buffer", "inputBuffer");
// seek if necessary (hopefully this is minimized)
var position = TileSet.TileSource.PointDataOffset + (PointOffset * TileSet.TileSource.PointSizeBytes);
if (inputStream.Position != position)
inputStream.Seek(position);
// read available points from main tile area and get low-res points from source
var localStorageSize = (PointCount - LowResCount) * TileSet.TileSource.PointSizeBytes;
var bytesRead = inputStream.Read(inputBuffer, index, localStorageSize);
//bytesRead += TileSet.TileSource.ReadLowResTile(this, inputBuffer, index + bytesRead);
var lowResPosition = TileSet.TileSource.PointDataOffset + ((TileSet.PointCount - TileSet.LowResCount + LowResOffset) * TileSet.TileSource.PointSizeBytes);
inputStream.Seek(lowResPosition);
var lowResStorageSize = LowResCount * TileSet.TileSource.PointSizeBytes;
bytesRead += inputStream.Read(inputBuffer, index + bytesRead, lowResStorageSize);
return bytesRead;
}
public static IOutputStream <TDst> Transform <TSrc, TDst>(this IOutputStream <TSrc> stream, ITransformation <TSrc, TDst> transformation, int readBufSize = 1024)
{
//Encoding.UTF8.GetBytes(str, strPos, charsToBuffer, intBuf, 0);
return(CreateOutput <TDst>(() => {
var inBuf = new TSrc[readBufSize];
var inBufPos = 0;
var inBufCnt = 0;
IStreamReader <TSrc> reader = null;
if (stream != null)
{
reader = stream.CreateReader();
}
return new AnonymousStreamReader <TDst>((buf, ofs, cnt) => {
var totalBytesCopied = 0;
while (cnt > 0)
{
var bytesCopied = transformation.Copy(buf, ofs, cnt);
if (bytesCopied > 0)
{
ofs += bytesCopied;
cnt -= bytesCopied;
totalBytesCopied += bytesCopied;
}
else
{
Action process = () => {
var itemsProcessed = transformation.Process(inBuf, inBufPos, inBufCnt);
inBufCnt -= itemsProcessed;
inBufPos += itemsProcessed;
};
if (inBufCnt > 0)
{
process();
}
else
{
if (reader == null)
{
return totalBytesCopied;
}
inBufPos = 0;
//try {
inBufCnt = reader.Read(inBuf, 0, inBuf.Length);
//} catch (Exception err) {
//inBufCnt = 0;
//try {
// reader.Dispose();
//} finally {
// reader = null;
//}
//}
if (inBufCnt > 0)
{
process();
}
else
{
try {
reader.Dispose();
} finally {
reader = null;
}
return totalBytesCopied;
}
}
}
}
return totalBytesCopied;
});
}));
}
