public override bool Sync() { // Sync source IsChanged = FDataStream.Sync() | FBinSizeStream.Sync(); if (IsChanged) { int dataLength = FDataStream.Length; int binSizeLength = FBinSizeStream.Length; int binSizeSum = 0; foreach (var binSize in FBinSizeStream) { binSizeSum += SpreadUtils.NormalizeBinSize(dataLength, binSize); } int binTimes = SpreadUtils.DivByBinSize(dataLength, binSizeSum); binTimes = binTimes > 0 ? binTimes : 1; Length = binTimes * binSizeLength; var binSizeBuffer = MemoryPool <int> .GetArray(); try { using (var binSizeReader = FBinSizeStream.GetCyclicReader()) { var numSlicesToWrite = Length; var offsetIntoDataStream = 0; while (numSlicesToWrite > 0) { var numSlicesToRead = Math.Min(numSlicesToWrite, binSizeBuffer.Length); binSizeReader.Read(binSizeBuffer, 0, numSlicesToRead); var offset = Length - numSlicesToWrite; for (int i = offset; i < offset + numSlicesToRead; i++) { var binSize = SpreadUtils.NormalizeBinSize(dataLength, binSizeBuffer[i - offset]); var innerStream = this[i] as InnerStream; // Inner stream will use cyclic reader when offset + length > length of data stream. innerStream.Offset = offsetIntoDataStream; innerStream.Length = binSize; offsetIntoDataStream += binSize; } numSlicesToWrite -= numSlicesToRead; } } } finally { MemoryPool <int> .PutArray(binSizeBuffer); } } return(base.Sync()); }
public override bool Sync() { // Sync source IsChanged = FBinSizeStream.Sync() | FDataStream.Sync(); if (IsChanged) { // Normalize bin size and compute sum int dataStreamLength = FDataStream.Length; int binSizeSum = 0; foreach (var binSize in FBinSizeStream) { binSizeSum += SpreadUtils.NormalizeBinSize(dataStreamLength, binSize); } int binTimes = SpreadUtils.DivByBinSize(dataStreamLength, binSizeSum); binTimes = binTimes > 0 ? binTimes : 1; Length = binTimes * FBinSizeStream.Length; using (var binSizeReader = FBinSizeStream.GetCyclicReader()) using (var dataReader = FDataStream.GetCyclicReader()) { for (int i = 0; i < Length; i++) { var spread = Buffer[i]; var stream = spread.Stream; var binSize = SpreadUtils.NormalizeBinSize(dataStreamLength, binSizeReader.Read()); spread.SliceCount = binSize; switch (binSize) { case 0: break; case 1: stream.Buffer[0] = dataReader.Read(); break; default: dataReader.Read(stream.Buffer, 0, binSize); break; } // Mark the stream as changed stream.IsChanged = true; } } } return(base.Sync()); }
public VecBinSpread(IInStream <T> input, int vectorSize, IInStream <int> bin, int spreadMax = 0) { vecSize = vectorSize; buffer = new List <T[]>(); itemCount = 0; nonEmptyBinCount = 0; if (bin.Length > 0) { int sliceCount = (int)Math.Ceiling(input.Length / (double)vecSize); int incr = 0; using (var binReader = bin.GetCyclicReader()) { using (var dataReader = input.GetCyclicReader()) { while (incr < dataReader.Length || (binReader.Position % binReader.Length) != 0 || spreadMax > 0) { int curBin = SpreadUtils.NormalizeBinSize(sliceCount, binReader.Read()) * vecSize; itemCount += curBin; T[] data = new T[curBin]; if (curBin > 0) { dataReader.Read(data, 0, curBin); nonEmptyBinCount++; } buffer.Add(data); spreadMax--; incr += curBin; if (incr == 0) { if (binReader.Position == 0) { incr += dataReader.Length; } } } } } } }
public void Sync(IInStream <T> input, int vectorSize, IInStream <int> bin, int spreadMax = 0) { var itemCount = 0; var buffer = FBuffer; var nonEmptyBinCount = 0; if (bin.Length > 0) { using (var binReader = bin.GetCyclicReader()) using (var dataReader = input.GetCyclicReader()) { var sliceCount = (int)Math.Ceiling(input.Length / (double)vectorSize); var incr = 0; var i = 0; while (incr < dataReader.Length || (binReader.Position % binReader.Length) != 0 || spreadMax > 0) { int curBin = SpreadUtils.NormalizeBinSize(sliceCount, binReader.Read()) * vectorSize; itemCount += curBin; // Re-use existing arrays if possible T[] data; if (i < buffer.Count) { data = buffer[i]; if (data.Length != curBin) { data = buffer[i] = Allocate(curBin); } } else { data = Allocate(curBin); buffer.Add(data); } i++; if (curBin > 0) { dataReader.Read(data, 0, curBin); nonEmptyBinCount++; } spreadMax--; incr += curBin; if (incr == 0) { if (binReader.Position == 0) { incr += dataReader.Length; } } } if (i < buffer.Count) { buffer.RemoveRange(i, buffer.Count - i); } } } else { buffer.Clear(); } FVectorSize = vectorSize; FItemCount = itemCount; FNonEmptyBinCount = nonEmptyBinCount; }