/// <summary> /// Get New Data according to chunk size From Store and Fill data Chunk /// Stores New Data Locally also in case chunk not recieve by user next time can send from local /// </summary> /// <param name="lastChunkId"></param> /// <param name="dataChunk"></param> protected virtual bool FillNewData(int lastChunkId, ref IDataChunk dataChunk) { int count = 0; bool check = false; _lastChunkData.Clear(); if (_lastElementCheck) { IJSONDocument jsonDocument = _store.GetDocument(_lastElement, _context); if (jsonDocument != null) { jsonDocument = _transform.Transform(jsonDocument); if (jsonDocument != null) { dataChunk.Documents.Add(jsonDocument); _lastChunkData.Add(_lastElement); count++; } } _lastElementCheck = false; } while (_enumerator.MoveNext()) { if (check) { check = false; _lastElement = _enumerator.Current; ////To Save Round Trip in case Data Chunk fills and Data also finish so that can make the existing chunk as last chunk _lastElementCheck = true; break; } IJSONDocument jsonDocument = _store.GetDocument(_enumerator.Current, _context); if (jsonDocument != null) { jsonDocument = _transform.Transform(jsonDocument); if (jsonDocument != null) { dataChunk.Documents.Add(jsonDocument); _lastChunkData.Add(_enumerator.Current); count++; } } if (count == _chunkSize) { check = true; } } if (count < _chunkSize || check) { dataChunk.IsLastChunk = true; _isLastChunk = true; } else { dataChunk.IsLastChunk = false; } dataChunk.ReaderUID = _id; dataChunk.ChunkId = lastChunkId + 1; _lastChunkId = lastChunkId + 1; return(true); }
public void TestThatDataTransformCallsProviderAndExporters() { _target.Transform(); Assert.IsTrue(_exporters[0].Input.IsEqualTo(_dataInput), "Test failed: first exporter did not receive the data input."); Assert.IsTrue(_exporters[1].Input.IsEqualTo(_dataInput), "Test failed: second data exporter did not receive the data input."); }