/// <summary>
/// Get New Data according to chunk size From Store and Fill data Chunk
/// Stores New Data Locally also in case chunk not recieve by user next time can send from local
/// </summary>
/// <param name="lastChunkId"></param>
/// <param name="dataChunk"></param>
protected virtual bool FillNewData(int lastChunkId, ref IDataChunk dataChunk)
{
int count = 0;
bool check = false;
_lastChunkData.Clear();
if (_lastElementCheck)
{
IJSONDocument jsonDocument = _store.GetDocument(_lastElement, _context);
if (jsonDocument != null)
{
jsonDocument = _transform.Transform(jsonDocument);
if (jsonDocument != null)
{
dataChunk.Documents.Add(jsonDocument);
_lastChunkData.Add(_lastElement);
count++;
}
}
_lastElementCheck = false;
}
while (_enumerator.MoveNext())
{
if (check)
{
check = false;
_lastElement = _enumerator.Current; ////To Save Round Trip in case Data Chunk fills and Data also finish so that can make the existing chunk as last chunk
_lastElementCheck = true;
break;
}
IJSONDocument jsonDocument = _store.GetDocument(_enumerator.Current, _context);
if (jsonDocument != null)
{
jsonDocument = _transform.Transform(jsonDocument);
if (jsonDocument != null)
{
dataChunk.Documents.Add(jsonDocument);
_lastChunkData.Add(_enumerator.Current);
count++;
}
}
if (count == _chunkSize)
{
check = true;
}
}
if (count < _chunkSize || check)
{
dataChunk.IsLastChunk = true;
_isLastChunk = true;
}
else
{
dataChunk.IsLastChunk = false;
}
dataChunk.ReaderUID = _id;
dataChunk.ChunkId = lastChunkId + 1;
_lastChunkId = lastChunkId + 1;
return(true);
}
public void TestThatDataTransformCallsProviderAndExporters()
{
_target.Transform();
Assert.IsTrue(_exporters[0].Input.IsEqualTo(_dataInput), "Test failed: first exporter did not receive the data input.");
Assert.IsTrue(_exporters[1].Input.IsEqualTo(_dataInput), "Test failed: second data exporter did not receive the data input.");
}