protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
if (_alreadySorted)
{
if (await PrimaryTransform.ReadAsync(cancellationToken))
{
var values = new object[PrimaryTransform.FieldCount];
PrimaryTransform.GetValues(values);
return(values);
}
else
{
return(null);
}
}
if (_firstRead) //load the entire record into a sorted list.
{
_sortedDictionary = new SortedDictionary <object[], object[]>(new SortKeyComparer(_sortFields));
var rowcount = 0;
while (await PrimaryTransform.ReadAsync(cancellationToken))
{
var values = new object[PrimaryTransform.FieldCount];
var sortFields = new object[_sortFields.Count + 1];
PrimaryTransform.GetValues(values);
for (var i = 0; i < sortFields.Length - 1; i++)
{
sortFields[i] = PrimaryTransform[_sortFields[i].Column];
}
sortFields[sortFields.Length - 1] = rowcount; //add row count as last key field to ensure matching rows remain in original order.
_sortedDictionary.Add(sortFields, values);
rowcount++;
TransformRowsSorted++;
}
_firstRead = false;
if (rowcount == 0)
{
return(null);
}
_iterator = _sortedDictionary.Keys.GetEnumerator();
_iterator.MoveNext();
return(_sortedDictionary[_iterator.Current]);
}
var success = _iterator.MoveNext();
if (success)
{
return(_sortedDictionary[_iterator.Current]);
}
else
{
_sortedDictionary = null; //free up memory after all rows are read.
return(null);
}
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
if (_lastRecord)
{
return(null);
}
_lastRecord = !await PrimaryTransform.ReadAsync(cancellationToken);
if (!_lastRecord)
{
var newRow = new object[CacheTable.Columns.Count];
PrimaryTransform.GetValues(newRow);
foreach (var profile in _profiles)
{
foreach (var input in profile.Inputs.Where(c => c.IsColumn))
{
try
{
input.SetValue(PrimaryTransform[input.Column.Name]);
}
catch (Exception ex)
{
throw new TransformException($"The profile transform {Name} failed setting inputs on the function {profile.FunctionName} parameter {input.Name} column {input.Column.Name}. {ex.Message}", ex, PrimaryTransform[input.Column.Name]);
}
}
try
{
profile.Invoke();
}
catch (Exception ex)
{
throw new TransformException($"The profile transform {Name} failed on the function {profile.FunctionName}. {ex.Message}", ex);
}
}
return(newRow);
}
else
{
var profileResults = GetProfileTable("ProfileResults");
foreach (var profile in _profiles)
{
object profileResult = null;
try
{
profileResult = profile.ReturnValue();
}
catch (Exception ex)
{
throw new TransformException($"The profile transform {Name} failed getting the return value on the function {profile.FunctionName}. {ex.Message}", ex);
}
var row = new object[6];
row[0] = AuditKey;
row[1] = profile.FunctionName;
row[2] = profile.Inputs[0].Column.Name;
row[3] = true;
row[4] = profileResult;
profileResults.Data.Add(row);
if (profile.Outputs.Length > 0)
{
var details = (Dictionary <string, int>)profile.Outputs[0].Value;
if (details != null && details.Count > 0)
{
foreach (var value in details.Keys)
{
row = new object[6];
row[0] = AuditKey;
row[1] = profile.FunctionName;
row[2] = profile.Inputs[0].Column.Name;
row[3] = false;
row[4] = value;
row[5] = details[value];
profileResults.Data.Add(row);
}
}
}
_profileResults = profileResults;
}
return(null);
}
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
// sorted merge will concatenate 2 sorted incoming datasets, and maintain the sort order.
if (_sortedMerge)
{
if (_firstRead)
{
// read one row for each reader.
var primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
var referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
await Task.WhenAll(primaryReadTask, referenceReadTask);
if (primaryReadTask.IsFaulted)
{
throw primaryReadTask.Exception;
}
if (referenceReadTask.IsFaulted)
{
throw referenceReadTask.Exception;
}
_primaryMoreRecords = primaryReadTask.Result;
_referenceMoreRecords = referenceReadTask.Result;
_firstRead = false;
}
if (_primaryReadTask != null)
{
_primaryMoreRecords = await _primaryReadTask;
}
if (_referenceReadTask != null)
{
_referenceMoreRecords = await _referenceReadTask;
}
if (!_primaryMoreRecords && !_referenceMoreRecords)
{
return(null);
}
var newRow = new object[FieldCount];
// no more primary records, then just read from the reference
if (!_primaryMoreRecords)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_primaryReadTask = null;
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
// no more reference record ,just read from the primary.
else if (!_referenceMoreRecords)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
PrimaryTransform.GetValues(newRow);
_referenceReadTask = null;
_primaryReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
else
{
//more records in both, then compare the rows and take the next in sort order.
var usePrimary = true;
for (var i = 0; i < _primarySortOrdinals.Count; i++)
{
var compareResult = Dexih.Utils.DataType.DataType.Compare(
PrimaryTransform.CacheTable.Columns[_primarySortOrdinals[i]].DataType,
PrimaryTransform[_primarySortOrdinals[i]], ReferenceTransform[_referenceSortOrdinals[i]]);
if ((compareResult == DataType.ECompareResult.Greater &&
SortFields[i].Direction == Sort.EDirection.Ascending) ||
(compareResult == DataType.ECompareResult.Less &&
SortFields[i].Direction == Sort.EDirection.Descending))
{
usePrimary = false;
break;
}
}
if (usePrimary)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
else
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
}
}
else
{
// if no sorting specified, concatenate will be in any order as the records arrive.
if (_firstRead)
{
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
_firstRead = false;
}
if (_primaryReadTask != null && _referenceReadTask != null)
{
await Task.WhenAny(_primaryReadTask, _referenceReadTask);
if (_primaryReadTask.IsCanceled || _referenceReadTask.IsCanceled ||
cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException("The read record task was cancelled");
}
if (_primaryReadTask.IsFaulted)
{
throw _primaryReadTask.Exception;
}
if (_referenceReadTask.IsFaulted)
{
throw _referenceReadTask.Exception;
}
if (_primaryReadTask.IsCompleted)
{
var result = _primaryReadTask.Result;
if (result)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
if (_referenceReadTask.IsCompleted)
{
var result = _referenceReadTask.Result;
if (result)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
}
if (_primaryReadTask != null)
{
var result = await _primaryReadTask;
if (result)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
if (_referenceReadTask != null)
{
var result = await _referenceReadTask;
if (result)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_referenceReadTask = null;
}
}
return(null);
}
