protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
if (_alreadySorted)
{
if (await PrimaryTransform.ReadAsync(cancellationToken))
{
var values = new object[PrimaryTransform.FieldCount];
PrimaryTransform.GetValues(values);
return(values);
}
else
{
return(null);
}
}
if (_firstRead) //load the entire record into a sorted list.
{
_sortedDictionary = new SortedDictionary <object[], object[]>(new SortKeyComparer(_sortFields));
var rowcount = 0;
while (await PrimaryTransform.ReadAsync(cancellationToken))
{
var values = new object[PrimaryTransform.FieldCount];
var sortFields = new object[_sortFields.Count + 1];
PrimaryTransform.GetValues(values);
for (var i = 0; i < sortFields.Length - 1; i++)
{
sortFields[i] = PrimaryTransform[_sortFields[i].Column];
}
sortFields[sortFields.Length - 1] = rowcount; //add row count as last key field to ensure matching rows remain in original order.
_sortedDictionary.Add(sortFields, values);
rowcount++;
TransformRowsSorted++;
}
_firstRead = false;
if (rowcount == 0)
{
return(null);
}
_iterator = _sortedDictionary.Keys.GetEnumerator();
_iterator.MoveNext();
return(_sortedDictionary[_iterator.Current]);
}
var success = _iterator.MoveNext();
if (success)
{
return(_sortedDictionary[_iterator.Current]);
}
else
{
_sortedDictionary = null; //free up memory after all rows are read.
return(null);
}
}
public override Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken)
{
AuditKey = auditKey;
if (query == null)
{
query = new SelectQuery();
}
var requiredSorts = RequiredSortFields();
if (query.Sorts != null && query.Sorts.Count > 0)
{
for (var i = 0; i < requiredSorts.Count; i++)
{
if (query.Sorts[i].Column == requiredSorts[i].Column)
{
requiredSorts[i].Direction = query.Sorts[i].Direction;
}
else
{
break;
}
}
}
query.Sorts = requiredSorts;
var returnValue = PrimaryTransform.Open(auditKey, query, cancellationToken);
return(returnValue);
}
public override bool InitializeOutputFields()
{
CacheTable = PrimaryTransform.CacheTable.Copy();
//add the operation type, which indicates whether record is rejected 'R' or 'C/U/D' create/update/delete
if (CacheTable.Columns.SingleOrDefault(c => c.DeltaType == TableColumn.EDeltaType.DatabaseOperation) == null)
{
CacheTable.Columns.Insert(0, new TableColumn("Operation", ETypeCode.Byte)
{
DeltaType = TableColumn.EDeltaType.DatabaseOperation
});
}
//add the rejection reason, which details the reason for a rejection.
if (CacheTable.Columns.SingleOrDefault(c => c.DeltaType == TableColumn.EDeltaType.RejectedReason) == null)
{
CacheTable.Columns.Add(new TableColumn("RejectReason", ETypeCode.String)
{
DeltaType = TableColumn.EDeltaType.RejectedReason
});
}
//add the rejection reason, which details the reason for a rejection.
if (CacheTable.Columns.SingleOrDefault(c => c.DeltaType == TableColumn.EDeltaType.ValidationStatus) == null)
{
CacheTable.Columns.Add(new TableColumn("ValidationStatus", ETypeCode.String)
{
DeltaType = TableColumn.EDeltaType.ValidationStatus
});
}
//store reject column details to improve performance.
_rejectReasonOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.RejectedReason);
if (_rejectReasonOrdinal >= 0)
{
_rejectReasonColumnName = CacheTable.Columns[_rejectReasonOrdinal].Name;
}
_operationOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.DatabaseOperation);
_validationStatusOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ValidationStatus);
_primaryFieldCount = PrimaryTransform.FieldCount;
_columnCount = CacheTable.Columns.Count;
_mapFieldOrdinals = new List <int>();
for (var i = 0; i < _primaryFieldCount; i++)
{
_mapFieldOrdinals.Add(GetOrdinal(PrimaryTransform.GetName(i)));
}
return(true);
}
public override async Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken)
{
AuditKey = auditKey;
if (query == null)
{
query = new SelectQuery();
}
query.Sorts = RequiredSortFields();
var returnValue = await PrimaryTransform.Open(auditKey, query, cancellationToken);
//check if the transform has already sorted the data, using sql or a presort.
_alreadySorted = SortFieldsMatch(_sortFields, PrimaryTransform.SortFields);
return(returnValue);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
if (_lastRecord)
{
return(null);
}
_lastRecord = !await PrimaryTransform.ReadAsync(cancellationToken);
if (!_lastRecord)
{
var newRow = new object[CacheTable.Columns.Count];
PrimaryTransform.GetValues(newRow);
foreach (var profile in _profiles)
{
foreach (var input in profile.Inputs.Where(c => c.IsColumn))
{
try
{
input.SetValue(PrimaryTransform[input.Column.Name]);
}
catch (Exception ex)
{
throw new TransformException($"The profile transform {Name} failed setting inputs on the function {profile.FunctionName} parameter {input.Name} column {input.Column.Name}. {ex.Message}", ex, PrimaryTransform[input.Column.Name]);
}
}
try
{
profile.Invoke();
}
catch (Exception ex)
{
throw new TransformException($"The profile transform {Name} failed on the function {profile.FunctionName}. {ex.Message}", ex);
}
}
return(newRow);
}
else
{
var profileResults = GetProfileTable("ProfileResults");
foreach (var profile in _profiles)
{
object profileResult = null;
try
{
profileResult = profile.ReturnValue();
}
catch (Exception ex)
{
throw new TransformException($"The profile transform {Name} failed getting the return value on the function {profile.FunctionName}. {ex.Message}", ex);
}
var row = new object[6];
row[0] = AuditKey;
row[1] = profile.FunctionName;
row[2] = profile.Inputs[0].Column.Name;
row[3] = true;
row[4] = profileResult;
profileResults.Data.Add(row);
if (profile.Outputs.Length > 0)
{
var details = (Dictionary <string, int>)profile.Outputs[0].Value;
if (details != null && details.Count > 0)
{
foreach (var value in details.Keys)
{
row = new object[6];
row[0] = AuditKey;
row[1] = profile.FunctionName;
row[2] = profile.Inputs[0].Column.Name;
row[3] = false;
row[4] = value;
row[5] = details[value];
profileResults.Data.Add(row);
}
}
}
_profileResults = profileResults;
}
return(null);
}
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
object[] newRow = null;
// if there is a previous lookup cache, then just populated that as the next row.
if (_lookupCache != null && _lookupCache.MoveNext())
{
newRow = new object[FieldCount];
var pos1 = 0;
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos1] = PrimaryTransform[i];
pos1++;
}
var lookup = _lookupCache.Current;
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos1] = lookup[i];
pos1++;
}
return(newRow);
}
else
{
_lookupCache = null;
}
if (await PrimaryTransform.ReadAsync(cancellationToken) == false)
{
return(null);
}
//load in the primary table values
newRow = new object[FieldCount];
var pos = 0;
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
//set the values for the lookup
var selectQuery = new SelectQuery();
foreach (var joinPair in JoinPairs)
{
var value = joinPair.SourceColumn == null ? joinPair.JoinValue : PrimaryTransform[joinPair.SourceColumn];
selectQuery.Filters.Add(new Filter
{
Column1 = joinPair.JoinColumn,
CompareDataType = ETypeCode.String,
Operator = Filter.ECompare.IsEqual,
Value2 = value
});
}
var lookupResult = await ReferenceTransform.Lookup(selectQuery, JoinDuplicateStrategy ?? EDuplicateStrategy.Abend, cancellationToken);
if (lookupResult != null)
{
_lookupCache = lookupResult.GetEnumerator();
if (_lookupCache.MoveNext())
{
var lookup = _lookupCache.Current;
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = lookup[i];
pos++;
}
}
else
{
_lookupCache = null;
}
}
else
{
_lookupCache = null;
}
return(newRow);
}
// public override List<Sort> SortFields => new List<Sort>();
public override async Task <bool> Open(long auditKey, SelectQuery query, CancellationToken cancellationToken)
{
AuditKey = auditKey;
var primarySorts = new List <Sort>();
var referenceSorts = new List <Sort>();
//we need to translate filters and sorts to source column names before passing them through.
if (query?.Sorts != null)
{
foreach (var sort in query.Sorts)
{
if (sort.Column != null)
{
var column = PrimaryTransform.CacheTable[sort.Column.Name];
if (column != null)
{
primarySorts.Add(new Sort(column, sort.Direction));
}
column = ReferenceTransform.CacheTable[sort.Column.Name];
if (column != null)
{
referenceSorts.Add(new Sort(column, sort.Direction));
}
}
}
}
var primaryQuery = new SelectQuery()
{
Sorts = primarySorts
};
var referenceQuery = new SelectQuery()
{
Sorts = referenceSorts
};
var returnValue = await PrimaryTransform.Open(auditKey, primaryQuery, cancellationToken);
if (!returnValue)
{
return(false);
}
returnValue = await ReferenceTransform.Open(auditKey, referenceQuery, cancellationToken);
//if the primary & reference transforms are sorted, we will merge sort the items.
if (PrimaryTransform.SortFields != null && ReferenceTransform.SortFields != null)
{
var newSortFields = new List <Sort>();
_primarySortOrdinals = new List <int>();
_referenceSortOrdinals = new List <int>();
var index = 0;
var referenceSortFields = ReferenceTransform.SortFields;
foreach (var sortField in PrimaryTransform.SortFields)
{
if (referenceSortFields.Count <= index)
{
break;
}
var referenceSortField = referenceSortFields[index];
if (sortField.Column.Name == referenceSortField.Column.Name &&
sortField.Direction == referenceSortField.Direction)
{
newSortFields.Add(sortField);
_primarySortOrdinals.Add(PrimaryTransform.CacheTable.GetOrdinal(sortField.Column.Name));
_referenceSortOrdinals.Add(ReferenceTransform.CacheTable.GetOrdinal(sortField.Column.Name));
}
else
{
break;
}
index++;
}
if (newSortFields.Count > 0)
{
_sortedMerge = true;
CacheTable.OutputSortFields = newSortFields;
}
}
return(returnValue);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
// sorted merge will concatenate 2 sorted incoming datasets, and maintain the sort order.
if (_sortedMerge)
{
if (_firstRead)
{
// read one row for each reader.
var primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
var referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
await Task.WhenAll(primaryReadTask, referenceReadTask);
if (primaryReadTask.IsFaulted)
{
throw primaryReadTask.Exception;
}
if (referenceReadTask.IsFaulted)
{
throw referenceReadTask.Exception;
}
_primaryMoreRecords = primaryReadTask.Result;
_referenceMoreRecords = referenceReadTask.Result;
_firstRead = false;
}
if (_primaryReadTask != null)
{
_primaryMoreRecords = await _primaryReadTask;
}
if (_referenceReadTask != null)
{
_referenceMoreRecords = await _referenceReadTask;
}
if (!_primaryMoreRecords && !_referenceMoreRecords)
{
return(null);
}
var newRow = new object[FieldCount];
// no more primary records, then just read from the reference
if (!_primaryMoreRecords)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_primaryReadTask = null;
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
// no more reference record ,just read from the primary.
else if (!_referenceMoreRecords)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
PrimaryTransform.GetValues(newRow);
_referenceReadTask = null;
_primaryReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
else
{
//more records in both, then compare the rows and take the next in sort order.
var usePrimary = true;
for (var i = 0; i < _primarySortOrdinals.Count; i++)
{
var compareResult = Dexih.Utils.DataType.DataType.Compare(
PrimaryTransform.CacheTable.Columns[_primarySortOrdinals[i]].DataType,
PrimaryTransform[_primarySortOrdinals[i]], ReferenceTransform[_referenceSortOrdinals[i]]);
if ((compareResult == DataType.ECompareResult.Greater &&
SortFields[i].Direction == Sort.EDirection.Ascending) ||
(compareResult == DataType.ECompareResult.Less &&
SortFields[i].Direction == Sort.EDirection.Descending))
{
usePrimary = false;
break;
}
}
if (usePrimary)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
else
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
}
}
else
{
// if no sorting specified, concatenate will be in any order as the records arrive.
if (_firstRead)
{
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
_firstRead = false;
}
if (_primaryReadTask != null && _referenceReadTask != null)
{
await Task.WhenAny(_primaryReadTask, _referenceReadTask);
if (_primaryReadTask.IsCanceled || _referenceReadTask.IsCanceled ||
cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException("The read record task was cancelled");
}
if (_primaryReadTask.IsFaulted)
{
throw _primaryReadTask.Exception;
}
if (_referenceReadTask.IsFaulted)
{
throw _referenceReadTask.Exception;
}
if (_primaryReadTask.IsCompleted)
{
var result = _primaryReadTask.Result;
if (result)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
if (_referenceReadTask.IsCompleted)
{
var result = _referenceReadTask.Result;
if (result)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
}
if (_primaryReadTask != null)
{
var result = await _primaryReadTask;
if (result)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
if (_referenceReadTask != null)
{
var result = await _referenceReadTask;
if (result)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_referenceReadTask = null;
}
}
return(null);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
//the saved reject row is when a validation outputs two rows (pass & fail).
if (_savedRejectRow != null)
{
var row = _savedRejectRow;
_savedRejectRow = null;
return(row);
}
if (_lastRecord)
{
return(null);
}
while (await PrimaryTransform.ReadAsync(cancellationToken))
{
var rejectReason = new StringBuilder();
var finalInvalidAction = TransformFunction.EInvalidAction.Pass;
//copy row data.
var passRow = new object[_columnCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
passRow[_mapFieldOrdinals[i]] = PrimaryTransform[i];
}
if (passRow[_operationOrdinal] == null)
{
passRow[_operationOrdinal] = 'C';
}
object[] rejectRow = null;
//run the validation functions
if (Validations != null)
{
foreach (var validation in Validations)
{
//set inputs for the validation function
foreach (var input in validation.Inputs.Where(c => c.IsColumn))
{
try
{
input.SetValue(PrimaryTransform[input.Column]);
}
catch (Exception ex)
{
throw new TransformException($"The validation transform {Name} failed setting input parameters on the function {validation.FunctionName} parameter {input.Name} for column {input.Column.TableColumnName()}. {ex.Message}", ex, PrimaryTransform[input.Column.TableColumnName()]);
}
}
bool validationResult;
try
{
var invokeresult = validation.Invoke();
validationResult = (bool)invokeresult;
}
catch (FunctionIgnoreRowException)
{
validationResult = false;
}
catch (Exception ex)
{
throw new TransformException($"The validation transform {Name} failed on the function {validation.FunctionName}. {ex.Message}", ex);
}
//if the validation is false. apply any output columns, and set a reject status
if (!validationResult)
{
rejectReason.AppendLine("function: " + validation.FunctionName + ", parameters: " + string.Join(",", validation.Inputs.Select(c => c.Name + "=" + (c.IsColumn ? c.Column.TableColumnName() : c.Value.ToString())).ToArray()) + ".");
// fail job immediately.
if (validation.InvalidAction == TransformFunction.EInvalidAction.Abend)
{
var reason = $"The validation rule abended as the invalid action is set to abend. " + rejectReason.ToString();
throw new Exception(reason);
}
//if the record is to be discarded, continue the loop and get the next source record.
if (validation.InvalidAction == TransformFunction.EInvalidAction.Discard)
{
continue;
}
//set the final invalidation action based on priority order of other rejections.
finalInvalidAction = finalInvalidAction < validation.InvalidAction ? validation.InvalidAction : finalInvalidAction;
if (validation.InvalidAction == TransformFunction.EInvalidAction.Reject || validation.InvalidAction == TransformFunction.EInvalidAction.RejectClean)
{
//if the row is rejected, copy unmodified row to a reject row.
if (rejectRow == null)
{
rejectRow = new object[CacheTable.Columns.Count];
passRow.CopyTo(rejectRow, 0);
rejectRow[_operationOrdinal] = 'R';
TransformRowsRejected++;
}
//add a reject reason if it exists
if (_rejectReasonOrdinal >= 0)
{
if (validation.Outputs != null)
{
var param = validation.Outputs.SingleOrDefault(c => c.Column.TableColumnName() == _rejectReasonColumnName);
if (param != null)
{
rejectReason.Append(" Reason: " + (string)param.Value);
}
}
}
}
if (validation.InvalidAction == TransformFunction.EInvalidAction.Clean || validation.InvalidAction == TransformFunction.EInvalidAction.RejectClean)
{
validation.ReturnValue();
if (validation.Outputs != null)
{
foreach (var output in validation.Outputs)
{
if (output.Column.TableColumnName() != "")
{
var ordinal = CacheTable.GetOrdinal(output.Column);
var col = CacheTable.Columns[ordinal];
if (ordinal >= 0)
{
try
{
var parseresult = TryParse(col.DataType, output.Value, col.MaxLength);
passRow[ordinal] = parseresult;
}
catch (Exception ex)
{
throw new TransformException($"The validation transform {Name} failed parsing output values on the function {validation.FunctionName} parameter {output.Name} column {col.Name}. {ex.Message}", ex, output.Value);
}
}
}
}
}
}
}
}
}
if (ValidateDataTypes && (finalInvalidAction == TransformFunction.EInvalidAction.Pass || finalInvalidAction == TransformFunction.EInvalidAction.Clean))
{
for (var i = 0; i < _columnCount; i++)
{
var value = passRow[i];
var col = CacheTable.Columns[i];
if (col.DeltaType == TableColumn.EDeltaType.TrackingField || col.DeltaType == TableColumn.EDeltaType.NonTrackingField)
{
if (value == null || value is DBNull)
{
if (col.AllowDbNull == false)
{
if (rejectRow == null)
{
rejectRow = new object[_columnCount];
passRow.CopyTo(rejectRow, 0);
rejectRow[_operationOrdinal] = 'R';
TransformRowsRejected++;
}
rejectReason.AppendLine("Column:" + col.Name + ": Tried to insert null into non-null column.");
finalInvalidAction = TransformFunction.EInvalidAction.Reject;
}
passRow[i] = DBNull.Value;
}
else
{
try
{
var parseresult = TryParse(col.DataType, value, col.MaxLength);
passRow[i] = parseresult;
}
catch (Exception ex)
{
// if the parse fails on the column, then write out a reject record.
if (rejectRow == null)
{
rejectRow = new object[_columnCount];
passRow.CopyTo(rejectRow, 0);
rejectRow[_operationOrdinal] = 'R';
TransformRowsRejected++;
}
rejectReason.AppendLine(ex.Message);
finalInvalidAction = TransformFunction.EInvalidAction.Reject;
}
}
}
}
}
switch (finalInvalidAction)
{
case TransformFunction.EInvalidAction.Pass:
passRow[_validationStatusOrdinal] = "passed";
return(passRow);
case TransformFunction.EInvalidAction.Clean:
passRow[_validationStatusOrdinal] = "cleaned";
return(passRow);
case TransformFunction.EInvalidAction.RejectClean:
passRow[_validationStatusOrdinal] = "rejected-cleaned";
rejectRow[_validationStatusOrdinal] = "rejected-cleaned";
rejectRow[_rejectReasonOrdinal] = rejectReason.ToString();
_savedRejectRow = rejectRow;
return(passRow);
case TransformFunction.EInvalidAction.Reject:
passRow[_validationStatusOrdinal] = "rejected";
rejectRow[_validationStatusOrdinal] = "rejected";
rejectRow[_rejectReasonOrdinal] = rejectReason.ToString();
return(rejectRow);
}
//should never get here.
throw new TransformException("Validation failed due to an unknown error.");
}
return(null);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
object[] newRow = null;
var pos = 0;
//this writes out duplicates of the primary reader when a duplicate match occurrs on the join table
//i.e. outer join.
if (_writeGroup)
{
//create a new row and write the primary fields out
newRow = new object[FieldCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
var joinRow = _filterdGroupData[_writeGroupPosition];
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = joinRow[i];
pos++;
}
_writeGroupPosition++;
//if last join record, then set the flag=false so the next read will read another primary row record.
if (_writeGroupPosition >= _filterdGroupData.Count)
{
_writeGroup = false;
}
return(newRow);
}
//read a new row from the primary table.
if (await PrimaryTransform.ReadAsync(cancellationToken) == false)
{
return(null);
}
var joinMatchFound = false;
//if input is sorted, then run a sortedjoin
if (JoinAlgorithm == EJoinAlgorithm.Sorted)
{
//first read get a row from the join table.
if (_firstRead)
{
//get the first two rows from the join table.
_joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken);
_groupsOpen = await ReadNextGroup();
_firstRead = false;
}
//loop through join table until we find a matching row.
if (_joinColumns != null)
{
while (_groupsOpen)
{
var joinFields = new object[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
joinFields[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]];
}
var compare = _joinKeyComparer.Compare(_groupFields, joinFields);
var done = false;
switch (compare)
{
case 1:
joinMatchFound = false;
done = true;
break;
case -1:
if (_groupsOpen)
{
_groupsOpen = await ReadNextGroup();
}
break;
case 0:
joinMatchFound = true;
done = true;
break;
}
if (done)
{
break;
}
}
}
}
else //if input is not sorted, then run a hash join.
{
//first read load the join table into memory
if (_firstRead)
{
_joinHashData = new SortedDictionary <object[], List <object[]> >(new JoinKeyComparer());
_joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken);
_groupsOpen = await ReadNextGroup();
//load all the join data into an a dictionary
while (_groupsOpen)
{
_joinHashData.Add(_groupFields, _groupData);
_groupsOpen = await ReadNextGroup();
}
_firstRead = false;
}
object[] sourceKeys;
//set the values for the lookup
if (_joinColumns != null)
{
sourceKeys = new object[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
sourceKeys[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]];
}
}
else
{
sourceKeys = new object[0];
}
if (_joinHashData.Keys.Contains(sourceKeys))
{
_groupData = _joinHashData[sourceKeys];
_groupsOpen = true;
joinMatchFound = true;
}
else
{
joinMatchFound = false;
}
}
//create a new row and write the primary fields out
newRow = new object[FieldCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
if (joinMatchFound)
{
//if there are additional join functions, we run them
if (_joinFilters.Count == 0)
{
_filterdGroupData = _groupData;
}
else
{
_filterdGroupData = new List <object[]>();
//filter out the current group based on the functions defined.
foreach (var row in _groupData)
{
var matchFound = true;
foreach (var condition in _joinFilters)
{
foreach (var input in condition.Inputs.Where(c => c.IsColumn))
{
object value = null;
try
{
if (input.Column.ReferenceTable == _referenceTableName)
{
value = row[ReferenceTransform.GetOrdinal(input.Column)];
}
else
{
value = PrimaryTransform[input.Column];
}
input.SetValue(value);
}
catch (Exception ex)
{
throw new TransformException($"The join tansform {Name} failed setting parameters on the condition {condition.FunctionName} with the parameter {input.Name}. {ex.Message}.", ex, value);
}
}
try
{
var invokeresult = condition.Invoke();
if ((bool)invokeresult == false)
{
matchFound = false;
break;
}
}
catch (FunctionIgnoreRowException)
{
matchFound = false;
TransformRowsIgnored++;
break;
}
catch (Exception ex)
{
throw new TransformException($"The join transform {Name} failed calling the function {condition.FunctionName}. {ex.Message}.", ex);
}
}
if (matchFound)
{
_filterdGroupData.Add(row);
}
}
}
object[] joinRow = null;
if (_filterdGroupData.Count > 0)
{
if (_filterdGroupData.Count > 1)
{
switch (JoinDuplicateStrategy)
{
case EDuplicateStrategy.Abend:
throw new DuplicateJoinKeyException("The join transform failed as the selected columns on the join table " + ReferenceTableAlias + " are not unique. To continue when duplicates occur set the join strategy to first, last or all.", ReferenceTableAlias, _groupFields);
case EDuplicateStrategy.First:
joinRow = _filterdGroupData[0];
break;
case EDuplicateStrategy.Last:
joinRow = _filterdGroupData.Last();
break;
case EDuplicateStrategy.All:
joinRow = _filterdGroupData[0];
_writeGroup = true;
_writeGroupPosition = 1;
break;
default:
throw new TransformException("The join transform failed due to an unknown join strategy " + JoinDuplicateStrategy);
}
}
else
{
joinRow = _filterdGroupData[0];
}
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = joinRow[i];
pos++;
}
}
}
return(newRow);
}
public override async Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken)
{
AuditKey = auditKey;
if (query == null)
{
query = new SelectQuery();
}
//only apply a sort if there is not already a sort applied.
// if(query.Sorts == null || query.Sorts.Count == 0)
query.Sorts = RequiredSortFields();
var returnValue = await PrimaryTransform.Open(auditKey, query, cancellationToken);
if (!returnValue)
{
return(false);
}
var referenceQuery = new SelectQuery()
{
Sorts = RequiredReferenceSortFields()
};
returnValue = await ReferenceTransform.Open(auditKey, referenceQuery, cancellationToken);
if (!returnValue)
{
return(false);
}
//check if the primary and reference transform are sorted in the join
if (SortFieldsMatch(RequiredSortFields(), PrimaryTransform.SortFields) && SortFieldsMatch(RequiredReferenceSortFields(), ReferenceTransform.SortFields))
{
JoinAlgorithm = EJoinAlgorithm.Sorted;
}
else
{
JoinAlgorithm = EJoinAlgorithm.Hash;
}
//store the ordinals for the joins to improve performance.
if (_joinColumns == null)
{
_joinKeyOrdinals = new int[0];
_sourceKeyOrdinals = new int[0];
}
else
{
_joinKeyOrdinals = new int[_joinColumns.Length];
_sourceKeyOrdinals = new int[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
_joinKeyOrdinals[i] = ReferenceTransform.GetOrdinal(_joinColumns[i].JoinColumn.Name);
_sourceKeyOrdinals[i] = _joinColumns[i].SourceColumn == null ? -1 : PrimaryTransform.GetOrdinal(_joinColumns[i].SourceColumn.Name);
}
}
return(true);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
object[] newRow = null;
//if there are records in the passthrough cache, then empty them out before getting new records.
if (PassThroughColumns)
{
if (_firstRecord)
{
_passThroughValues = new List <object[]>();
}
else if (_passThroughIndex > 0 && _passThroughIndex < _passThroughCount)
{
newRow = _passThroughValues[_passThroughIndex];
_passThroughIndex++;
return(newRow);
}
//if all rows have been iterated through, reset the cache and add the stored row for the next group
else if (_passThroughIndex >= _passThroughCount && _firstRecord == false && _lastRecord == false)
{
_passThroughValues.Clear();
_passThroughValues.Add(_nextPassThroughRow);
}
}
newRow = new object[FieldCount];
int i;
var groupChanged = false;
object[] newGroupValues = null;
if (await PrimaryTransform.ReadAsync(cancellationToken) == false)
{
if (_lastRecord) //return false is all record have been written.
{
return(null);
}
}
else
{
do
{
_lastRecord = false;
i = 0;
//if it's the first record then the groupvalues are being set for the first time.
if (_firstRecord)
{
groupChanged = false;
_firstRecord = false;
if (GroupFields != null)
{
foreach (var groupField in GroupFields)
{
// _groupValues[i] = PrimaryTransform[groupField.SourceColumn]?.ToString() ?? "";
_groupValues[i] = PrimaryTransform[groupField.SourceColumn];
i++;
}
}
newGroupValues = _groupValues;
}
else
{
//if not first row, then check if the group values have changed from the previous row
if (GroupFields != null)
{
newGroupValues = new object[GroupFields.Count];
foreach (var groupField in GroupFields)
{
//newGroupValues[i] = PrimaryTransform[groupField.SourceColumn]?.ToString() ?? "";
newGroupValues[i] = PrimaryTransform[groupField.SourceColumn];
if ((newGroupValues[i] == null && _groupValues != null) ||
(newGroupValues[i] != null && _groupValues == null) ||
!Equals(newGroupValues[i], _groupValues[i]))
{
groupChanged = true;
}
i++;
}
}
}
//if the group values have changed, write out the previous group values.
if (groupChanged)
{
i = 0;
if (GroupFields != null)
{
for (; i < GroupFields.Count; i++)
{
newRow[i] = _groupValues[i];
}
}
if (PassThroughColumns == false)
{
if (AggregatePairs != null)
{
foreach (var aggregate in AggregatePairs)
{
newRow[i] = aggregate.GetValue();
i++;
}
}
if (Aggregates != null)
{
foreach (var mapping in Aggregates)
{
try
{
newRow[i] = mapping.ReturnValue(0);
i++;
if (mapping.Outputs != null)
{
foreach (var output in mapping.Outputs)
{
newRow[i] = output.Value;
i++;
}
}
}
catch (Exception ex)
{
throw new TransformException($"The group transform {Name} failed on function {mapping.FunctionName}. {ex.Message}.", ex);
}
}
}
}
//for passthrough, write out the aggregated values to the cached passthrough set
else
{
var index = 0;
var startColumn = i;
foreach (var row in _passThroughValues)
{
i = startColumn;
if (AggregatePairs != null)
{
foreach (var aggregate in AggregatePairs)
{
row[i] = aggregate.GetValue();
i++;
}
}
if (Aggregates != null)
{
foreach (var mapping in Aggregates)
{
try
{
row[i] = mapping.ReturnValue(index);
i++;
if (mapping.Outputs != null)
{
foreach (var output in mapping.Outputs)
{
row[i] = output.Value;
i++;
}
}
}
catch (Exception ex)
{
throw new TransformException($"The group transform {Name} failed, retrieving results from function {mapping.FunctionName}. {ex.Message}.", ex);
}
}
}
index++;
}
//the first row of the next group has been read, so this is to store it until ready to write out.
_nextPassThroughRow = new object[FieldCount];
for (var j = 0; j < newGroupValues.Length; j++)
{
_nextPassThroughRow[j] = newGroupValues[j];
}
for (var j = _passThroughStartIndex; j < FieldCount; j++)
{
_nextPassThroughRow[j] = PrimaryTransform[GetName(j)];
}
////set the first cached row to current
newRow = _passThroughValues[0];
_passThroughIndex = 1;
_passThroughCount = _passThroughValues.Count;
}
//reset the functions
if (AggregatePairs != null)
{
foreach (var aggregate in AggregatePairs)
{
aggregate.Reset();
}
}
if (Aggregates != null)
{
foreach (var mapping in Aggregates)
{
mapping.Reset();
}
}
//store the last groupvalues read to start the next grouping.
_groupValues = newGroupValues;
}
else
{
if (PassThroughColumns)
{
//create a cached current row. this will be output when the group has changed.
var cacheRow = new object[newRow.Length];
if (_groupValues != null)
{
for (var j = 0; j < _groupValues.Length; j++)
{
cacheRow[j] = _groupValues[j];
}
}
for (var j = _passThroughStartIndex; j < FieldCount; j++)
{
cacheRow[j] = PrimaryTransform[GetName(j)];
}
_passThroughValues.Add(cacheRow);
}
}
// update the aggregate pairs
if (AggregatePairs != null)
{
foreach (var aggregate in AggregatePairs)
{
aggregate.AddValue(PrimaryTransform[aggregate.SourceColumn]);
}
}
// update the aggregate functions
if (Aggregates != null)
{
foreach (var mapping in Aggregates)
{
if (mapping.Inputs != null)
{
foreach (var input in mapping.Inputs.Where(c => c.IsColumn))
{
try
{
input.SetValue(PrimaryTransform[input.Column]);
}
catch (Exception ex)
{
throw new TransformException($"The group transform {Name} failed setting an incompatible value to column {input.Column.Name}. {ex.Message}.", ex, PrimaryTransform[input.Column]);
}
}
}
try
{
var invokeresult = mapping.Invoke();
}
catch (FunctionIgnoreRowException)
{
//TODO: Issue that some of the aggregate values may be calculated prior to the ignorerow being set.
TransformRowsIgnored++;
continue;
}
catch (Exception ex)
{
throw new TransformException($"The group transform {Name} failed running the function {mapping.FunctionName}. {ex.Message}.", ex);
}
}
}
if (groupChanged)
{
break;
}
} while (await PrimaryTransform.ReadAsync(cancellationToken));
}
if (groupChanged == false) //if the reader has finished with no group change, write the values and set last record
{
i = 0;
if (GroupFields != null)
{
for (; i < GroupFields.Count; i++)
{
newRow[i] = _groupValues[i];
}
}
if (PassThroughColumns == false)
{
if (AggregatePairs != null)
{
foreach (var aggregate in AggregatePairs)
{
newRow[i] = aggregate.GetValue();
i++;
aggregate.Reset();
}
}
if (Aggregates != null)
{
foreach (var mapping in Aggregates)
{
try
{
newRow[i] = mapping.ReturnValue(0);
}
catch (Exception ex)
{
throw new TransformException($"The group transform {Name} failed, retrieving results from function {mapping.FunctionName}. {ex.Message}.", ex);
}
i++;
if (mapping.Outputs != null)
{
foreach (var output in mapping.Outputs)
{
newRow[i] = output.Value;
i++;
}
}
mapping.Reset();
}
}
}
else
{
//for passthrough, write out the aggregated values to the cached passthrough set
var index = 0;
var startColumn = i;
foreach (var row in _passThroughValues)
{
i = startColumn;
if (AggregatePairs != null)
{
foreach (var aggregate in AggregatePairs)
{
newRow[i] = aggregate.GetValue();
i++;
}
}
if (Aggregates != null)
{
foreach (var mapping in Aggregates)
{
try
{
row[i] = mapping.ReturnValue(index);
}
catch (Exception ex)
{
throw new TransformException($"The group transform {Name} failed retrieving results from function {mapping.FunctionName}. {ex.Message}.", ex);
}
i++;
if (mapping.Outputs != null)
{
foreach (var output in mapping.Outputs)
{
row[i] = output.Value;
i++;
}
}
}
}
index++;
}
//set the first cached row to current
newRow = _passThroughValues[0];
_passThroughIndex = 1;
_passThroughCount = _passThroughValues.Count;
}
_groupValues = newGroupValues;
_lastRecord = true;
}
return(newRow);
}
