protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
object[] newRow = null;
var pos = 0;
//this writes out duplicates of the primary reader when a duplicate match occurrs on the join table
//i.e. outer join.
if (_writeGroup)
{
//create a new row and write the primary fields out
newRow = new object[FieldCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
var joinRow = _filterdGroupData[_writeGroupPosition];
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = joinRow[i];
pos++;
}
_writeGroupPosition++;
//if last join record, then set the flag=false so the next read will read another primary row record.
if (_writeGroupPosition >= _filterdGroupData.Count)
{
_writeGroup = false;
}
return(newRow);
}
//read a new row from the primary table.
if (await PrimaryTransform.ReadAsync(cancellationToken) == false)
{
return(null);
}
var joinMatchFound = false;
//if input is sorted, then run a sortedjoin
if (JoinAlgorithm == EJoinAlgorithm.Sorted)
{
//first read get a row from the join table.
if (_firstRead)
{
//get the first two rows from the join table.
_joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken);
_groupsOpen = await ReadNextGroup();
_firstRead = false;
}
//loop through join table until we find a matching row.
if (_joinColumns != null)
{
while (_groupsOpen)
{
var joinFields = new object[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
joinFields[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]];
}
var compare = _joinKeyComparer.Compare(_groupFields, joinFields);
var done = false;
switch (compare)
{
case 1:
joinMatchFound = false;
done = true;
break;
case -1:
if (_groupsOpen)
{
_groupsOpen = await ReadNextGroup();
}
break;
case 0:
joinMatchFound = true;
done = true;
break;
}
if (done)
{
break;
}
}
}
}
else //if input is not sorted, then run a hash join.
{
//first read load the join table into memory
if (_firstRead)
{
_joinHashData = new SortedDictionary <object[], List <object[]> >(new JoinKeyComparer());
_joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken);
_groupsOpen = await ReadNextGroup();
//load all the join data into an a dictionary
while (_groupsOpen)
{
_joinHashData.Add(_groupFields, _groupData);
_groupsOpen = await ReadNextGroup();
}
_firstRead = false;
}
object[] sourceKeys;
//set the values for the lookup
if (_joinColumns != null)
{
sourceKeys = new object[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
sourceKeys[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]];
}
}
else
{
sourceKeys = new object[0];
}
if (_joinHashData.Keys.Contains(sourceKeys))
{
_groupData = _joinHashData[sourceKeys];
_groupsOpen = true;
joinMatchFound = true;
}
else
{
joinMatchFound = false;
}
}
//create a new row and write the primary fields out
newRow = new object[FieldCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
if (joinMatchFound)
{
//if there are additional join functions, we run them
if (_joinFilters.Count == 0)
{
_filterdGroupData = _groupData;
}
else
{
_filterdGroupData = new List <object[]>();
//filter out the current group based on the functions defined.
foreach (var row in _groupData)
{
var matchFound = true;
foreach (var condition in _joinFilters)
{
foreach (var input in condition.Inputs.Where(c => c.IsColumn))
{
object value = null;
try
{
if (input.Column.ReferenceTable == _referenceTableName)
{
value = row[ReferenceTransform.GetOrdinal(input.Column)];
}
else
{
value = PrimaryTransform[input.Column];
}
input.SetValue(value);
}
catch (Exception ex)
{
throw new TransformException($"The join tansform {Name} failed setting parameters on the condition {condition.FunctionName} with the parameter {input.Name}. {ex.Message}.", ex, value);
}
}
try
{
var invokeresult = condition.Invoke();
if ((bool)invokeresult == false)
{
matchFound = false;
break;
}
}
catch (FunctionIgnoreRowException)
{
matchFound = false;
TransformRowsIgnored++;
break;
}
catch (Exception ex)
{
throw new TransformException($"The join transform {Name} failed calling the function {condition.FunctionName}. {ex.Message}.", ex);
}
}
if (matchFound)
{
_filterdGroupData.Add(row);
}
}
}
object[] joinRow = null;
if (_filterdGroupData.Count > 0)
{
if (_filterdGroupData.Count > 1)
{
switch (JoinDuplicateStrategy)
{
case EDuplicateStrategy.Abend:
throw new DuplicateJoinKeyException("The join transform failed as the selected columns on the join table " + ReferenceTableAlias + " are not unique. To continue when duplicates occur set the join strategy to first, last or all.", ReferenceTableAlias, _groupFields);
case EDuplicateStrategy.First:
joinRow = _filterdGroupData[0];
break;
case EDuplicateStrategy.Last:
joinRow = _filterdGroupData.Last();
break;
case EDuplicateStrategy.All:
joinRow = _filterdGroupData[0];
_writeGroup = true;
_writeGroupPosition = 1;
break;
default:
throw new TransformException("The join transform failed due to an unknown join strategy " + JoinDuplicateStrategy);
}
}
else
{
joinRow = _filterdGroupData[0];
}
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = joinRow[i];
pos++;
}
}
}
return(newRow);
}
public override async Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken)
{
AuditKey = auditKey;
if (query == null)
{
query = new SelectQuery();
}
//only apply a sort if there is not already a sort applied.
// if(query.Sorts == null || query.Sorts.Count == 0)
query.Sorts = RequiredSortFields();
var returnValue = await PrimaryTransform.Open(auditKey, query, cancellationToken);
if (!returnValue)
{
return(false);
}
var referenceQuery = new SelectQuery()
{
Sorts = RequiredReferenceSortFields()
};
returnValue = await ReferenceTransform.Open(auditKey, referenceQuery, cancellationToken);
if (!returnValue)
{
return(false);
}
//check if the primary and reference transform are sorted in the join
if (SortFieldsMatch(RequiredSortFields(), PrimaryTransform.SortFields) && SortFieldsMatch(RequiredReferenceSortFields(), ReferenceTransform.SortFields))
{
JoinAlgorithm = EJoinAlgorithm.Sorted;
}
else
{
JoinAlgorithm = EJoinAlgorithm.Hash;
}
//store the ordinals for the joins to improve performance.
if (_joinColumns == null)
{
_joinKeyOrdinals = new int[0];
_sourceKeyOrdinals = new int[0];
}
else
{
_joinKeyOrdinals = new int[_joinColumns.Length];
_sourceKeyOrdinals = new int[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
_joinKeyOrdinals[i] = ReferenceTransform.GetOrdinal(_joinColumns[i].JoinColumn.Name);
_sourceKeyOrdinals[i] = _joinColumns[i].SourceColumn == null ? -1 : PrimaryTransform.GetOrdinal(_joinColumns[i].SourceColumn.Name);
}
}
return(true);
}