protected override async Task <ReturnValue <object[]> > ReadRecord(CancellationToken cancellationToken)
{
try
{
if (await ReferenceTransform.ReadAsync(cancellationToken) == false)
{
return(new ReturnValue <object[]>(false, null));
}
else
{
List <Filter> filters = new List <Filter>();
foreach (JoinPair join in JoinPairs)
{
var joinValue = join.JoinColumn == null ? join.JoinValue : ReferenceTransform[join.JoinColumn].ToString();
filters.Add(new Filter()
{
Column1 = join.SourceColumn,
CompareDataType = ETypeCode.String,
Operator = Filter.ECompare.IsEqual,
Value2 = joinValue
});
}
var result = await LookupRow(filters, cancellationToken);
return(result);
}
}
catch (Exception ex)
{
throw new Exception("The restful service failed due to the following error: " + ex.Message, ex);
}
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
try
{
if (!_isOpen)
{
throw new ConnectionException($"The web service is not open");
}
if (_cachedRows == null && await ReferenceTransform.ReadAsync(cancellationToken) == false)
{
return(null);
}
if (_cachedRows != null)
{
if (!_cachedRows.MoveNext())
{
_cachedRows = null;
if (await ReferenceTransform.ReadAsync(cancellationToken) == false)
{
return(null);
}
}
else
{
var row = _cachedRows.Current;
return(row);
}
}
var rows = await((ConnectionRestful)ReferenceConnection).LookupRow(CacheTable, _filter, cancellationToken);
if (rows != null && rows.Any())
{
_cachedRows = rows.GetEnumerator();
_cachedRows.MoveNext();
return(_cachedRows.Current);
}
return(null);
}
catch (Exception ex)
{
throw new ConnectionException($"Read web service record failed. {ex.Message}", ex);
}
}
/// <summary>
/// Reads the next group of rows (based on join key) from the reference transform.
/// </summary>
/// <returns></returns>
private async Task <bool> ReadNextGroup()
{
_groupData = new List <object[]>();
if (_joinReaderOpen)
{
_groupData.Add(ReferenceTransform.CurrentRow);
}
while (_joinReaderOpen)
{
// if no joins, then the whole reference table is the group
if (_joinColumns == null)
{
_joinReaderOpen = await ReferenceTransform.ReadAsync();
_groupFields = new object[0];
if (!_joinReaderOpen)
{
break;
}
_groupData.Add(ReferenceTransform.CurrentRow);
}
else
{
_groupFields = new object[_joinColumns.Length];
for (var i = 0; i < _groupFields.Length; i++)
{
_groupFields[i] = ReferenceTransform[_joinKeyOrdinals[i]];
}
_joinReaderOpen = await ReferenceTransform.ReadAsync();
if (!_joinReaderOpen)
{
break;
}
var duplicateCheck = true;
for (var i = 0; i < _joinColumns.Length; i++)
{
if (!Equals(_groupFields[i], ReferenceTransform[_joinKeyOrdinals[i]]))
{
duplicateCheck = false;
break;
}
}
if (duplicateCheck)
{
_groupData.Add(ReferenceTransform.CurrentRow);
}
else
{
break;
}
}
}
return(_groupData.Count > 0);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
// sorted merge will concatenate 2 sorted incoming datasets, and maintain the sort order.
if (_sortedMerge)
{
if (_firstRead)
{
// read one row for each reader.
var primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
var referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
await Task.WhenAll(primaryReadTask, referenceReadTask);
if (primaryReadTask.IsFaulted)
{
throw primaryReadTask.Exception;
}
if (referenceReadTask.IsFaulted)
{
throw referenceReadTask.Exception;
}
_primaryMoreRecords = primaryReadTask.Result;
_referenceMoreRecords = referenceReadTask.Result;
_firstRead = false;
}
if (_primaryReadTask != null)
{
_primaryMoreRecords = await _primaryReadTask;
}
if (_referenceReadTask != null)
{
_referenceMoreRecords = await _referenceReadTask;
}
if (!_primaryMoreRecords && !_referenceMoreRecords)
{
return(null);
}
var newRow = new object[FieldCount];
// no more primary records, then just read from the reference
if (!_primaryMoreRecords)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_primaryReadTask = null;
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
// no more reference record ,just read from the primary.
else if (!_referenceMoreRecords)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
PrimaryTransform.GetValues(newRow);
_referenceReadTask = null;
_primaryReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
else
{
//more records in both, then compare the rows and take the next in sort order.
var usePrimary = true;
for (var i = 0; i < _primarySortOrdinals.Count; i++)
{
var compareResult = Dexih.Utils.DataType.DataType.Compare(
PrimaryTransform.CacheTable.Columns[_primarySortOrdinals[i]].DataType,
PrimaryTransform[_primarySortOrdinals[i]], ReferenceTransform[_referenceSortOrdinals[i]]);
if ((compareResult == DataType.ECompareResult.Greater &&
SortFields[i].Direction == Sort.EDirection.Ascending) ||
(compareResult == DataType.ECompareResult.Less &&
SortFields[i].Direction == Sort.EDirection.Descending))
{
usePrimary = false;
break;
}
}
if (usePrimary)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
else
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
}
}
else
{
// if no sorting specified, concatenate will be in any order as the records arrive.
if (_firstRead)
{
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
_firstRead = false;
}
if (_primaryReadTask != null && _referenceReadTask != null)
{
await Task.WhenAny(_primaryReadTask, _referenceReadTask);
if (_primaryReadTask.IsCanceled || _referenceReadTask.IsCanceled ||
cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException("The read record task was cancelled");
}
if (_primaryReadTask.IsFaulted)
{
throw _primaryReadTask.Exception;
}
if (_referenceReadTask.IsFaulted)
{
throw _referenceReadTask.Exception;
}
if (_primaryReadTask.IsCompleted)
{
var result = _primaryReadTask.Result;
if (result)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
if (_referenceReadTask.IsCompleted)
{
var result = _referenceReadTask.Result;
if (result)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
}
if (_primaryReadTask != null)
{
var result = await _primaryReadTask;
if (result)
{
var returnValue = CreateRecord(PrimaryTransform, _primaryMappings);
_primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_primaryReadTask = null;
}
if (_referenceReadTask != null)
{
var result = await _referenceReadTask;
if (result)
{
var returnValue = CreateRecord(ReferenceTransform, _referenceMappings);
_referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken);
return(returnValue);
}
_referenceReadTask = null;
}
}
return(null);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
object[] newRow = null;
var pos = 0;
//this writes out duplicates of the primary reader when a duplicate match occurrs on the join table
//i.e. outer join.
if (_writeGroup)
{
//create a new row and write the primary fields out
newRow = new object[FieldCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
var joinRow = _filterdGroupData[_writeGroupPosition];
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = joinRow[i];
pos++;
}
_writeGroupPosition++;
//if last join record, then set the flag=false so the next read will read another primary row record.
if (_writeGroupPosition >= _filterdGroupData.Count)
{
_writeGroup = false;
}
return(newRow);
}
//read a new row from the primary table.
if (await PrimaryTransform.ReadAsync(cancellationToken) == false)
{
return(null);
}
var joinMatchFound = false;
//if input is sorted, then run a sortedjoin
if (JoinAlgorithm == EJoinAlgorithm.Sorted)
{
//first read get a row from the join table.
if (_firstRead)
{
//get the first two rows from the join table.
_joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken);
_groupsOpen = await ReadNextGroup();
_firstRead = false;
}
//loop through join table until we find a matching row.
if (_joinColumns != null)
{
while (_groupsOpen)
{
var joinFields = new object[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
joinFields[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]];
}
var compare = _joinKeyComparer.Compare(_groupFields, joinFields);
var done = false;
switch (compare)
{
case 1:
joinMatchFound = false;
done = true;
break;
case -1:
if (_groupsOpen)
{
_groupsOpen = await ReadNextGroup();
}
break;
case 0:
joinMatchFound = true;
done = true;
break;
}
if (done)
{
break;
}
}
}
}
else //if input is not sorted, then run a hash join.
{
//first read load the join table into memory
if (_firstRead)
{
_joinHashData = new SortedDictionary <object[], List <object[]> >(new JoinKeyComparer());
_joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken);
_groupsOpen = await ReadNextGroup();
//load all the join data into an a dictionary
while (_groupsOpen)
{
_joinHashData.Add(_groupFields, _groupData);
_groupsOpen = await ReadNextGroup();
}
_firstRead = false;
}
object[] sourceKeys;
//set the values for the lookup
if (_joinColumns != null)
{
sourceKeys = new object[_joinColumns.Length];
for (var i = 0; i < _joinColumns.Length; i++)
{
sourceKeys[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]];
}
}
else
{
sourceKeys = new object[0];
}
if (_joinHashData.Keys.Contains(sourceKeys))
{
_groupData = _joinHashData[sourceKeys];
_groupsOpen = true;
joinMatchFound = true;
}
else
{
joinMatchFound = false;
}
}
//create a new row and write the primary fields out
newRow = new object[FieldCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
newRow[pos] = PrimaryTransform[i];
pos++;
}
if (joinMatchFound)
{
//if there are additional join functions, we run them
if (_joinFilters.Count == 0)
{
_filterdGroupData = _groupData;
}
else
{
_filterdGroupData = new List <object[]>();
//filter out the current group based on the functions defined.
foreach (var row in _groupData)
{
var matchFound = true;
foreach (var condition in _joinFilters)
{
foreach (var input in condition.Inputs.Where(c => c.IsColumn))
{
object value = null;
try
{
if (input.Column.ReferenceTable == _referenceTableName)
{
value = row[ReferenceTransform.GetOrdinal(input.Column)];
}
else
{
value = PrimaryTransform[input.Column];
}
input.SetValue(value);
}
catch (Exception ex)
{
throw new TransformException($"The join tansform {Name} failed setting parameters on the condition {condition.FunctionName} with the parameter {input.Name}. {ex.Message}.", ex, value);
}
}
try
{
var invokeresult = condition.Invoke();
if ((bool)invokeresult == false)
{
matchFound = false;
break;
}
}
catch (FunctionIgnoreRowException)
{
matchFound = false;
TransformRowsIgnored++;
break;
}
catch (Exception ex)
{
throw new TransformException($"The join transform {Name} failed calling the function {condition.FunctionName}. {ex.Message}.", ex);
}
}
if (matchFound)
{
_filterdGroupData.Add(row);
}
}
}
object[] joinRow = null;
if (_filterdGroupData.Count > 0)
{
if (_filterdGroupData.Count > 1)
{
switch (JoinDuplicateStrategy)
{
case EDuplicateStrategy.Abend:
throw new DuplicateJoinKeyException("The join transform failed as the selected columns on the join table " + ReferenceTableAlias + " are not unique. To continue when duplicates occur set the join strategy to first, last or all.", ReferenceTableAlias, _groupFields);
case EDuplicateStrategy.First:
joinRow = _filterdGroupData[0];
break;
case EDuplicateStrategy.Last:
joinRow = _filterdGroupData.Last();
break;
case EDuplicateStrategy.All:
joinRow = _filterdGroupData[0];
_writeGroup = true;
_writeGroupPosition = 1;
break;
default:
throw new TransformException("The join transform failed due to an unknown join strategy " + JoinDuplicateStrategy);
}
}
else
{
joinRow = _filterdGroupData[0];
}
for (var i = 0; i < _referenceFieldCount; i++)
{
newRow[pos] = joinRow[i];
pos++;
}
}
}
return(newRow);
}