private object[] GetRow(DynamicTableEntity currentEntity)
{
var row = new object[CacheTable.Columns.Count];
var partitionKeyOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.AzurePartitionKey);
if (partitionKeyOrdinal >= 0)
{
row[partitionKeyOrdinal] = currentEntity.PartitionKey;
}
var rowKeyOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.AzureRowKey);
if (rowKeyOrdinal >= 0)
{
row[rowKeyOrdinal] = currentEntity.RowKey;
}
var timestampOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.TimeStamp);
if (timestampOrdinal >= 0)
{
row[timestampOrdinal] = currentEntity.Timestamp;
}
foreach (var value in currentEntity.Properties)
{
var returnValue = value.Value.PropertyAsObject;
if (returnValue == null)
{
row[CacheTable.GetOrdinal(value.Key)] = DBNull.Value;
}
else
{
row[CacheTable.GetOrdinal(value.Key)] = _connection.ConvertEntityProperty(CacheTable[value.Key].DataType, returnValue);
}
}
return(row);
}
public override async Task <bool> Open(long auditKey, SelectQuery query, CancellationToken cancellationToken)
{
try
{
AuditKey = auditKey;
if (_isOpen)
{
throw new ConnectionException("The reader is already open.");
}
_sqlConnection = await((ConnectionSql)ReferenceConnection).NewConnection();
_sqlReader = await ReferenceConnection.GetDatabaseReader(CacheTable, _sqlConnection, query, cancellationToken);
_fieldCount = _sqlReader.FieldCount;
_fieldOrdinals = new List <int>();
for (var i = 0; i < _sqlReader.FieldCount; i++)
{
var fieldName = _sqlReader.GetName(i);
var ordinal = CacheTable.GetOrdinal(fieldName);
if (ordinal < 0)
{
throw new ConnectionException($"The reader could not be opened as column {fieldName} could not be found in the table {CacheTable.Name}.");
}
_fieldOrdinals.Add(ordinal);
}
_sortFields = query?.Sorts;
_isOpen = true;
return(true);
}
catch (Exception ex)
{
throw new ConnectionException($"Open reader failed. {ex.Message}", ex);
}
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
//the saved reject row is when a validation outputs two rows (pass & fail).
if (_savedRejectRow != null)
{
var row = _savedRejectRow;
_savedRejectRow = null;
return(row);
}
if (_lastRecord)
{
return(null);
}
while (await PrimaryTransform.ReadAsync(cancellationToken))
{
var rejectReason = new StringBuilder();
var finalInvalidAction = TransformFunction.EInvalidAction.Pass;
//copy row data.
var passRow = new object[_columnCount];
for (var i = 0; i < _primaryFieldCount; i++)
{
passRow[_mapFieldOrdinals[i]] = PrimaryTransform[i];
}
if (passRow[_operationOrdinal] == null)
{
passRow[_operationOrdinal] = 'C';
}
object[] rejectRow = null;
//run the validation functions
if (Validations != null)
{
foreach (var validation in Validations)
{
//set inputs for the validation function
foreach (var input in validation.Inputs.Where(c => c.IsColumn))
{
try
{
input.SetValue(PrimaryTransform[input.Column]);
}
catch (Exception ex)
{
throw new TransformException($"The validation transform {Name} failed setting input parameters on the function {validation.FunctionName} parameter {input.Name} for column {input.Column.TableColumnName()}. {ex.Message}", ex, PrimaryTransform[input.Column.TableColumnName()]);
}
}
bool validationResult;
try
{
var invokeresult = validation.Invoke();
validationResult = (bool)invokeresult;
}
catch (FunctionIgnoreRowException)
{
validationResult = false;
}
catch (Exception ex)
{
throw new TransformException($"The validation transform {Name} failed on the function {validation.FunctionName}. {ex.Message}", ex);
}
//if the validation is false. apply any output columns, and set a reject status
if (!validationResult)
{
rejectReason.AppendLine("function: " + validation.FunctionName + ", parameters: " + string.Join(",", validation.Inputs.Select(c => c.Name + "=" + (c.IsColumn ? c.Column.TableColumnName() : c.Value.ToString())).ToArray()) + ".");
// fail job immediately.
if (validation.InvalidAction == TransformFunction.EInvalidAction.Abend)
{
var reason = $"The validation rule abended as the invalid action is set to abend. " + rejectReason.ToString();
throw new Exception(reason);
}
//if the record is to be discarded, continue the loop and get the next source record.
if (validation.InvalidAction == TransformFunction.EInvalidAction.Discard)
{
continue;
}
//set the final invalidation action based on priority order of other rejections.
finalInvalidAction = finalInvalidAction < validation.InvalidAction ? validation.InvalidAction : finalInvalidAction;
if (validation.InvalidAction == TransformFunction.EInvalidAction.Reject || validation.InvalidAction == TransformFunction.EInvalidAction.RejectClean)
{
//if the row is rejected, copy unmodified row to a reject row.
if (rejectRow == null)
{
rejectRow = new object[CacheTable.Columns.Count];
passRow.CopyTo(rejectRow, 0);
rejectRow[_operationOrdinal] = 'R';
TransformRowsRejected++;
}
//add a reject reason if it exists
if (_rejectReasonOrdinal >= 0)
{
if (validation.Outputs != null)
{
var param = validation.Outputs.SingleOrDefault(c => c.Column.TableColumnName() == _rejectReasonColumnName);
if (param != null)
{
rejectReason.Append(" Reason: " + (string)param.Value);
}
}
}
}
if (validation.InvalidAction == TransformFunction.EInvalidAction.Clean || validation.InvalidAction == TransformFunction.EInvalidAction.RejectClean)
{
validation.ReturnValue();
if (validation.Outputs != null)
{
foreach (var output in validation.Outputs)
{
if (output.Column.TableColumnName() != "")
{
var ordinal = CacheTable.GetOrdinal(output.Column);
var col = CacheTable.Columns[ordinal];
if (ordinal >= 0)
{
try
{
var parseresult = TryParse(col.DataType, output.Value, col.MaxLength);
passRow[ordinal] = parseresult;
}
catch (Exception ex)
{
throw new TransformException($"The validation transform {Name} failed parsing output values on the function {validation.FunctionName} parameter {output.Name} column {col.Name}. {ex.Message}", ex, output.Value);
}
}
}
}
}
}
}
}
}
if (ValidateDataTypes && (finalInvalidAction == TransformFunction.EInvalidAction.Pass || finalInvalidAction == TransformFunction.EInvalidAction.Clean))
{
for (var i = 0; i < _columnCount; i++)
{
var value = passRow[i];
var col = CacheTable.Columns[i];
if (col.DeltaType == TableColumn.EDeltaType.TrackingField || col.DeltaType == TableColumn.EDeltaType.NonTrackingField)
{
if (value == null || value is DBNull)
{
if (col.AllowDbNull == false)
{
if (rejectRow == null)
{
rejectRow = new object[_columnCount];
passRow.CopyTo(rejectRow, 0);
rejectRow[_operationOrdinal] = 'R';
TransformRowsRejected++;
}
rejectReason.AppendLine("Column:" + col.Name + ": Tried to insert null into non-null column.");
finalInvalidAction = TransformFunction.EInvalidAction.Reject;
}
passRow[i] = DBNull.Value;
}
else
{
try
{
var parseresult = TryParse(col.DataType, value, col.MaxLength);
passRow[i] = parseresult;
}
catch (Exception ex)
{
// if the parse fails on the column, then write out a reject record.
if (rejectRow == null)
{
rejectRow = new object[_columnCount];
passRow.CopyTo(rejectRow, 0);
rejectRow[_operationOrdinal] = 'R';
TransformRowsRejected++;
}
rejectReason.AppendLine(ex.Message);
finalInvalidAction = TransformFunction.EInvalidAction.Reject;
}
}
}
}
}
switch (finalInvalidAction)
{
case TransformFunction.EInvalidAction.Pass:
passRow[_validationStatusOrdinal] = "passed";
return(passRow);
case TransformFunction.EInvalidAction.Clean:
passRow[_validationStatusOrdinal] = "cleaned";
return(passRow);
case TransformFunction.EInvalidAction.RejectClean:
passRow[_validationStatusOrdinal] = "rejected-cleaned";
rejectRow[_validationStatusOrdinal] = "rejected-cleaned";
rejectRow[_rejectReasonOrdinal] = rejectReason.ToString();
_savedRejectRow = rejectRow;
return(passRow);
case TransformFunction.EInvalidAction.Reject:
passRow[_validationStatusOrdinal] = "rejected";
rejectRow[_validationStatusOrdinal] = "rejected";
rejectRow[_rejectReasonOrdinal] = rejectReason.ToString();
return(rejectRow);
}
//should never get here.
throw new TransformException("Validation failed due to an unknown error.");
}
return(null);
}
