protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { if (_alreadySorted) { if (await PrimaryTransform.ReadAsync(cancellationToken)) { var values = new object[PrimaryTransform.FieldCount]; PrimaryTransform.GetValues(values); return(values); } else { return(null); } } if (_firstRead) //load the entire record into a sorted list. { _sortedDictionary = new SortedDictionary <object[], object[]>(new SortKeyComparer(_sortFields)); var rowcount = 0; while (await PrimaryTransform.ReadAsync(cancellationToken)) { var values = new object[PrimaryTransform.FieldCount]; var sortFields = new object[_sortFields.Count + 1]; PrimaryTransform.GetValues(values); for (var i = 0; i < sortFields.Length - 1; i++) { sortFields[i] = PrimaryTransform[_sortFields[i].Column]; } sortFields[sortFields.Length - 1] = rowcount; //add row count as last key field to ensure matching rows remain in original order. _sortedDictionary.Add(sortFields, values); rowcount++; TransformRowsSorted++; } _firstRead = false; if (rowcount == 0) { return(null); } _iterator = _sortedDictionary.Keys.GetEnumerator(); _iterator.MoveNext(); return(_sortedDictionary[_iterator.Current]); } var success = _iterator.MoveNext(); if (success) { return(_sortedDictionary[_iterator.Current]); } else { _sortedDictionary = null; //free up memory after all rows are read. return(null); } }
public override Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken) { AuditKey = auditKey; if (query == null) { query = new SelectQuery(); } var requiredSorts = RequiredSortFields(); if (query.Sorts != null && query.Sorts.Count > 0) { for (var i = 0; i < requiredSorts.Count; i++) { if (query.Sorts[i].Column == requiredSorts[i].Column) { requiredSorts[i].Direction = query.Sorts[i].Direction; } else { break; } } } query.Sorts = requiredSorts; var returnValue = PrimaryTransform.Open(auditKey, query, cancellationToken); return(returnValue); }
public override bool InitializeOutputFields() { CacheTable = PrimaryTransform.CacheTable.Copy(); //add the operation type, which indicates whether record is rejected 'R' or 'C/U/D' create/update/delete if (CacheTable.Columns.SingleOrDefault(c => c.DeltaType == TableColumn.EDeltaType.DatabaseOperation) == null) { CacheTable.Columns.Insert(0, new TableColumn("Operation", ETypeCode.Byte) { DeltaType = TableColumn.EDeltaType.DatabaseOperation }); } //add the rejection reason, which details the reason for a rejection. if (CacheTable.Columns.SingleOrDefault(c => c.DeltaType == TableColumn.EDeltaType.RejectedReason) == null) { CacheTable.Columns.Add(new TableColumn("RejectReason", ETypeCode.String) { DeltaType = TableColumn.EDeltaType.RejectedReason }); } //add the rejection reason, which details the reason for a rejection. if (CacheTable.Columns.SingleOrDefault(c => c.DeltaType == TableColumn.EDeltaType.ValidationStatus) == null) { CacheTable.Columns.Add(new TableColumn("ValidationStatus", ETypeCode.String) { DeltaType = TableColumn.EDeltaType.ValidationStatus }); } //store reject column details to improve performance. _rejectReasonOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.RejectedReason); if (_rejectReasonOrdinal >= 0) { _rejectReasonColumnName = CacheTable.Columns[_rejectReasonOrdinal].Name; } _operationOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.DatabaseOperation); _validationStatusOrdinal = CacheTable.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ValidationStatus); _primaryFieldCount = PrimaryTransform.FieldCount; _columnCount = CacheTable.Columns.Count; _mapFieldOrdinals = new List <int>(); for (var i = 0; i < _primaryFieldCount; i++) { _mapFieldOrdinals.Add(GetOrdinal(PrimaryTransform.GetName(i))); } return(true); }
public override async Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken) { AuditKey = auditKey; if (query == null) { query = new SelectQuery(); } query.Sorts = RequiredSortFields(); var returnValue = await PrimaryTransform.Open(auditKey, query, cancellationToken); //check if the transform has already sorted the data, using sql or a presort. _alreadySorted = SortFieldsMatch(_sortFields, PrimaryTransform.SortFields); return(returnValue); }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { if (_lastRecord) { return(null); } _lastRecord = !await PrimaryTransform.ReadAsync(cancellationToken); if (!_lastRecord) { var newRow = new object[CacheTable.Columns.Count]; PrimaryTransform.GetValues(newRow); foreach (var profile in _profiles) { foreach (var input in profile.Inputs.Where(c => c.IsColumn)) { try { input.SetValue(PrimaryTransform[input.Column.Name]); } catch (Exception ex) { throw new TransformException($"The profile transform {Name} failed setting inputs on the function {profile.FunctionName} parameter {input.Name} column {input.Column.Name}. {ex.Message}", ex, PrimaryTransform[input.Column.Name]); } } try { profile.Invoke(); } catch (Exception ex) { throw new TransformException($"The profile transform {Name} failed on the function {profile.FunctionName}. {ex.Message}", ex); } } return(newRow); } else { var profileResults = GetProfileTable("ProfileResults"); foreach (var profile in _profiles) { object profileResult = null; try { profileResult = profile.ReturnValue(); } catch (Exception ex) { throw new TransformException($"The profile transform {Name} failed getting the return value on the function {profile.FunctionName}. {ex.Message}", ex); } var row = new object[6]; row[0] = AuditKey; row[1] = profile.FunctionName; row[2] = profile.Inputs[0].Column.Name; row[3] = true; row[4] = profileResult; profileResults.Data.Add(row); if (profile.Outputs.Length > 0) { var details = (Dictionary <string, int>)profile.Outputs[0].Value; if (details != null && details.Count > 0) { foreach (var value in details.Keys) { row = new object[6]; row[0] = AuditKey; row[1] = profile.FunctionName; row[2] = profile.Inputs[0].Column.Name; row[3] = false; row[4] = value; row[5] = details[value]; profileResults.Data.Add(row); } } } _profileResults = profileResults; } return(null); } }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { object[] newRow = null; // if there is a previous lookup cache, then just populated that as the next row. if (_lookupCache != null && _lookupCache.MoveNext()) { newRow = new object[FieldCount]; var pos1 = 0; for (var i = 0; i < _primaryFieldCount; i++) { newRow[pos1] = PrimaryTransform[i]; pos1++; } var lookup = _lookupCache.Current; for (var i = 0; i < _referenceFieldCount; i++) { newRow[pos1] = lookup[i]; pos1++; } return(newRow); } else { _lookupCache = null; } if (await PrimaryTransform.ReadAsync(cancellationToken) == false) { return(null); } //load in the primary table values newRow = new object[FieldCount]; var pos = 0; for (var i = 0; i < _primaryFieldCount; i++) { newRow[pos] = PrimaryTransform[i]; pos++; } //set the values for the lookup var selectQuery = new SelectQuery(); foreach (var joinPair in JoinPairs) { var value = joinPair.SourceColumn == null ? joinPair.JoinValue : PrimaryTransform[joinPair.SourceColumn]; selectQuery.Filters.Add(new Filter { Column1 = joinPair.JoinColumn, CompareDataType = ETypeCode.String, Operator = Filter.ECompare.IsEqual, Value2 = value }); } var lookupResult = await ReferenceTransform.Lookup(selectQuery, JoinDuplicateStrategy ?? EDuplicateStrategy.Abend, cancellationToken); if (lookupResult != null) { _lookupCache = lookupResult.GetEnumerator(); if (_lookupCache.MoveNext()) { var lookup = _lookupCache.Current; for (var i = 0; i < _referenceFieldCount; i++) { newRow[pos] = lookup[i]; pos++; } } else { _lookupCache = null; } } else { _lookupCache = null; } return(newRow); }
// public override List<Sort> SortFields => new List<Sort>(); public override async Task <bool> Open(long auditKey, SelectQuery query, CancellationToken cancellationToken) { AuditKey = auditKey; var primarySorts = new List <Sort>(); var referenceSorts = new List <Sort>(); //we need to translate filters and sorts to source column names before passing them through. if (query?.Sorts != null) { foreach (var sort in query.Sorts) { if (sort.Column != null) { var column = PrimaryTransform.CacheTable[sort.Column.Name]; if (column != null) { primarySorts.Add(new Sort(column, sort.Direction)); } column = ReferenceTransform.CacheTable[sort.Column.Name]; if (column != null) { referenceSorts.Add(new Sort(column, sort.Direction)); } } } } var primaryQuery = new SelectQuery() { Sorts = primarySorts }; var referenceQuery = new SelectQuery() { Sorts = referenceSorts }; var returnValue = await PrimaryTransform.Open(auditKey, primaryQuery, cancellationToken); if (!returnValue) { return(false); } returnValue = await ReferenceTransform.Open(auditKey, referenceQuery, cancellationToken); //if the primary & reference transforms are sorted, we will merge sort the items. if (PrimaryTransform.SortFields != null && ReferenceTransform.SortFields != null) { var newSortFields = new List <Sort>(); _primarySortOrdinals = new List <int>(); _referenceSortOrdinals = new List <int>(); var index = 0; var referenceSortFields = ReferenceTransform.SortFields; foreach (var sortField in PrimaryTransform.SortFields) { if (referenceSortFields.Count <= index) { break; } var referenceSortField = referenceSortFields[index]; if (sortField.Column.Name == referenceSortField.Column.Name && sortField.Direction == referenceSortField.Direction) { newSortFields.Add(sortField); _primarySortOrdinals.Add(PrimaryTransform.CacheTable.GetOrdinal(sortField.Column.Name)); _referenceSortOrdinals.Add(ReferenceTransform.CacheTable.GetOrdinal(sortField.Column.Name)); } else { break; } index++; } if (newSortFields.Count > 0) { _sortedMerge = true; CacheTable.OutputSortFields = newSortFields; } } return(returnValue); }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { // sorted merge will concatenate 2 sorted incoming datasets, and maintain the sort order. if (_sortedMerge) { if (_firstRead) { // read one row for each reader. var primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken); var referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken); await Task.WhenAll(primaryReadTask, referenceReadTask); if (primaryReadTask.IsFaulted) { throw primaryReadTask.Exception; } if (referenceReadTask.IsFaulted) { throw referenceReadTask.Exception; } _primaryMoreRecords = primaryReadTask.Result; _referenceMoreRecords = referenceReadTask.Result; _firstRead = false; } if (_primaryReadTask != null) { _primaryMoreRecords = await _primaryReadTask; } if (_referenceReadTask != null) { _referenceMoreRecords = await _referenceReadTask; } if (!_primaryMoreRecords && !_referenceMoreRecords) { return(null); } var newRow = new object[FieldCount]; // no more primary records, then just read from the reference if (!_primaryMoreRecords) { var returnValue = CreateRecord(ReferenceTransform, _referenceMappings); _primaryReadTask = null; _referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken); return(returnValue); } // no more reference record ,just read from the primary. else if (!_referenceMoreRecords) { var returnValue = CreateRecord(PrimaryTransform, _primaryMappings); PrimaryTransform.GetValues(newRow); _referenceReadTask = null; _primaryReadTask = ReferenceTransform.ReadAsync(cancellationToken); return(returnValue); } else { //more records in both, then compare the rows and take the next in sort order. var usePrimary = true; for (var i = 0; i < _primarySortOrdinals.Count; i++) { var compareResult = Dexih.Utils.DataType.DataType.Compare( PrimaryTransform.CacheTable.Columns[_primarySortOrdinals[i]].DataType, PrimaryTransform[_primarySortOrdinals[i]], ReferenceTransform[_referenceSortOrdinals[i]]); if ((compareResult == DataType.ECompareResult.Greater && SortFields[i].Direction == Sort.EDirection.Ascending) || (compareResult == DataType.ECompareResult.Less && SortFields[i].Direction == Sort.EDirection.Descending)) { usePrimary = false; break; } } if (usePrimary) { var returnValue = CreateRecord(PrimaryTransform, _primaryMappings); _primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken); return(returnValue); } else { var returnValue = CreateRecord(ReferenceTransform, _referenceMappings); _referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken); return(returnValue); } } } else { // if no sorting specified, concatenate will be in any order as the records arrive. if (_firstRead) { _primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken); _referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken); _firstRead = false; } if (_primaryReadTask != null && _referenceReadTask != null) { await Task.WhenAny(_primaryReadTask, _referenceReadTask); if (_primaryReadTask.IsCanceled || _referenceReadTask.IsCanceled || cancellationToken.IsCancellationRequested) { throw new OperationCanceledException("The read record task was cancelled"); } if (_primaryReadTask.IsFaulted) { throw _primaryReadTask.Exception; } if (_referenceReadTask.IsFaulted) { throw _referenceReadTask.Exception; } if (_primaryReadTask.IsCompleted) { var result = _primaryReadTask.Result; if (result) { var returnValue = CreateRecord(PrimaryTransform, _primaryMappings); _primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken); return(returnValue); } _primaryReadTask = null; } if (_referenceReadTask.IsCompleted) { var result = _referenceReadTask.Result; if (result) { var returnValue = CreateRecord(ReferenceTransform, _referenceMappings); _referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken); return(returnValue); } _primaryReadTask = null; } } if (_primaryReadTask != null) { var result = await _primaryReadTask; if (result) { var returnValue = CreateRecord(PrimaryTransform, _primaryMappings); _primaryReadTask = PrimaryTransform.ReadAsync(cancellationToken); return(returnValue); } _primaryReadTask = null; } if (_referenceReadTask != null) { var result = await _referenceReadTask; if (result) { var returnValue = CreateRecord(ReferenceTransform, _referenceMappings); _referenceReadTask = ReferenceTransform.ReadAsync(cancellationToken); return(returnValue); } _referenceReadTask = null; } } return(null); }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { //the saved reject row is when a validation outputs two rows (pass & fail). if (_savedRejectRow != null) { var row = _savedRejectRow; _savedRejectRow = null; return(row); } if (_lastRecord) { return(null); } while (await PrimaryTransform.ReadAsync(cancellationToken)) { var rejectReason = new StringBuilder(); var finalInvalidAction = TransformFunction.EInvalidAction.Pass; //copy row data. var passRow = new object[_columnCount]; for (var i = 0; i < _primaryFieldCount; i++) { passRow[_mapFieldOrdinals[i]] = PrimaryTransform[i]; } if (passRow[_operationOrdinal] == null) { passRow[_operationOrdinal] = 'C'; } object[] rejectRow = null; //run the validation functions if (Validations != null) { foreach (var validation in Validations) { //set inputs for the validation function foreach (var input in validation.Inputs.Where(c => c.IsColumn)) { try { input.SetValue(PrimaryTransform[input.Column]); } catch (Exception ex) { throw new TransformException($"The validation transform {Name} failed setting input parameters on the function {validation.FunctionName} parameter {input.Name} for column {input.Column.TableColumnName()}. {ex.Message}", ex, PrimaryTransform[input.Column.TableColumnName()]); } } bool validationResult; try { var invokeresult = validation.Invoke(); validationResult = (bool)invokeresult; } catch (FunctionIgnoreRowException) { validationResult = false; } catch (Exception ex) { throw new TransformException($"The validation transform {Name} failed on the function {validation.FunctionName}. {ex.Message}", ex); } //if the validation is false. apply any output columns, and set a reject status if (!validationResult) { rejectReason.AppendLine("function: " + validation.FunctionName + ", parameters: " + string.Join(",", validation.Inputs.Select(c => c.Name + "=" + (c.IsColumn ? c.Column.TableColumnName() : c.Value.ToString())).ToArray()) + "."); // fail job immediately. if (validation.InvalidAction == TransformFunction.EInvalidAction.Abend) { var reason = $"The validation rule abended as the invalid action is set to abend. " + rejectReason.ToString(); throw new Exception(reason); } //if the record is to be discarded, continue the loop and get the next source record. if (validation.InvalidAction == TransformFunction.EInvalidAction.Discard) { continue; } //set the final invalidation action based on priority order of other rejections. finalInvalidAction = finalInvalidAction < validation.InvalidAction ? validation.InvalidAction : finalInvalidAction; if (validation.InvalidAction == TransformFunction.EInvalidAction.Reject || validation.InvalidAction == TransformFunction.EInvalidAction.RejectClean) { //if the row is rejected, copy unmodified row to a reject row. if (rejectRow == null) { rejectRow = new object[CacheTable.Columns.Count]; passRow.CopyTo(rejectRow, 0); rejectRow[_operationOrdinal] = 'R'; TransformRowsRejected++; } //add a reject reason if it exists if (_rejectReasonOrdinal >= 0) { if (validation.Outputs != null) { var param = validation.Outputs.SingleOrDefault(c => c.Column.TableColumnName() == _rejectReasonColumnName); if (param != null) { rejectReason.Append(" Reason: " + (string)param.Value); } } } } if (validation.InvalidAction == TransformFunction.EInvalidAction.Clean || validation.InvalidAction == TransformFunction.EInvalidAction.RejectClean) { validation.ReturnValue(); if (validation.Outputs != null) { foreach (var output in validation.Outputs) { if (output.Column.TableColumnName() != "") { var ordinal = CacheTable.GetOrdinal(output.Column); var col = CacheTable.Columns[ordinal]; if (ordinal >= 0) { try { var parseresult = TryParse(col.DataType, output.Value, col.MaxLength); passRow[ordinal] = parseresult; } catch (Exception ex) { throw new TransformException($"The validation transform {Name} failed parsing output values on the function {validation.FunctionName} parameter {output.Name} column {col.Name}. {ex.Message}", ex, output.Value); } } } } } } } } } if (ValidateDataTypes && (finalInvalidAction == TransformFunction.EInvalidAction.Pass || finalInvalidAction == TransformFunction.EInvalidAction.Clean)) { for (var i = 0; i < _columnCount; i++) { var value = passRow[i]; var col = CacheTable.Columns[i]; if (col.DeltaType == TableColumn.EDeltaType.TrackingField || col.DeltaType == TableColumn.EDeltaType.NonTrackingField) { if (value == null || value is DBNull) { if (col.AllowDbNull == false) { if (rejectRow == null) { rejectRow = new object[_columnCount]; passRow.CopyTo(rejectRow, 0); rejectRow[_operationOrdinal] = 'R'; TransformRowsRejected++; } rejectReason.AppendLine("Column:" + col.Name + ": Tried to insert null into non-null column."); finalInvalidAction = TransformFunction.EInvalidAction.Reject; } passRow[i] = DBNull.Value; } else { try { var parseresult = TryParse(col.DataType, value, col.MaxLength); passRow[i] = parseresult; } catch (Exception ex) { // if the parse fails on the column, then write out a reject record. if (rejectRow == null) { rejectRow = new object[_columnCount]; passRow.CopyTo(rejectRow, 0); rejectRow[_operationOrdinal] = 'R'; TransformRowsRejected++; } rejectReason.AppendLine(ex.Message); finalInvalidAction = TransformFunction.EInvalidAction.Reject; } } } } } switch (finalInvalidAction) { case TransformFunction.EInvalidAction.Pass: passRow[_validationStatusOrdinal] = "passed"; return(passRow); case TransformFunction.EInvalidAction.Clean: passRow[_validationStatusOrdinal] = "cleaned"; return(passRow); case TransformFunction.EInvalidAction.RejectClean: passRow[_validationStatusOrdinal] = "rejected-cleaned"; rejectRow[_validationStatusOrdinal] = "rejected-cleaned"; rejectRow[_rejectReasonOrdinal] = rejectReason.ToString(); _savedRejectRow = rejectRow; return(passRow); case TransformFunction.EInvalidAction.Reject: passRow[_validationStatusOrdinal] = "rejected"; rejectRow[_validationStatusOrdinal] = "rejected"; rejectRow[_rejectReasonOrdinal] = rejectReason.ToString(); return(rejectRow); } //should never get here. throw new TransformException("Validation failed due to an unknown error."); } return(null); }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { object[] newRow = null; var pos = 0; //this writes out duplicates of the primary reader when a duplicate match occurrs on the join table //i.e. outer join. if (_writeGroup) { //create a new row and write the primary fields out newRow = new object[FieldCount]; for (var i = 0; i < _primaryFieldCount; i++) { newRow[pos] = PrimaryTransform[i]; pos++; } var joinRow = _filterdGroupData[_writeGroupPosition]; for (var i = 0; i < _referenceFieldCount; i++) { newRow[pos] = joinRow[i]; pos++; } _writeGroupPosition++; //if last join record, then set the flag=false so the next read will read another primary row record. if (_writeGroupPosition >= _filterdGroupData.Count) { _writeGroup = false; } return(newRow); } //read a new row from the primary table. if (await PrimaryTransform.ReadAsync(cancellationToken) == false) { return(null); } var joinMatchFound = false; //if input is sorted, then run a sortedjoin if (JoinAlgorithm == EJoinAlgorithm.Sorted) { //first read get a row from the join table. if (_firstRead) { //get the first two rows from the join table. _joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken); _groupsOpen = await ReadNextGroup(); _firstRead = false; } //loop through join table until we find a matching row. if (_joinColumns != null) { while (_groupsOpen) { var joinFields = new object[_joinColumns.Length]; for (var i = 0; i < _joinColumns.Length; i++) { joinFields[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]]; } var compare = _joinKeyComparer.Compare(_groupFields, joinFields); var done = false; switch (compare) { case 1: joinMatchFound = false; done = true; break; case -1: if (_groupsOpen) { _groupsOpen = await ReadNextGroup(); } break; case 0: joinMatchFound = true; done = true; break; } if (done) { break; } } } } else //if input is not sorted, then run a hash join. { //first read load the join table into memory if (_firstRead) { _joinHashData = new SortedDictionary <object[], List <object[]> >(new JoinKeyComparer()); _joinReaderOpen = await ReferenceTransform.ReadAsync(cancellationToken); _groupsOpen = await ReadNextGroup(); //load all the join data into an a dictionary while (_groupsOpen) { _joinHashData.Add(_groupFields, _groupData); _groupsOpen = await ReadNextGroup(); } _firstRead = false; } object[] sourceKeys; //set the values for the lookup if (_joinColumns != null) { sourceKeys = new object[_joinColumns.Length]; for (var i = 0; i < _joinColumns.Length; i++) { sourceKeys[i] = _joinColumns[i].SourceColumn == null ? _joinColumns[i].JoinValue : PrimaryTransform[_sourceKeyOrdinals[i]]; } } else { sourceKeys = new object[0]; } if (_joinHashData.Keys.Contains(sourceKeys)) { _groupData = _joinHashData[sourceKeys]; _groupsOpen = true; joinMatchFound = true; } else { joinMatchFound = false; } } //create a new row and write the primary fields out newRow = new object[FieldCount]; for (var i = 0; i < _primaryFieldCount; i++) { newRow[pos] = PrimaryTransform[i]; pos++; } if (joinMatchFound) { //if there are additional join functions, we run them if (_joinFilters.Count == 0) { _filterdGroupData = _groupData; } else { _filterdGroupData = new List <object[]>(); //filter out the current group based on the functions defined. foreach (var row in _groupData) { var matchFound = true; foreach (var condition in _joinFilters) { foreach (var input in condition.Inputs.Where(c => c.IsColumn)) { object value = null; try { if (input.Column.ReferenceTable == _referenceTableName) { value = row[ReferenceTransform.GetOrdinal(input.Column)]; } else { value = PrimaryTransform[input.Column]; } input.SetValue(value); } catch (Exception ex) { throw new TransformException($"The join tansform {Name} failed setting parameters on the condition {condition.FunctionName} with the parameter {input.Name}. {ex.Message}.", ex, value); } } try { var invokeresult = condition.Invoke(); if ((bool)invokeresult == false) { matchFound = false; break; } } catch (FunctionIgnoreRowException) { matchFound = false; TransformRowsIgnored++; break; } catch (Exception ex) { throw new TransformException($"The join transform {Name} failed calling the function {condition.FunctionName}. {ex.Message}.", ex); } } if (matchFound) { _filterdGroupData.Add(row); } } } object[] joinRow = null; if (_filterdGroupData.Count > 0) { if (_filterdGroupData.Count > 1) { switch (JoinDuplicateStrategy) { case EDuplicateStrategy.Abend: throw new DuplicateJoinKeyException("The join transform failed as the selected columns on the join table " + ReferenceTableAlias + " are not unique. To continue when duplicates occur set the join strategy to first, last or all.", ReferenceTableAlias, _groupFields); case EDuplicateStrategy.First: joinRow = _filterdGroupData[0]; break; case EDuplicateStrategy.Last: joinRow = _filterdGroupData.Last(); break; case EDuplicateStrategy.All: joinRow = _filterdGroupData[0]; _writeGroup = true; _writeGroupPosition = 1; break; default: throw new TransformException("The join transform failed due to an unknown join strategy " + JoinDuplicateStrategy); } } else { joinRow = _filterdGroupData[0]; } for (var i = 0; i < _referenceFieldCount; i++) { newRow[pos] = joinRow[i]; pos++; } } } return(newRow); }
public override async Task <bool> Open(Int64 auditKey, SelectQuery query, CancellationToken cancellationToken) { AuditKey = auditKey; if (query == null) { query = new SelectQuery(); } //only apply a sort if there is not already a sort applied. // if(query.Sorts == null || query.Sorts.Count == 0) query.Sorts = RequiredSortFields(); var returnValue = await PrimaryTransform.Open(auditKey, query, cancellationToken); if (!returnValue) { return(false); } var referenceQuery = new SelectQuery() { Sorts = RequiredReferenceSortFields() }; returnValue = await ReferenceTransform.Open(auditKey, referenceQuery, cancellationToken); if (!returnValue) { return(false); } //check if the primary and reference transform are sorted in the join if (SortFieldsMatch(RequiredSortFields(), PrimaryTransform.SortFields) && SortFieldsMatch(RequiredReferenceSortFields(), ReferenceTransform.SortFields)) { JoinAlgorithm = EJoinAlgorithm.Sorted; } else { JoinAlgorithm = EJoinAlgorithm.Hash; } //store the ordinals for the joins to improve performance. if (_joinColumns == null) { _joinKeyOrdinals = new int[0]; _sourceKeyOrdinals = new int[0]; } else { _joinKeyOrdinals = new int[_joinColumns.Length]; _sourceKeyOrdinals = new int[_joinColumns.Length]; for (var i = 0; i < _joinColumns.Length; i++) { _joinKeyOrdinals[i] = ReferenceTransform.GetOrdinal(_joinColumns[i].JoinColumn.Name); _sourceKeyOrdinals[i] = _joinColumns[i].SourceColumn == null ? -1 : PrimaryTransform.GetOrdinal(_joinColumns[i].SourceColumn.Name); } } return(true); }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { object[] newRow = null; //if there are records in the passthrough cache, then empty them out before getting new records. if (PassThroughColumns) { if (_firstRecord) { _passThroughValues = new List <object[]>(); } else if (_passThroughIndex > 0 && _passThroughIndex < _passThroughCount) { newRow = _passThroughValues[_passThroughIndex]; _passThroughIndex++; return(newRow); } //if all rows have been iterated through, reset the cache and add the stored row for the next group else if (_passThroughIndex >= _passThroughCount && _firstRecord == false && _lastRecord == false) { _passThroughValues.Clear(); _passThroughValues.Add(_nextPassThroughRow); } } newRow = new object[FieldCount]; int i; var groupChanged = false; object[] newGroupValues = null; if (await PrimaryTransform.ReadAsync(cancellationToken) == false) { if (_lastRecord) //return false is all record have been written. { return(null); } } else { do { _lastRecord = false; i = 0; //if it's the first record then the groupvalues are being set for the first time. if (_firstRecord) { groupChanged = false; _firstRecord = false; if (GroupFields != null) { foreach (var groupField in GroupFields) { // _groupValues[i] = PrimaryTransform[groupField.SourceColumn]?.ToString() ?? ""; _groupValues[i] = PrimaryTransform[groupField.SourceColumn]; i++; } } newGroupValues = _groupValues; } else { //if not first row, then check if the group values have changed from the previous row if (GroupFields != null) { newGroupValues = new object[GroupFields.Count]; foreach (var groupField in GroupFields) { //newGroupValues[i] = PrimaryTransform[groupField.SourceColumn]?.ToString() ?? ""; newGroupValues[i] = PrimaryTransform[groupField.SourceColumn]; if ((newGroupValues[i] == null && _groupValues != null) || (newGroupValues[i] != null && _groupValues == null) || !Equals(newGroupValues[i], _groupValues[i])) { groupChanged = true; } i++; } } } //if the group values have changed, write out the previous group values. if (groupChanged) { i = 0; if (GroupFields != null) { for (; i < GroupFields.Count; i++) { newRow[i] = _groupValues[i]; } } if (PassThroughColumns == false) { if (AggregatePairs != null) { foreach (var aggregate in AggregatePairs) { newRow[i] = aggregate.GetValue(); i++; } } if (Aggregates != null) { foreach (var mapping in Aggregates) { try { newRow[i] = mapping.ReturnValue(0); i++; if (mapping.Outputs != null) { foreach (var output in mapping.Outputs) { newRow[i] = output.Value; i++; } } } catch (Exception ex) { throw new TransformException($"The group transform {Name} failed on function {mapping.FunctionName}. {ex.Message}.", ex); } } } } //for passthrough, write out the aggregated values to the cached passthrough set else { var index = 0; var startColumn = i; foreach (var row in _passThroughValues) { i = startColumn; if (AggregatePairs != null) { foreach (var aggregate in AggregatePairs) { row[i] = aggregate.GetValue(); i++; } } if (Aggregates != null) { foreach (var mapping in Aggregates) { try { row[i] = mapping.ReturnValue(index); i++; if (mapping.Outputs != null) { foreach (var output in mapping.Outputs) { row[i] = output.Value; i++; } } } catch (Exception ex) { throw new TransformException($"The group transform {Name} failed, retrieving results from function {mapping.FunctionName}. {ex.Message}.", ex); } } } index++; } //the first row of the next group has been read, so this is to store it until ready to write out. _nextPassThroughRow = new object[FieldCount]; for (var j = 0; j < newGroupValues.Length; j++) { _nextPassThroughRow[j] = newGroupValues[j]; } for (var j = _passThroughStartIndex; j < FieldCount; j++) { _nextPassThroughRow[j] = PrimaryTransform[GetName(j)]; } ////set the first cached row to current newRow = _passThroughValues[0]; _passThroughIndex = 1; _passThroughCount = _passThroughValues.Count; } //reset the functions if (AggregatePairs != null) { foreach (var aggregate in AggregatePairs) { aggregate.Reset(); } } if (Aggregates != null) { foreach (var mapping in Aggregates) { mapping.Reset(); } } //store the last groupvalues read to start the next grouping. _groupValues = newGroupValues; } else { if (PassThroughColumns) { //create a cached current row. this will be output when the group has changed. var cacheRow = new object[newRow.Length]; if (_groupValues != null) { for (var j = 0; j < _groupValues.Length; j++) { cacheRow[j] = _groupValues[j]; } } for (var j = _passThroughStartIndex; j < FieldCount; j++) { cacheRow[j] = PrimaryTransform[GetName(j)]; } _passThroughValues.Add(cacheRow); } } // update the aggregate pairs if (AggregatePairs != null) { foreach (var aggregate in AggregatePairs) { aggregate.AddValue(PrimaryTransform[aggregate.SourceColumn]); } } // update the aggregate functions if (Aggregates != null) { foreach (var mapping in Aggregates) { if (mapping.Inputs != null) { foreach (var input in mapping.Inputs.Where(c => c.IsColumn)) { try { input.SetValue(PrimaryTransform[input.Column]); } catch (Exception ex) { throw new TransformException($"The group transform {Name} failed setting an incompatible value to column {input.Column.Name}. {ex.Message}.", ex, PrimaryTransform[input.Column]); } } } try { var invokeresult = mapping.Invoke(); } catch (FunctionIgnoreRowException) { //TODO: Issue that some of the aggregate values may be calculated prior to the ignorerow being set. TransformRowsIgnored++; continue; } catch (Exception ex) { throw new TransformException($"The group transform {Name} failed running the function {mapping.FunctionName}. {ex.Message}.", ex); } } } if (groupChanged) { break; } } while (await PrimaryTransform.ReadAsync(cancellationToken)); } if (groupChanged == false) //if the reader has finished with no group change, write the values and set last record { i = 0; if (GroupFields != null) { for (; i < GroupFields.Count; i++) { newRow[i] = _groupValues[i]; } } if (PassThroughColumns == false) { if (AggregatePairs != null) { foreach (var aggregate in AggregatePairs) { newRow[i] = aggregate.GetValue(); i++; aggregate.Reset(); } } if (Aggregates != null) { foreach (var mapping in Aggregates) { try { newRow[i] = mapping.ReturnValue(0); } catch (Exception ex) { throw new TransformException($"The group transform {Name} failed, retrieving results from function {mapping.FunctionName}. {ex.Message}.", ex); } i++; if (mapping.Outputs != null) { foreach (var output in mapping.Outputs) { newRow[i] = output.Value; i++; } } mapping.Reset(); } } } else { //for passthrough, write out the aggregated values to the cached passthrough set var index = 0; var startColumn = i; foreach (var row in _passThroughValues) { i = startColumn; if (AggregatePairs != null) { foreach (var aggregate in AggregatePairs) { newRow[i] = aggregate.GetValue(); i++; } } if (Aggregates != null) { foreach (var mapping in Aggregates) { try { row[i] = mapping.ReturnValue(index); } catch (Exception ex) { throw new TransformException($"The group transform {Name} failed retrieving results from function {mapping.FunctionName}. {ex.Message}.", ex); } i++; if (mapping.Outputs != null) { foreach (var output in mapping.Outputs) { row[i] = output.Value; i++; } } } } index++; } //set the first cached row to current newRow = _passThroughValues[0]; _passThroughIndex = 1; _passThroughCount = _passThroughValues.Count; } _groupValues = newGroupValues; _lastRecord = true; } return(newRow); }