/// <summary>
/// Create a basic repository with hub.
/// </summary>
/// <returns></returns>
public async Task <RepositoryManager> CreateRepositoryWithHub()
{
var context = await CreateRepositoryEmpty();
var userStore = new UserStore <ApplicationUser>(context);
var userManager = MockHelpers.TestUserManager(userStore);
// instance of the memory cache.
var memoryCache = new MemoryDistributedCache(new OptionsWrapper <MemoryDistributedCacheOptions>(new MemoryDistributedCacheOptions()));
var cacheService = new CacheService(memoryCache);
var database = new RepositoryManager(context, userManager, cacheService, _loggerFactory, null);
var hub = new DexihHub()
{
Name = "test",
Description = "hub description",
EncryptionKey = "123"
};
var user = await database.FindByEmailAsync("*****@*****.**");
var newHub = await database.SaveHub(hub, user, CancellationToken.None);
Assert.Equal(1, context.DexihHubs.Count());
return(database);
}
public DatajobRun(TransformSettings transformSettings, ILogger logger, DexihDatajob datajob, DexihHub hub, TransformWriterOptions transformWriterOptions, IAlertQueue alertQueue, string[] alertEmails)
{
_transformSettings = transformSettings;
_logger = logger;
_alertQueue = alertQueue;
_alertEmails = alertEmails;
_transformWriterOptions = transformWriterOptions;
if (datajob.AuditConnectionKey > 0)
{
var dbAuditConnection =
hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == datajob.AuditConnectionKey);
if (dbAuditConnection == null)
{
throw new DatalinkRunException(
$"Audit connection with key {datajob.AuditConnectionKey} was not found.");
}
_auditConnection = dbAuditConnection.GetConnection(_transformSettings);
}
else
{
_auditConnection = new ConnectionMemory();
}
Datajob = datajob;
_hub = hub;
}
public DatalinkTestRun(
TransformSettings transformSettings,
ILogger logger,
DexihDatalinkTest datalinkTest,
DexihHub hub,
TransformWriterOptions transformWriterOptions,
IAlertQueue alertQueue,
string[] alertEmails
)
{
_transformSettings = transformSettings;
_transformWriterOptions = transformWriterOptions;
_logger = logger;
// create a copy of the hub as the test run will update objects.
_hub = hub.Serialize().Deserialize <DexihHub>();
// _hub = hub.CloneProperties();
_datalinkTest = datalinkTest;
_alertQueue = alertQueue;
_alertEmails = alertEmails;
Connection auditConnection;
if (datalinkTest.AuditConnectionKey > 0)
{
var dbAuditConnection =
_hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == datalinkTest.AuditConnectionKey);
if (dbAuditConnection == null)
{
throw new DatalinkRunException(
$"Audit connection with key {datalinkTest.AuditConnectionKey} was not found.");
}
auditConnection = dbAuditConnection.GetConnection(_transformSettings);
}
else
{
auditConnection = new ConnectionMemory();
}
TestResults = new List <TestResult>();
WriterResult = new TransformWriterResult()
{
AuditConnection = auditConnection,
AuditConnectionKey = datalinkTest.AuditConnectionKey ?? 0,
AuditType = Constants.DatalinkTest,
HubKey = _hub.HubKey,
ReferenceKey = datalinkTest.Key,
ParentAuditKey = 0,
ReferenceName = datalinkTest.Name,
SourceTableKey = 0,
SourceTableName = "",
TransformWriterOptions = _transformWriterOptions
};
}
/// Searches all the tables in a datalink for a particular columnKey
public DexihDatalinkColumn GetDatalinkColumn(DexihHub hub, DexihDatalink hubDatalink, long?datalinkColumnKey)
{
if (datalinkColumnKey == null || hubDatalink == null)
{
return(null);
}
DexihDatalinkColumn column;
column = hubDatalink.SourceDatalinkTable.DexihDatalinkColumns.SingleOrDefault(c => c.Key == (long)datalinkColumnKey);
if (column != null)
{
return(column);
}
foreach (var datalinkTransform in hubDatalink.DexihDatalinkTransforms)
{
foreach (var item in datalinkTransform.DexihDatalinkTransformItems)
{
if (item.SourceDatalinkColumnKey == datalinkColumnKey)
{
return(item.SourceDatalinkColumn);
}
if (item.TargetDatalinkColumnKey == datalinkColumnKey)
{
return(item.TargetDatalinkColumn);
}
foreach (var param in item.DexihFunctionParameters)
{
if (param.DatalinkColumnKey == datalinkColumnKey)
{
return(param.DatalinkColumn);
}
}
}
if (datalinkTransform.JoinDatalinkTableKey != null)
{
column = datalinkTransform.JoinDatalinkTable.DexihDatalinkColumns.SingleOrDefault(c => c.Key == (long)datalinkColumnKey);
if (column != null)
{
return(column);
}
}
}
return(null);
}
public async void SaveHub()
{
var hub = new DexihHub()
{
Name = HubName,
Description = "The description",
EncryptionKey = "123"
};
var message = await Configuration.Post("/api/Account/SaveHub", hub);
var user = JsonExtensions.Deserialize <ReturnValue <DexihHub> >(message);
Assert.True(user.Success);
}
private async Task <DexihHub> CreateHub()
{
var user = await _repositoryManager.FindByEmailAsync("*****@*****.**");
user = await _repositoryManager.GetUserAsync(user.Id, CancellationToken.None);
// use a guid to make the hub name unique
var guid = Guid.NewGuid().ToString();
var hub = new DexihHub()
{
Name = guid,
Description = "Description",
};
return(await _repositoryManager.SaveHub(hub, user, CancellationToken.None));
}
public void UpdateRemoteAgentHubCacheItems(DexihHub hub)
{
foreach (var updatedItem in RemoteAgentHubs)
{
switch (updatedItem.ImportAction)
{
case EImportAction.Replace:
foreach (var existingItem in hub.DexihRemoteAgentHubs)
{
if (existingItem.RemoteAgentKey == updatedItem.Item.RemoteAgentKey)
{
hub.DexihRemoteAgentHubs.Remove(existingItem);
break;
}
}
hub.DexihRemoteAgentHubs.Add(updatedItem.Item);
break;
case EImportAction.New:
hub.DexihRemoteAgentHubs.Add(updatedItem.Item);
break;
case EImportAction.Leave:
break;
case EImportAction.Skip:
break;
case EImportAction.Delete:
foreach (var existingItem in hub.DexihRemoteAgentHubs)
{
if (existingItem.RemoteAgentKey == updatedItem.Item.RemoteAgentKey)
{
hub.DexihRemoteAgentHubs.Remove(existingItem);
break;
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
}
public void UpdateTagObjectCacheItems(DexihHub hub)
{
foreach (var updatedItem in TagObjects)
{
switch (updatedItem.ImportAction)
{
case EImportAction.Replace:
foreach (var existingItem in hub.DexihTagObjects)
{
if (existingItem.TagKey == updatedItem.Item.TagKey && existingItem.ObjectKey == updatedItem.Item.ObjectKey && existingItem.ObjectType == updatedItem.Item.ObjectType)
{
hub.DexihTagObjects.Remove(existingItem);
break;
}
}
hub.DexihTagObjects.Add(updatedItem.Item);
break;
case EImportAction.New:
hub.DexihTagObjects.Add(updatedItem.Item);
break;
case EImportAction.Leave:
break;
case EImportAction.Skip:
break;
case EImportAction.Delete:
foreach (var existingItem in hub.DexihTagObjects)
{
if (existingItem.TagKey == updatedItem.Item.TagKey && existingItem.ObjectKey == updatedItem.Item.ObjectKey && existingItem.ObjectType == updatedItem.Item.ObjectType)
{
hub.DexihTagObjects.Remove(existingItem);
break;
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
}
public void UpdateCache(DexihHub hub)
{
UpdateCacheItems(hub.DexihHubVariables, HubVariables);
UpdateCacheItems(hub.DexihListOfValues, ListOfValues);
UpdateCacheItems(hub.DexihCustomFunctions, CustomFunctions);
UpdateCacheItems(hub.DexihFileFormats, FileFormats);
UpdateCacheItems(hub.DexihConnections, Connections);
UpdateCacheItems(hub.DexihTables, Tables);
UpdateCacheItems(hub.DexihColumnValidations, ColumnValidations);
UpdateCacheItems(hub.DexihDatalinks, Datalinks);
UpdateCacheItems(hub.DexihDatajobs, Datajobs);
UpdateCacheItems(hub.DexihDatalinkTests, DatalinkTests);
UpdateCacheItems(hub.DexihViews, Views);
UpdateCacheItems(hub.DexihApis, Apis);
UpdateCacheItems(hub.DexihDashboards, Dashboards);
UpdateCacheItems(hub.DexihListOfValues, ListOfValues);
UpdateCacheItems(hub.DexihTags, Tags);
UpdateTagObjectCacheItems(hub);
UpdateRemoteAgentHubCacheItems(hub);
}
public ColumnValidationRun(TransformSettings transformSettings, DexihColumnValidation columnValidation, DexihHub hub)
{
ColumnValidation = columnValidation;
Hub = hub;
_transformSettings = transformSettings;
}
public (Transform sourceTransform, Table sourceTable) CreateRunPlan(DexihHub hub, DexihDatalink hubDatalink, InputColumn[] inputColumns, long?maxDatalinkTransformKey, object maxIncrementalValue, TransformWriterOptions transformWriterOptions) //Last datatransform key is used to preview the output of a specific transform in the series.
{
try
{
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name} started.");
var timer = Stopwatch.StartNew();
if (transformWriterOptions == null)
{
transformWriterOptions = new TransformWriterOptions();
}
var primaryTransformResult = GetSourceTransform(hub, hubDatalink.SourceDatalinkTable, inputColumns, transformWriterOptions);
var primaryTransform = primaryTransformResult.sourceTransform;
var sourceTable = primaryTransformResult.sourceTable;
var updateStrategy = hubDatalink.UpdateStrategy;
//add a filter for the incremental column (if there is one)
TableColumn incrementalCol = null;
if (updateStrategy == EUpdateStrategy.AppendUpdateDeletePreserve || updateStrategy == EUpdateStrategy.AppendUpdatePreserve)
{
incrementalCol = primaryTransform.CacheTable?.GetColumn(EDeltaType.ValidFromDate);
}
else
{
incrementalCol = primaryTransform.CacheTable?.Columns.SingleOrDefault(c => c.IsIncrementalUpdate);
}
if (transformWriterOptions.ResetIncremental)
{
maxIncrementalValue = transformWriterOptions.ResetIncrementalValue;
}
if (maxDatalinkTransformKey == null &&
transformWriterOptions.IsEmptyTarget() == false &&
!(updateStrategy == EUpdateStrategy.Reload || updateStrategy == EUpdateStrategy.AppendUpdateDelete || updateStrategy == EUpdateStrategy.AppendUpdateDeletePreserve) &&
incrementalCol != null &&
maxIncrementalValue != null &&
maxIncrementalValue.ToString() != "")
{
var mappings = new Mappings()
{
new MapFilter(incrementalCol, maxIncrementalValue, ECompare.GreaterThan)
};
var filterTransform = new TransformFilter(primaryTransform, mappings)
{
Name = $"Prefilter maxIncremental {maxIncrementalValue}"
};
filterTransform.SetInTransform(primaryTransform);
primaryTransform = filterTransform;
}
DexihTable targetTable = null;
var target = hubDatalink.DexihDatalinkTargets.FirstOrDefault(c => c.NodeDatalinkColumnKey == null);
if (target != null)
{
targetTable = hub.GetTableFromKey(target.TableKey);
}
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}. Added incremental filter. Elapsed: {timer.Elapsed}");
//loop through the transforms to create the chain.
foreach (var datalinkTransform in hubDatalink.DexihDatalinkTransforms.OrderBy(c => c.Position).Where(c => c.IsValid))
{
//if this is an empty transform, then ignore it.
if (datalinkTransform.DexihDatalinkTransformItems.Count == 0)
{
if (datalinkTransform.TransformType == ETransformType.Filter || datalinkTransform.TransformType == ETransformType.Mapping && datalinkTransform.PassThroughColumns)
{
if (datalinkTransform.Key == maxDatalinkTransformKey)
{
break;
}
continue;
}
}
//if contains a join table, then add it in.
Transform referenceTransform = null;
if (datalinkTransform.JoinDatalinkTable != null)
{
var joinTransformResult = GetSourceTransform(hub, datalinkTransform.JoinDatalinkTable, null, transformWriterOptions);
referenceTransform = joinTransformResult.sourceTransform;
}
var transform = datalinkTransform.GetTransform(hub, hubDatalink, transformWriterOptions.GlobalSettings, _transformSettings, primaryTransform, referenceTransform, targetTable, _logger);
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}, adding transform {datalinkTransform.Name}. Elapsed: {timer.Elapsed}");
primaryTransform = transform;
if (datalinkTransform.Key == maxDatalinkTransformKey)
{
break;
}
}
//if the maxDatalinkTransformKey is null (i.e. we are not doing a preview), and there are profiles add a profile transform.
if (maxDatalinkTransformKey == null && hubDatalink.DexihDatalinkProfiles != null && hubDatalink.DexihDatalinkProfiles.Count > 0 && targetTable != null)
{
var profileRules = new Mappings();
foreach (var profile in hubDatalink.DexihDatalinkProfiles)
{
foreach (var column in targetTable.DexihTableColumns.Where(c => c.IsSourceColumn))
{
var profileFunction = GetProfileFunction(profile.FunctionAssemblyName, profile.FunctionClassName, profile.FunctionMethodName, column.Name, profile.DetailedResults, transformWriterOptions.GlobalSettings);
profileRules.Add(profileFunction);
}
}
var transform = new TransformProfile(primaryTransform, profileRules)
{
Name = "User defined profiles"
};
primaryTransform = transform;
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}, adding profiling. Elapsed: {timer.Elapsed}");
}
if (transformWriterOptions.SelectQuery != null)
{
var transform = new TransformQuery(primaryTransform, transformWriterOptions.SelectQuery)
{
Name = "Select Query Filter"
};
primaryTransform = transform;
}
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}, completed. Elapsed: {timer.Elapsed}");
return(primaryTransform, sourceTable);
}
catch (Exception ex)
{
throw new TransformManagerException($"Create run plan failed. {ex.Message}", ex);
}
}
public (Transform sourceTransform, Table sourceTable) GetSourceTransform(DexihHub hub, DexihDatalinkTable hubDatalinkTable, InputColumn[] inputColumns, TransformWriterOptions transformWriterOptions)
{
try
{
Transform sourceTransform;
Table sourceTable;
var referenceTableAlias = hubDatalinkTable.Key.ToString();
switch (hubDatalinkTable.SourceType)
{
case ESourceType.Datalink:
var datalink = hub.DexihDatalinks.SingleOrDefault(c => c.Key == hubDatalinkTable.SourceDatalinkKey);
if (datalink == null)
{
throw new TransformManagerException($"The source datalink with the key {hubDatalinkTable.SourceDatalinkKey} was not found");
}
(sourceTransform, sourceTable) = CreateRunPlan(hub, datalink, inputColumns, null, false, null);
break;
case ESourceType.Table:
if (hubDatalinkTable.SourceTableKey == null)
{
throw new TransformManagerException($"The source table key was null.");
}
var sourceDbTable = hub.GetTableFromKey(hubDatalinkTable.SourceTableKey.Value);
if (sourceDbTable == null)
{
throw new TransformManagerException($"The source table with the key {hubDatalinkTable.SourceTableKey.Value} could not be found.");
}
var sourceDbConnection = hub.DexihConnections.SingleOrDefault(c => c.Key == sourceDbTable.ConnectionKey && c.IsValid);
if (sourceDbConnection == null)
{
throw new TransformException($"The connection with key {sourceDbTable.ConnectionKey} could not be found.");
}
var sourceConnection = sourceDbConnection.GetConnection(_transformSettings);
sourceTable = sourceDbTable.GetTable(hub, sourceConnection, inputColumns, _transformSettings, referenceTableAlias);
if (hubDatalinkTable.DisableVersioning)
{
sourceTable.IsVersioned = false;
}
sourceTransform = sourceConnection.GetTransformReader(sourceTable, transformWriterOptions.PreviewMode);
break;
case ESourceType.Rows:
var rowCreator = new ReaderRowCreator();
rowCreator.InitializeRowCreator(hubDatalinkTable.RowsStartAt ?? 1, hubDatalinkTable.RowsEndAt ?? 1, hubDatalinkTable.RowsIncrement ?? 1);
rowCreator.TableAlias = hubDatalinkTable.Key.ToString();
sourceTable = rowCreator.GetTable();
sourceTransform = rowCreator;
break;
case ESourceType.Function:
sourceTable = hubDatalinkTable.GetTable(null, inputColumns);
var data = new object[sourceTable.Columns.Count];
for (var i = 0; i < sourceTable.Columns.Count; i++)
{
data[i] = sourceTable.Columns[i].DefaultValue;
}
sourceTable.Data.Add(data);
var defaultRow = new ReaderDynamic(sourceTable);
defaultRow.Reset();
sourceTransform = defaultRow;
break;
default:
throw new TransformManagerException($"Error getting the source transform.");
}
sourceTransform.TableAlias = referenceTableAlias;
// compare the table in the transform to the source datalink columns. If any are missing, add a mapping
// transform to include them.
var transformColumns = sourceTransform.CacheTable.Columns;
var datalinkColumns = hubDatalinkTable.DexihDatalinkColumns;
// add a mapping transform to include inputColumns.
var mappings = new Mappings();
foreach (var column in datalinkColumns.Where(c => c.DeltaType != EDeltaType.IgnoreField))
{
var transformColumn = transformColumns.SingleOrDefault(c => c.Name == column.Name);
if (transformColumn == null)
{
var newColumn = column.GetTableColumn(inputColumns);
mappings.Add(new MapInputColumn(newColumn));
}
else
{
transformColumn.DeltaType = column.DeltaType;
transformColumn.IsIncrementalUpdate = column.IsIncrementalUpdate;
}
}
if (mappings.Count > 0)
{
sourceTransform = new TransformMapping(sourceTransform, mappings)
{
Name = "Input Column Mapping"
};
}
sourceTransform.IgnoreQuery = hubDatalinkTable.DisablePushDown;
return(sourceTransform, sourceTable);
}
catch (Exception ex)
{
throw new TransformManagerException($"Get source transform failed. {ex.Message}", ex);
}
}