/// <summary>
/// Backup a repository in according to the source and destination.
/// </summary>
public void Backup()
{
DisplayHeader();
SourceSettings srcSettings = new SourceSettings()
{
Repository = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.Source).ConvertToSourceRepository(),
Path = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.SourcePath),
CheckLevel = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.Check).ConvertToCheckLevel(),
NoCleaning = ConsoleHelpers.CommandArgExists(_arguments, Arguments.NoCleaning),
WideDisplay = ConsoleHelpers.CommandArgExists(_arguments, Arguments.Wide),
ApiKey = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.SourceApiKey),
NoLocalDelete = ConsoleHelpers.CommandArgExists(_arguments, Arguments.NoLocalDelete)
};
DestinationSettings dstSettings = new DestinationSettings()
{
Repository = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.Dest).ConvertToDestinationRepository(),
Path = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.DestPath),
ApiKey = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.DestApiKey)
};
TransformSettings trfSettings = new TransformSettings()
{
Kind = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.Transform).ConvertToTransformKind(),
TripleDesFilename = ConsoleHelpers.GetCommandArgValue(_arguments, Arguments.CryptoKey)
};
IRepositorySource source = new RepositorySourceFactory(_logger, this).Create(srcSettings, dstSettings, trfSettings);
WriteComponentLabels(source);
source.Process();
DisplayFooter();
}
private void btnStart_Click(object sender, EventArgs e)
{
if (string.IsNullOrEmpty(inputDirTextBox.Text))
{
MessageBox.Show("Input directory cannot be none!");
return;
}
if (string.IsNullOrEmpty(outputDirTextBox.Text))
{
MessageBox.Show("Output directory cannot be none!");
return;
}
if (string.IsNullOrEmpty(formatComboBox.Text))
{
MessageBox.Show("You must choose a format");
return;
}
int searchDepth = TransformSettings.getSearchDepth();
FileFilter fileFilter = getFileFilter();
if (TransformSettings.isKeepOriginDirs())
{
transformFilesKeepDirs(inputDirTextBox.Text, outputDirTextBox.Text, fileFilter, 0, searchDepth);
}
else
{
transformFilesNotKeepDirs(inputDirTextBox.Text, outputDirTextBox.Text, fileFilter, 0, searchDepth);
}
MessageBox.Show("Transform Complete!");
}
public DatajobRun(TransformSettings transformSettings, ILogger logger, DexihDatajob datajob, DexihHub hub, TransformWriterOptions transformWriterOptions, IAlertQueue alertQueue, string[] alertEmails)
{
_transformSettings = transformSettings;
_logger = logger;
_alertQueue = alertQueue;
_alertEmails = alertEmails;
_transformWriterOptions = transformWriterOptions;
if (datajob.AuditConnectionKey > 0)
{
var dbAuditConnection =
hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == datajob.AuditConnectionKey);
if (dbAuditConnection == null)
{
throw new DatalinkRunException(
$"Audit connection with key {datajob.AuditConnectionKey} was not found.");
}
_auditConnection = dbAuditConnection.GetConnection(_transformSettings);
}
else
{
_auditConnection = new ConnectionMemory();
}
Datajob = datajob;
_hub = hub;
}
public DownloadData(TransformSettings transformSettings, CacheManager cache, DownloadObject[] downloadObjects, EDownloadFormat downloadFormat, bool zipFiles)
{
TransformSettings = transformSettings;
Cache = cache;
DownloadObjects = downloadObjects;
DownloadFormat = downloadFormat;
ZipFiles = zipFiles;
}
public DatalinkTestRun(
TransformSettings transformSettings,
ILogger logger,
DexihDatalinkTest datalinkTest,
DexihHub hub,
TransformWriterOptions transformWriterOptions,
IAlertQueue alertQueue,
string[] alertEmails
)
{
_transformSettings = transformSettings;
_transformWriterOptions = transformWriterOptions;
_logger = logger;
// create a copy of the hub as the test run will update objects.
_hub = hub.Serialize().Deserialize <DexihHub>();
// _hub = hub.CloneProperties();
_datalinkTest = datalinkTest;
_alertQueue = alertQueue;
_alertEmails = alertEmails;
Connection auditConnection;
if (datalinkTest.AuditConnectionKey > 0)
{
var dbAuditConnection =
_hub.DexihConnections.SingleOrDefault(c => c.IsValid && c.Key == datalinkTest.AuditConnectionKey);
if (dbAuditConnection == null)
{
throw new DatalinkRunException(
$"Audit connection with key {datalinkTest.AuditConnectionKey} was not found.");
}
auditConnection = dbAuditConnection.GetConnection(_transformSettings);
}
else
{
auditConnection = new ConnectionMemory();
}
TestResults = new List <TestResult>();
WriterResult = new TransformWriterResult()
{
AuditConnection = auditConnection,
AuditConnectionKey = datalinkTest.AuditConnectionKey ?? 0,
AuditType = Constants.DatalinkTest,
HubKey = _hub.HubKey,
ReferenceKey = datalinkTest.Key,
ParentAuditKey = 0,
ReferenceName = datalinkTest.Name,
SourceTableKey = 0,
SourceTableName = "",
TransformWriterOptions = _transformWriterOptions
};
}
public (string securityKey, DexihApi api) ActivateApi(AutoStart autoStart)
{
var dbApi = autoStart.Hub.DexihApis.SingleOrDefault(c => c.IsValid && c.Key == autoStart.Key);
if (dbApi == null)
{
throw new Exception($"Api with key {autoStart.Key} was not found");
}
_logger.LogInformation("Starting API - {api}.", dbApi.Name);
var settings = new TransformSettings()
{
HubVariables = autoStart.HubVariables,
RemoteSettings = _sharedSettings.RemoteSettings,
ClientFactory = _clientFactory
};
var hub = autoStart.Hub;
string key;
Transform transform;
if (dbApi.SourceType == ESourceType.Table)
{
var dbTable = hub.GetTableFromKey(dbApi.SourceTableKey.Value);
var dbConnection = hub.DexihConnections.Single(c => c.IsValid && c.Key == dbTable.ConnectionKey);
var connection = dbConnection.GetConnection(settings);
var table = dbTable.GetTable(hub, connection, settings);
transform = connection.GetTransformReader(table);
}
else
{
var dbDatalink =
hub.DexihDatalinks.Single(c => c.IsValid && c.Key == dbApi.SourceDatalinkKey.Value);
var transformOperations = new TransformsManager(settings);
var runPlan = transformOperations.CreateRunPlan(hub, dbDatalink, null, null, null, null);
transform = runPlan.sourceTransform;
}
transform.SetCacheMethod(dbApi.CacheQueries ? ECacheMethod.LookupCache : ECacheMethod.NoCache);
key = Add(hub.HubKey, autoStart.Key, transform, dbApi.CacheResetInterval, autoStart.SecurityKey, dbApi.SelectQuery);
return(key, dbApi);
}
private async Task <(Transform transform, string name)> GetDatalinkTransform(long key, long?datalinkTransformKey, InputColumn[] inputColumns, InputParameters inputParameters, SelectQuery selectQuery, CancellationToken cancellationToken)
{
var dbDatalink =
Cache.Hub.DexihDatalinks.SingleOrDefault(c => c.Key == key);
if (dbDatalink == null)
{
throw new DownloadDataException(
$"The datalink with key {key} could not be found in the cache.");
}
dbDatalink.UpdateParameters(inputParameters);
var transformWriterOptions = new TransformWriterOptions()
{
PreviewMode = true,
};
var transformSettings = new TransformSettings()
{
HubVariables = TransformSettings.HubVariables,
InputParameters = dbDatalink.Parameters.ToArray(),
RemoteSettings = TransformSettings.RemoteSettings,
ClientFactory = TransformSettings.ClientFactory
};
var transformManager = new TransformsManager(transformSettings);
//Get the last Transform that will load the target table.
var runPlan = transformManager.CreateRunPlan(Cache.Hub, dbDatalink,
inputColumns, datalinkTransformKey, null,
transformWriterOptions);
var transform = runPlan.sourceTransform;
var openReturn = await transform.Open(0, selectQuery, cancellationToken);
if (!openReturn)
{
throw new DownloadDataException(
$"The datalink {dbDatalink.Name} failed to open for reading.");
}
// transform.SetCacheMethod(ECacheMethod.DemandCache);
transform.SetEncryptionMethod(EEncryptionMethod.MaskSecureFields, "");
return(transform, dbDatalink.Name);
}
private void transformFilesKeepDirs(string inputPath, string outputPath, FileFilter fileFilter, int curDepth, int maxDepth)
{
// if current search depth > max search depth, then quit.
if (curDepth >= maxDepth)
{
return;
}
// first, check the directory in the output path is exists, if not exists, then create one.
if (!Directory.Exists(outputPath))
{
Directory.CreateDirectory(outputPath);
}
// second, get all the sub-directories and traserse it.
DirectoryInfo directoryInfo = new DirectoryInfo(inputPath);
DirectoryInfo[] childDirectoryInfos = directoryInfo.GetDirectories();
foreach (DirectoryInfo childDirectoryInfo in childDirectoryInfos)
{
string path = childDirectoryInfo.FullName;
string curDir = FileUtil.getCurDir(path);
transformFilesKeepDirs(path, outputPath + "\\" + curDir, fileFilter, curDepth + 1, maxDepth);
}
// third, get all the sub-files and use file filter to get image files.
FileInfo[] fileInfos = FileUtil.getSubFileInfos(inputPath, fileFilter).ToArray <FileInfo>();
// forth, accroding to the settings to transform all the files.
int length = fileInfos.Length;
mainProgressBar.Maximum = length;
mainProgressBar.Value = 0;
for (int i = 0; i < length; ++i)
{
string name;
if (TransformSettings.isKeepOriginName())
{
name = FileUtil.getName(fileInfos[i].FullName);
}
else
{
name = i.ToString();
}
ImageUtil.transform(fileInfos[i], string.Format("{0}\\{1}.{2}", outputPath, name, formatComboBox.Text));
mainProgressBar.PerformStep();
}
}
private void transformFilesNotKeepDirs(string inputPath, string outputPath, FileFilter fileFilter, int curDepth, int maxDepth)
{
IList <FileInfo> fileInfoList = new List <FileInfo>();
Queue <DirectoryInfo> dirInfoQueue = new Queue <DirectoryInfo>();
dirInfoQueue.Enqueue(new DirectoryInfo(inputPath));
while (dirInfoQueue.Count > 0)
{
DirectoryInfo directoryInfo = dirInfoQueue.Dequeue();
DirectoryInfo[] childDirectoryInfos = directoryInfo.GetDirectories();
foreach (DirectoryInfo childDirectoryInfo in childDirectoryInfos)
{
dirInfoQueue.Enqueue(childDirectoryInfo);
}
fileInfoList = fileInfoList.Concat <FileInfo>(FileUtil.getSubFileInfos(directoryInfo.FullName, fileFilter)).ToList <FileInfo>();
}
FileInfo[] fileInfos = fileInfoList.ToArray <FileInfo>();
int length = fileInfos.Length;
mainProgressBar.Maximum = length;
mainProgressBar.Value = 0;
for (int i = 0; i < length; ++i)
{
string name;
if (TransformSettings.isKeepOriginName())
{
name = FileUtil.getName(fileInfos[i].FullName);
}
else
{
name = i.ToString();
}
ImageUtil.transform(fileInfos[i], string.Format("{0}\\{1}.{2}", outputPath, name, formatComboBox.Text));
mainProgressBar.PerformStep();
}
}
/// <summary>
/// Create a <see cref="IRepositorySource"/> instance in according to the command arguments
/// </summary>
public IRepositorySource Create(SourceSettings srcSettings, DestinationSettings dstSettings, TransformSettings trfSettings)
{
srcSettings = srcSettings ?? throw new ArgumentNullException("srcSettings");
dstSettings = dstSettings ?? throw new ArgumentNullException("dstSettings");
trfSettings = trfSettings ?? throw new ArgumentNullException("trfSettings");
IRepositorySource source;
switch (srcSettings.Repository)
{
case SourceRepository.Local:
{
Helpers.MandatoryValue("source path", srcSettings.Path);
source = new LocalDirectorySource(srcSettings.Path, _progress, srcSettings.WideDisplay,
srcSettings.CheckLevel, srcSettings.NoCleaning);
break;
}
case SourceRepository.GCS:
{
Helpers.MandatoryValue("source path", srcSettings.Path);
Helpers.MandatoryValue("source api key filename", srcSettings.ApiKey);
source = new GcsBucketSource(srcSettings.Path, srcSettings.ApiKey, _progress, srcSettings.WideDisplay);
break;
}
case SourceRepository.S3:
{
Helpers.MandatoryValue("source path", srcSettings.Path);
Helpers.MandatoryValue("source api key filename", srcSettings.ApiKey);
source = new S3BucketSource(srcSettings.Path, srcSettings.ApiKey, _progress, srcSettings.WideDisplay);
break;
}
default:
throw new SmkException($"Unsupported repository source '{srcSettings.Repository}");
}
source.Transform = new TransformFactory(_logger).Create(trfSettings);
source.Destination = new RepositoryDestinationFactory(_logger, source.Transform).Create(dstSettings);
source.Logger = _logger;
return(source);
}
public async Task <bool> Run(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
WriterResult.StartTime = DateTime.Now;
WriterResult.LastUpdateTime = DateTime.Now;
var runStatus = WriterResult.SetRunStatus(ERunStatus.Started, null, null);
if (!runStatus)
{
throw new DatajobRunException($"Failed to set status");
}
if (_transformWriterOptions.TargetAction == TransformWriterOptions.ETargetAction.Truncate)
{
runStatus = WriterResult.SetRunStatus(ERunStatus.Started, "Truncating tables...", null);
var targetTables = new HashSet <DexihTable>();
foreach (var step in Datajob.DexihDatalinkSteps.Where(c => c.IsValid))
{
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatajobRunException($"The datalink in the step {step.Name} with the datalink key {step.DatalinkKey} cound not be found.");
}
foreach (var target in datalink.DexihDatalinkTargets)
{
if (target.TableKey > 0)
{
var table = _hub.GetTableFromKey(target.TableKey);
if (table != null)
{
targetTables.Add(table);
}
}
}
}
// this loops through attempting to truncate tables one by one.
// is repeats when there are failures to accomodate for some table having foriegn keys
var atLeastOneSuccess = true;
var atLeastOneFail = true;
while (atLeastOneSuccess && atLeastOneFail)
{
atLeastOneSuccess = false;
atLeastOneFail = false;
var newTagetTables = new HashSet <DexihTable>();
foreach (var dbTable in targetTables)
{
var dbConnection = _hub.DexihConnections.SingleOrDefault(c => c.Key == dbTable.ConnectionKey);
if (dbConnection == null)
{
throw new DatajobRunException($"The connection for the table {dbTable.Name} with the connection key {dbTable.ConnectionKey} could not be found.");
}
var connection = dbConnection.GetConnection(_transformSettings);
var table = dbTable.GetTable(_hub, connection, _transformSettings);
try
{
await connection.TruncateTable(table, cancellationToken);
atLeastOneSuccess = true;
}
catch
{
atLeastOneFail = true;
newTagetTables.Add(dbTable);
}
}
targetTables = newTagetTables;
}
if (targetTables.Count > 0)
{
var message = $"The job failed as the following tables could not be truncated: {string.Join(", ", targetTables.Select(c => c.Name))}";
WriterResult.SetRunStatus(ERunStatus.Abended, message, null);
return(false);
}
}
var inputParameters = new InputParameters();
foreach (var parameter in Datajob.Parameters)
{
inputParameters.Add(new InputParameter()
{
Name = parameter.Name, Value = parameter.Value, Rank = parameter.Rank
});
}
//start all jobs async
foreach (var step in Datajob.DexihDatalinkSteps.Where(c => c.IsValid))
{
var datalink = _hub.DexihDatalinks.SingleOrDefault(c => c.IsValid && c.Key == step.DatalinkKey);
if (datalink == null)
{
throw new DatajobRunException($"The step {step.Name} contains a datalink with the key {step.DatalinkKey} which can not be found.");
}
foreach (var parameter in step.Parameters)
{
parameter.Value = inputParameters.SetParameters(parameter.Value, parameter.Rank);
}
var transformSettings = new TransformSettings
{
HubVariables = _transformSettings.HubVariables,
RemoteSettings = _transformSettings.RemoteSettings,
InputParameters = step.Parameters.ToArray <InputParameterBase>(),
ClientFactory = _transformSettings.ClientFactory
};
var inputColumns = step.DexihDatalinkStepColumns.Select(c => c.ToInputColumn()).ToArray();
var datalinkRun = new DatalinkRun(transformSettings, _logger, WriterResult.AuditKey, datalink, _hub, inputColumns, _transformWriterOptions, _alertQueue, _alertEmails)
{
DatalinkStepKey = step.Key
};
DatalinkSteps.Add(datalinkRun);
//start datalinks that have no dependencies.
if (step.DexihDatalinkDependencies == null || step.DexihDatalinkDependencies.Count == 0)
{
StartDatalink(datalinkRun);
}
}
WriterResult.SetRunStatus(ERunStatus.Running, null, null);
await WaitUntilFinished();
return(true);
}
catch (OperationCanceledException)
{
Cancel();
await WaitUntilFinished();
WriterResult.SetRunStatus(ERunStatus.Cancelled, "Datajob was cancelled", null);
throw new DatajobRunException($"The datajob {Datajob.Name} was cancelled.");
}
catch (Exception ex)
{
Cancel();
await WaitUntilFinished();
var message = $"The job {Datajob.Name} failed. {ex.Message}";
WriterResult?.SetRunStatus(ERunStatus.Abended, message, ex);
throw new DatajobRunException(message, ex);
}
finally
{
await WriterResult.CompleteDatabaseWrites();
OnFinish?.Invoke(this);
}
}
public ColumnValidationRun(TransformSettings transformSettings, DexihColumnValidation columnValidation, DexihHub hub)
{
ColumnValidation = columnValidation;
Hub = hub;
_transformSettings = transformSettings;
}
public TransformsManager(TransformSettings transformSettings, ILogger logger)
{
_transformSettings = transformSettings;
_logger = logger;
}
public TransformsManager(TransformSettings transformSettings)
{
_transformSettings = transformSettings;
}
