public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode == ExitCodeType.Success || exitCode == ExitCodeType.OperationNotRequired)
{
int countOfEntriesThatDisapeared = _entriesUnzipped.Count(e => !e.Exists);
if (countOfEntriesThatDisapeared != 0)
{
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning,
countOfEntriesThatDisapeared + " of " + _entriesUnzipped.Count + " entries were created by " +
GetType().Name +
" during unzip phase but had disapeared at cleanup time - following successful data load"));
}
//cleanup required
foreach (FileInfo f in _entriesUnzipped.Where(e => e.Exists))
{
try
{
f.Delete();
}
catch (Exception e)
{
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Could not delete file " + f.FullName, e));
}
}
}
}
public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
while (_toDispose.Any())
{
_toDispose.Pop().LoadCompletedSoDispose(exitCode, postLoadEventListener);
}
}
public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
{
if (Skip(job))
{
return(ExitCodeType.Error);
}
ExitCodeType toReturn = ExitCodeType.Success; //This default will be returned unless there is an explicit DataProvider or collection of runtime tasks to run which return a different result (See below)
// Figure out where we are getting the source files from
try
{
if (Components.Any())
{
toReturn = base.Run(job, cancellationToken);
}
else if (job.LoadDirectory.ForLoading.EnumerateFileSystemInfos().Any())
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Using existing files in '" + job.LoadDirectory.ForLoading.FullName + "', there are no GetFiles processes or DataProviders configured"));
}
else
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "There are no GetFiles tasks and there are no files in the ForLoading directory (" + job.LoadDirectory.ForLoading.FullName + ")"));
}
}
finally
{
// We can only clean up ForLoading after the job is finished, so give it the necessary disposal operation
job.PushForDisposal(new DeleteForLoadingFilesOperation(job));
}
return(toReturn);
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
{
foreach (DiscoveredTable rawTable in _rawTables)
{
rawTable.Drop();
}
}
public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
//dont bother cleaning up if it bombed
if (exitCode == ExitCodeType.Error)
{
return;
}
//its Abort,Success or LoadNotRequired
// Detach database
try
{
var dbToDropName = _dbInfo.GetRuntimeName();
if (!dbToDropName.EndsWith("_RAW"))
{
throw new Exception("We were in the cleanup phase and were about to drop the database that was created by MDFAttacher when we noticed it's name didn't end with _RAW!, it's name was:" + dbToDropName + " were we about to nuke your live database?");
}
_dbInfo.Drop();
DeleteFilesIfExist();
}
catch (Exception e)
{
throw new Exception("Could not detach database '" + _dbInfo.GetRuntimeName() + "': " + e);
}
}
public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode == ExitCodeType.Success && DeleteFilesOffFTPServerAfterSuccesfulDataLoad)
{
foreach (string file in _filesRetrieved)
{
FtpWebRequest reqFTP;
reqFTP = (FtpWebRequest)FtpWebRequest.Create(new Uri(file));
reqFTP.Credentials = new NetworkCredential(_username, _password);
reqFTP.KeepAlive = false;
reqFTP.Method = WebRequestMethods.Ftp.DeleteFile;
reqFTP.UseBinary = true;
reqFTP.Proxy = null;
reqFTP.UsePassive = true;
reqFTP.EnableSsl = _useSSL;
FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse();
if (response.StatusCode != FtpStatusCode.FileActionOK)
{
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Attempt to delete file at URI " + file + " resulted in response with StatusCode = " + response.StatusCode));
}
else
{
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Deleted FTP file at URI " + file + " status code was " + response.StatusCode));
}
response.Close();
}
}
}
override public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode != ExitCodeType.Success)
{
return;
}
base.LoadCompletedSoDispose(exitCode, postLoadEventListener);
foreach (KeyValuePair <DateTime, FileInfo> keyValuePair in _cacheFileMappings)
{
if (keyValuePair.Value == null)
{
continue;
}
try
{
keyValuePair.Value.Delete();
}
catch (IOException e)
{
Job.LogWarning(GetType().FullName, "Could not delete cached file " + keyValuePair.Value + " (" + e.Message + ")make sure to delete it manually otherwise Schedule and file system will be desynched");
}
}
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
{
while (_disposalStack.Any())
{
var disposable = _disposalStack.Pop();
disposable.LoadCompletedSoDispose(exitCode, postLoadEventsListener);
}
}
/// <summary>
/// sets the exit code.
/// </summary>
public int UpdateExitCode(ExitCodeType NewExitCode)
{
if (ExitCode == 0)
{
ExitCode = (int)NewExitCode;
}
return(ExitCode);
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode == ExitCodeType.Error)
{
return;
}
DropStaging();
}
/// <summary>
/// Runs the DLE using a custom names for RAW/STAGING. Pass in the load to execute and the files/directories to process
/// in the batch.
/// </summary>
/// <param name="lmd"></param>
/// <param name="payload"></param>
/// <returns>The exit code of the data load after it completes</returns>
private ExitCodeType RunDLE(LoadMetadata lmd, object payload)
{
var catalogueRepository = (CatalogueRepository)lmd.Repository;
//ensures that RAW/STAGING always have unique names
_configuration = new HICDatabaseConfiguration(lmd, _namer);
_configuration.UpdateButDoNotDiff = new Regex("^MessageGuid");
var logManager = catalogueRepository.GetDefaultLogManager();
logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask());
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(lmd, _configuration, new HICLoadConfigurationFlags(),
catalogueRepository, logManager);
var stagingCreator = _namer as ICreateAndDestroyStagingDuringLoads;
if (stagingCreator != null)
{
stagingCreator.CreateStaging(lmd.GetDistinctLiveDatabaseServer());
}
var listener = new NLogThrowerDataLoadEventListener(NLog.LogManager.GetCurrentClassLogger());
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IExternalDatabaseServer raw = catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.RAWDataLoadServer);
DiscoveredServer liveDb = lmd.GetDistinctLiveDatabaseServer();
//do we want to try to cut down the time it takes to do RAW=>STAGING by using INSERT INTO instead of running anonymisation/migration pipeline
if (_useInsertIntoForRawMigration)
{
//if it is on the same server swap out the migration engine for INSERT INTO
if (raw == null || (raw.Server != null && raw.Server.Equals(liveDb.Name) && raw.DatabaseType == liveDb.DatabaseType))
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "SWAPPING RAW=>STAGING migration strategy to INSERT INTO"));
SwapMigrateRAWToStagingComponent(execution.Components);
}
else
{
//Cannot use because different servers / DatabaseTypes.
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "CANNOT SWAP RAW=>STAGING migration strategy to INSERT INTO because RAW is on '" + raw.Server + "' (" + raw.DatabaseType + ") and STAGING is on '" + liveDb.Name + "' (" + liveDb.DatabaseType + ")"));
}
}
else
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Flag is false for SWAP RAW=>STAGING migration strategy to INSERT INTO So won't do it"));
}
var procedure = new DataLoadProcess(_repositoryLocator, lmd, null, logManager, listener, execution, _configuration);
ExitCodeType exitCode = procedure.Run(new GracefulCancellationToken(), payload);
return(exitCode);
}
public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode == ExitCodeType.Success)
{
DateToSetProgressTo = _delegateToRun();
}
base.LoadCompletedSoDispose(exitCode, postLoadEventListener);
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
{
if (exitCode == ExitCodeType.Success)
{
if (DeleteFilesOnsuccessfulLoad)
{
foreach (FileInfo f in _files)
{
postLoadEventsListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to delete " + f.FullName));
f.Delete();
}
}
}
}
private void TryDispose(ExitCodeType exitCode, IDataLoadJob job)
{
try
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Disposing disposables..."));
job.LoadCompletedSoDispose(exitCode, job);
job.CloseLogging();
}
catch (Exception e)
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Job " + job.JobID + " crashed again during disposing", e));
throw;
}
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
//dont bother cleaning up if it bombed
if (exitCode == ExitCodeType.Error)
{
return;
}
//its Abort,Success or LoadNotRequired
foreach (var cloneOperation in _tablesCreated)
{
cloneOperation.Undo();
}
foreach (var dbInfo in _databasesCreated)
{
dbInfo.Drop();
}
}
virtual public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode != ExitCodeType.Success)
{
return;
}
var progress = Job.LoadProgress;
if (progress.DataLoadProgress > DateToSetProgressTo)
{
throw new DataLoadProgressUpdateException("Cannot set DataLoadProgress to " + DateToSetProgressTo + " because it is less than the currently recorded progress:" + progress.DataLoadProgress);
}
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Updating DataLoadProgress of '" + progress + "' to " + DateToSetProgressTo));
progress.DataLoadProgress = DateToSetProgressTo;
progress.SaveToDatabase();
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
// We only delete ForLoading files after a successful load
if (exitCode == ExitCodeType.Success)
{
var LoadDirectory = _job.LoadDirectory;
//if there are no files and there are no directories
if (!LoadDirectory.ForLoading.GetFiles().Any() && !LoadDirectory.ForLoading.GetDirectories().Any())
{
//just skip it but tell user you are skipping it
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "No files found in ForLoading so not bothering to try and delete."));
return;
}
// Check if the attacher has communicated its intent to handle archiving
var archivingHandledByAttacher = File.Exists(Path.Combine(LoadDirectory.ForLoading.FullName, "attacher_is_handling_archiving"));
if (!archivingHandledByAttacher && !ArchiveHasBeenCreated())
{
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Refusing to delete files in ForLoading: the load has reported success but there is no archive of this dataset (was expecting the archive to be called '" + _job.ArchiveFilepath + "', check LoadMetadata.CacheArchiveType if the file extension is not what you expect)"));
return;
}
_job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Deleting files in ForLoading (" + LoadDirectory.ForLoading.FullName + ")"));
if (archivingHandledByAttacher)
{
LoadDirectory.ForLoading.EnumerateFiles().Where(info => info.Name != "attacher_is_handling_archiving").ToList().ForEach(info => info.Delete());
LoadDirectory.ForLoading.EnumerateDirectories().Where(info => info.Name != "__hidden_from_archiver__").ToList().ForEach(info => info.Delete(true));
}
else
{
LoadDirectory.ForLoading.EnumerateFiles().ToList().ForEach(info => info.Delete());
LoadDirectory.ForLoading.EnumerateDirectories().ToList().ForEach(info => info.Delete(true));
}
}
}
public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
if (exitCode == ExitCodeType.Success)
{
using (var sftp = new SftpClient(_host, _username, _password))
{
sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);
sftp.Connect();
foreach (string retrievedFiles in _filesRetrieved)
{
try
{
sftp.DeleteFile(retrievedFiles);
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Deleted SFTP file " + retrievedFiles + " from SFTP server"));
}
catch (Exception e)
{
postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Could not delete SFTP file " + retrievedFiles + " from SFTP server", e));
}
}
}
}
}
public abstract void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener);
public void AddCode(ExitCodeType codeType)
{
codeList.Add(codeType);
}
public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
}
public bool ContainsCode(ExitCodeType codeType)
{
return(codeList.Contains(codeType));
}
public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
{
}
public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
{
throw new NotImplementedException();
}
public override object GetParametersInstance()
{
WorkflowParameters wfp = new WorkflowParameters();
wfp.Servers = new List <string>();
wfp.Servers.Add("localhost");
wfp.Servers.Add("server001");
wfp.RunAs = new ImpersonationType();
wfp.RunAs.User = "******";
wfp.RunAs.Password = "******";
wfp.RunAs.Protocol = RunAsProtocolType.WinRm;
wfp.RunUsing = RunUsingProtocolType.ProcessStart;
wfp.WorkingDir = @"C:\Temp";
wfp.Timeout = new TimeoutType();
wfp.Timeout.Value = 60000;
wfp.Timeout.ActionOnTimeout = "Error";
wfp.Timeout.KillProcess = "true";
wfp.ValidExitCodes = new List <ExitCodeType>();
ExitCodeType code = new ExitCodeType();
code.Operator = Operators.NotEqualTo;
code.value = "0";
wfp.Ant = new AntCommandType();
wfp.Ant.Home = @"C:\ant";
wfp.Ant.BuildFile = @"C:\Temp\build.xml";
wfp.Ant.Target = "MyAntTarget";
wfp.Powershell = new PowershellCommandType();
wfp.Powershell.ArgType = new ArgTypeType();
wfp.Powershell.ArgType.PrefixWith = "-";
wfp.Powershell.ArgType.UseQuotes = "true";
wfp.Powershell.ArgType.JoinWith = ", ";
wfp.Powershell.Command = "hostname";
wfp.Powershell.ExecutionPolicy = ExecutionPolicyType.Bypass;
wfp.Powershell.Script = @"C:\Script\MyScript.ps1";
wfp.Plink = new PlinkCommandType();
wfp.Plink.ArgType = new ArgTypeType();
wfp.Plink.ArgType.SkipBlankValues = "true";
wfp.Plink.ArgType.JoinWith = " ";
wfp.Plink.ArgType.PrefixWith = "";
wfp.Plink.ArgType.UseQuotes = "false";
wfp.Plink.Command = "ls";
wfp.Plink.CommandFile = new CommandFileType();
wfp.Plink.CommandFile.Name = @"MyCommandFile.sh";
wfp.Plink.CommandFile.CodePage = "";
wfp.Plink.PrivateKey = "MyKey.ppk";
wfp.Plink.RunInteractive = true;
wfp.Plink.Users = new List <UserType>();
UserType user = new UserType();
user.Name = "user@server";
user.Password = "******";
wfp.Plink.Users.Add(user);
wfp.Pscp = new PscpCommandType();
wfp.Pscp.Destination = @"\Users\user\temp";
wfp.Pscp.KeepFileAttributes = "true";
wfp.Pscp.PrivateKey = "MyKey.ppk";
wfp.Pscp.Sources = new List <string>();
wfp.Pscp.Sources.Add("server001");
wfp.Pscp.Users = new List <UserType>();
UserType user2 = new UserType();
user2.Name = "user@server";
user2.Password = "******";
wfp.Pscp.Users.Add(user2);
String xml = wfp.Serialize(false);
xml = xml.Substring(xml.IndexOf("<"));
return(xml);
}
public ExitCode(ExitCodeType Code, string Msg)
{
this.Code = Code;
this.Msg = Msg;
}
public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postDataLoadEventListener)
{
MutilateDataTables.LoadCompletedSoDispose(exitCode, postDataLoadEventListener);
}
