protected override void Download(string file, ILoadDirectory destination, IDataLoadEventListener job)
{
if (file.Contains("/") || file.Contains("\\"))
{
throw new Exception("Was not expecting a relative path here");
}
Stopwatch s = new Stopwatch();
s.Start();
using (var sftp = new SftpClient(_host, _username, _password))
{
sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds);
sftp.Connect();
//if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList())
string fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file;
string destinationFilePath = Path.Combine(destination.ForLoading.FullName, file);
//register for events
Action <ulong> callback = (totalBytes) => job.OnProgress(this, new ProgressEventArgs(destinationFilePath, new ProgressMeasurement((int)(totalBytes * 0.001), ProgressType.Kilobytes), s.Elapsed));
using (var fs = new FileStream(destinationFilePath, FileMode.CreateNew))
{
//download
sftp.DownloadFile(fullFilePath, fs, callback);
fs.Close();
}
_filesRetrieved.Add(fullFilePath);
}
s.Stop();
}
private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
{
string[] files = GetFileList();
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, files.Aggregate("Identified the following files on the FTP server:", (s, f) => f + ",").TrimEnd(',')));
bool forLoadingContainedCachedFiles = false;
foreach (string file in files)
{
var action = GetSkipActionForFile(file, destination);
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "File " + file + " was evaluated as " + action));
if (action == SkipReason.DoNotSkip)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to download " + file));
Download(file, destination, listener);
}
if (action == SkipReason.InForLoading)
{
forLoadingContainedCachedFiles = true;
}
}
//if no files were downloaded (and there were none skiped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
if (!forLoadingContainedCachedFiles && !_filesRetrieved.Any() && SendLoadNotRequiredIfFileNotFound)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Could not find any files on the remote server worth downloading, so returning LoadNotRequired"));
return(ExitCodeType.OperationNotRequired);
}
//otherwise it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
return(ExitCodeType.Success);
}
private bool FilesInForLoadingMatchWorkload(ILoadDirectory directory)
{
var filesInForLoading = GetPathsRelativeToDirectory(directory.ForLoading.EnumerateFiles("*", SearchOption.AllDirectories).ToArray(), directory.ForLoading);
var filesFromCache = GetPathsRelativeToDirectory(_workload.Values.ToArray(), directory.Cache);
return(filesInForLoading.OrderBy(t => t).SequenceEqual(filesFromCache.OrderBy(t => t)));
}
private void UnzipWithEvents(ZipArchiveEntry entry, ILoadDirectory destination, IDataLoadJob job)
{
//create a task
string entryDestination = Path.Combine(destination.ForLoading.FullName, entry.Name);
Task unzipJob = Task.Factory.StartNew(() => entry.ExtractToFile(entryDestination, true));
//create a stopwatch to time how long bits take
Stopwatch s = new Stopwatch();
s.Start();
FileInfo f = new FileInfo(entryDestination);
_entriesUnzipped.Add(f);
//monitor it
while (!unzipJob.IsCompleted)
{
Thread.Sleep(200);
if (f.Exists)
{
job.OnProgress(this, new ProgressEventArgs(entryDestination, new ProgressMeasurement((int)(f.Length / 1000), ProgressType.Kilobytes), s.Elapsed));
}
}
s.Stop();
}
public bool Validate(ILoadDirectory destination)
{
if (UriToFile == null || string.IsNullOrWhiteSpace(UriToFile.PathAndQuery))
{
throw new MissingFieldException("PathToFile is null or whitespace - should be populated externally as a parameter");
}
return(true);
}
public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener)
{
// CacheDirectory overrides LoadDirectory, so only set CacheDirectory if it is null (i.e. no alternative cache location has been configured in the destination component)
if (CacheDirectory == null)
{
if (value.Cache == null)
{
throw new Exception("For some reason the LoadDirectory does not have a Cache specified and the FilesystemDestination component does not have an override CacheDirectory specified");
}
CacheDirectory = value.Cache;
}
}
public override void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
{
base.Initialize(directory, dbInfo);
try
{
Setup();
}
catch (Exception)
{
//use integrated security if this fails
}
}
public bool Validate(ILoadDirectory destination)
{
if (destination.Cache == null)
{
throw new NullReferenceException("Destination " + destination.RootPath.FullName + " does not have a 'Cache' folder");
}
if (!destination.Cache.Exists)
{
throw new DirectoryNotFoundException(destination.Cache.FullName);
}
return(true);
}
private ScheduledDataLoadJob CreateTestJob(ILoadDirectory directory)
{
var catalogue = Mock.Of <ICatalogue>(c =>
c.GetTableInfoList(false) == new TableInfo[0] &&
c.GetLookupTableInfoList() == new TableInfo[0] &&
c.LoggingDataTask == "TestLogging"
);
var logManager = Mock.Of <ILogManager>();
var loadMetadata = Mock.Of <ILoadMetadata>(lm => lm.GetAllCatalogues() == new[] { catalogue });
var j = new ScheduledDataLoadJob(RepositoryLocator, "Test job", logManager, loadMetadata, directory, new ThrowImmediatelyDataLoadEventListener(), null);
j.LoadProgress = _lpMock;
return(j);
}
public void Create(CatalogueRepository repository, DiscoveredDatabase database,
ILoadDirectory directory)
{
TableInfo = new TableInfo(repository, "TestData")
{
Server = database.Server.Name,
Database = database.GetRuntimeName()
};
TableInfo.SaveToDatabase();
if (!string.IsNullOrWhiteSpace(database.Server.ExplicitUsernameIfAny))
{
Credentials = new DataAccessCredentialsFactory(repository).Create(TableInfo,
database.Server.ExplicitUsernameIfAny, database.Server.ExplicitPasswordIfAny,
DataAccessContext.Any);
}
ColumnInfo = new ColumnInfo(repository, "Col1", "int", TableInfo)
{
IsPrimaryKey = true
};
ColumnInfo.SaveToDatabase();
LoadMetadata = new LoadMetadata(repository, "HICLoadPipelineTests")
{
LocationOfFlatFiles = directory.RootPath.FullName
};
LoadMetadata.SaveToDatabase();
Catalogue = new Catalogue(repository, "HICLoadPipelineTests")
{
LoggingDataTask = "Test",
LoadMetadata_ID = LoadMetadata.ID
};
Catalogue.SaveToDatabase();
var catalogueItem = new CatalogueItem(repository, Catalogue, "Test");
catalogueItem.SetColumnInfo(ColumnInfo);
SetupLoadProcessTasks(repository);
}
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration)
{
_logManager = logManager;
RepositoryLocator = repositoryLocator;
LoadMetadata = loadMetadata;
LoadDirectory = directory;
Configuration = configuration;
_listener = listener;
Description = description;
List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList();
if (LoadMetadata != null)
{
_loggingTask = GetLoggingTask(catalogues);
}
RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
}
protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
{
if (file.StartsWith("."))
{
return(SkipReason.IsImaginaryFile);
}
//if there is a regex pattern
if (FilePattern != null)
{
if (!FilePattern.IsMatch(file)) //and it does not match
{
return(SkipReason.DidNotMatchPattern); //skip because it did not match pattern
}
}
//if the file on the FTP already exists in the forLoading directory, skip it
if (destination.ForLoading.GetFiles(file).Any())
{
return(SkipReason.InForLoading);
}
return(SkipReason.DoNotSkip);
}
public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
{
_directory = directory;
}
public override void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
{
}
public abstract void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo);
public StageArgs(LoadStage loadStage, DiscoveredDatabase database, ILoadDirectory projectDirectory)
{
LoadStage = loadStage;
DbInfo = database;
RootDir = projectDirectory;
}
public ScheduledDataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration)
: base(repositoryLocator, description, logManager, loadMetadata, directory, listener, configuration)
{
}
public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener)
{
project = value;
}
public override void Initialize(ILoadDirectory hicProjectDirectory, DiscoveredDatabase dbInfo)
{
_forLoading = hicProjectDirectory.ForLoading;
}
public virtual void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
{
LoadDirectory = directory;
_dbInfo = dbInfo;
}
public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
{
throw new NotImplementedException();
}
public bool Validate(ILoadDirectory destination)
{
SetupFTP();
return(GetFileList().Any());
}
protected virtual void Download(string file, ILoadDirectory destination, IDataLoadEventListener job)
{
Stopwatch s = new Stopwatch();
s.Start();
string uri;
if (!string.IsNullOrWhiteSpace(RemoteDirectory))
{
uri = "ftp://" + _host + "/" + RemoteDirectory + "/" + file;
}
else
{
uri = "ftp://" + _host + "/" + file;
}
if (_useSSL)
{
uri = "s" + uri;
}
Uri serverUri = new Uri(uri);
if (serverUri.Scheme != Uri.UriSchemeFtp)
{
return;
}
FtpWebRequest reqFTP;
reqFTP = (FtpWebRequest)FtpWebRequest.Create(new Uri(uri));
reqFTP.Credentials = new NetworkCredential(_username, _password);
reqFTP.KeepAlive = false;
reqFTP.Method = WebRequestMethods.Ftp.DownloadFile;
reqFTP.UseBinary = true;
reqFTP.Proxy = null;
reqFTP.UsePassive = true;
reqFTP.EnableSsl = _useSSL;
reqFTP.Timeout = TimeoutInSeconds * 1000;
FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse();
Stream responseStream = response.GetResponseStream();
string destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
using (FileStream writeStream = new FileStream(destinationFileName, FileMode.Create))
{
int Length = 2048;
Byte[] buffer = new Byte[Length];
int bytesRead = responseStream.Read(buffer, 0, Length);
int totalBytesReadSoFar = bytesRead;
while (bytesRead > 0)
{
writeStream.Write(buffer, 0, bytesRead);
bytesRead = responseStream.Read(buffer, 0, Length);
//notify whoever is listening of how far along the process we are
totalBytesReadSoFar += bytesRead;
job.OnProgress(this, new ProgressEventArgs(destinationFileName, new ProgressMeasurement(totalBytesReadSoFar / 1024, ProgressType.Kilobytes), s.Elapsed));
}
writeStream.Close();
}
response.Close();
_filesRetrieved.Add(serverUri.ToString());
s.Stop();
}
