protected override void Download(string file, ILoadDirectory destination, IDataLoadEventListener job) { if (file.Contains("/") || file.Contains("\\")) { throw new Exception("Was not expecting a relative path here"); } Stopwatch s = new Stopwatch(); s.Start(); using (var sftp = new SftpClient(_host, _username, _password)) { sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds); sftp.Connect(); //if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList()) string fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file; string destinationFilePath = Path.Combine(destination.ForLoading.FullName, file); //register for events Action <ulong> callback = (totalBytes) => job.OnProgress(this, new ProgressEventArgs(destinationFilePath, new ProgressMeasurement((int)(totalBytes * 0.001), ProgressType.Kilobytes), s.Elapsed)); using (var fs = new FileStream(destinationFilePath, FileMode.CreateNew)) { //download sftp.DownloadFile(fullFilePath, fs, callback); fs.Close(); } _filesRetrieved.Add(fullFilePath); } s.Stop(); }
private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener) { string[] files = GetFileList(); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, files.Aggregate("Identified the following files on the FTP server:", (s, f) => f + ",").TrimEnd(','))); bool forLoadingContainedCachedFiles = false; foreach (string file in files) { var action = GetSkipActionForFile(file, destination); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "File " + file + " was evaluated as " + action)); if (action == SkipReason.DoNotSkip) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to download " + file)); Download(file, destination, listener); } if (action == SkipReason.InForLoading) { forLoadingContainedCachedFiles = true; } } //if no files were downloaded (and there were none skiped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so if (!forLoadingContainedCachedFiles && !_filesRetrieved.Any() && SendLoadNotRequiredIfFileNotFound) { listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Could not find any files on the remote server worth downloading, so returning LoadNotRequired")); return(ExitCodeType.OperationNotRequired); } //otherwise it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound return(ExitCodeType.Success); }
private bool FilesInForLoadingMatchWorkload(ILoadDirectory directory) { var filesInForLoading = GetPathsRelativeToDirectory(directory.ForLoading.EnumerateFiles("*", SearchOption.AllDirectories).ToArray(), directory.ForLoading); var filesFromCache = GetPathsRelativeToDirectory(_workload.Values.ToArray(), directory.Cache); return(filesInForLoading.OrderBy(t => t).SequenceEqual(filesFromCache.OrderBy(t => t))); }
private void UnzipWithEvents(ZipArchiveEntry entry, ILoadDirectory destination, IDataLoadJob job) { //create a task string entryDestination = Path.Combine(destination.ForLoading.FullName, entry.Name); Task unzipJob = Task.Factory.StartNew(() => entry.ExtractToFile(entryDestination, true)); //create a stopwatch to time how long bits take Stopwatch s = new Stopwatch(); s.Start(); FileInfo f = new FileInfo(entryDestination); _entriesUnzipped.Add(f); //monitor it while (!unzipJob.IsCompleted) { Thread.Sleep(200); if (f.Exists) { job.OnProgress(this, new ProgressEventArgs(entryDestination, new ProgressMeasurement((int)(f.Length / 1000), ProgressType.Kilobytes), s.Elapsed)); } } s.Stop(); }
public bool Validate(ILoadDirectory destination) { if (UriToFile == null || string.IsNullOrWhiteSpace(UriToFile.PathAndQuery)) { throw new MissingFieldException("PathToFile is null or whitespace - should be populated externally as a parameter"); } return(true); }
public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener) { // CacheDirectory overrides LoadDirectory, so only set CacheDirectory if it is null (i.e. no alternative cache location has been configured in the destination component) if (CacheDirectory == null) { if (value.Cache == null) { throw new Exception("For some reason the LoadDirectory does not have a Cache specified and the FilesystemDestination component does not have an override CacheDirectory specified"); } CacheDirectory = value.Cache; } }
public override void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) { base.Initialize(directory, dbInfo); try { Setup(); } catch (Exception) { //use integrated security if this fails } }
public bool Validate(ILoadDirectory destination) { if (destination.Cache == null) { throw new NullReferenceException("Destination " + destination.RootPath.FullName + " does not have a 'Cache' folder"); } if (!destination.Cache.Exists) { throw new DirectoryNotFoundException(destination.Cache.FullName); } return(true); }
private ScheduledDataLoadJob CreateTestJob(ILoadDirectory directory) { var catalogue = Mock.Of <ICatalogue>(c => c.GetTableInfoList(false) == new TableInfo[0] && c.GetLookupTableInfoList() == new TableInfo[0] && c.LoggingDataTask == "TestLogging" ); var logManager = Mock.Of <ILogManager>(); var loadMetadata = Mock.Of <ILoadMetadata>(lm => lm.GetAllCatalogues() == new[] { catalogue }); var j = new ScheduledDataLoadJob(RepositoryLocator, "Test job", logManager, loadMetadata, directory, new ThrowImmediatelyDataLoadEventListener(), null); j.LoadProgress = _lpMock; return(j); }
public void Create(CatalogueRepository repository, DiscoveredDatabase database, ILoadDirectory directory) { TableInfo = new TableInfo(repository, "TestData") { Server = database.Server.Name, Database = database.GetRuntimeName() }; TableInfo.SaveToDatabase(); if (!string.IsNullOrWhiteSpace(database.Server.ExplicitUsernameIfAny)) { Credentials = new DataAccessCredentialsFactory(repository).Create(TableInfo, database.Server.ExplicitUsernameIfAny, database.Server.ExplicitPasswordIfAny, DataAccessContext.Any); } ColumnInfo = new ColumnInfo(repository, "Col1", "int", TableInfo) { IsPrimaryKey = true }; ColumnInfo.SaveToDatabase(); LoadMetadata = new LoadMetadata(repository, "HICLoadPipelineTests") { LocationOfFlatFiles = directory.RootPath.FullName }; LoadMetadata.SaveToDatabase(); Catalogue = new Catalogue(repository, "HICLoadPipelineTests") { LoggingDataTask = "Test", LoadMetadata_ID = LoadMetadata.ID }; Catalogue.SaveToDatabase(); var catalogueItem = new CatalogueItem(repository, Catalogue, "Test"); catalogueItem.SetColumnInfo(ColumnInfo); SetupLoadProcessTasks(repository); }
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration) { _logManager = logManager; RepositoryLocator = repositoryLocator; LoadMetadata = loadMetadata; LoadDirectory = directory; Configuration = configuration; _listener = listener; Description = description; List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList(); if (LoadMetadata != null) { _loggingTask = GetLoggingTask(catalogues); } RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList(); LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList(); }
protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination) { if (file.StartsWith(".")) { return(SkipReason.IsImaginaryFile); } //if there is a regex pattern if (FilePattern != null) { if (!FilePattern.IsMatch(file)) //and it does not match { return(SkipReason.DidNotMatchPattern); //skip because it did not match pattern } } //if the file on the FTP already exists in the forLoading directory, skip it if (destination.ForLoading.GetFiles(file).Any()) { return(SkipReason.InForLoading); } return(SkipReason.DoNotSkip); }
public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) { _directory = directory; }
public override void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) { }
public abstract void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo);
public StageArgs(LoadStage loadStage, DiscoveredDatabase database, ILoadDirectory projectDirectory) { LoadStage = loadStage; DbInfo = database; RootDir = projectDirectory; }
public ScheduledDataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration) : base(repositoryLocator, description, logManager, loadMetadata, directory, listener, configuration) { }
public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener) { project = value; }
public override void Initialize(ILoadDirectory hicProjectDirectory, DiscoveredDatabase dbInfo) { _forLoading = hicProjectDirectory.ForLoading; }
public virtual void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) { LoadDirectory = directory; _dbInfo = dbInfo; }
public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) { throw new NotImplementedException(); }
public bool Validate(ILoadDirectory destination) { SetupFTP(); return(GetFileList().Any()); }
protected virtual void Download(string file, ILoadDirectory destination, IDataLoadEventListener job) { Stopwatch s = new Stopwatch(); s.Start(); string uri; if (!string.IsNullOrWhiteSpace(RemoteDirectory)) { uri = "ftp://" + _host + "/" + RemoteDirectory + "/" + file; } else { uri = "ftp://" + _host + "/" + file; } if (_useSSL) { uri = "s" + uri; } Uri serverUri = new Uri(uri); if (serverUri.Scheme != Uri.UriSchemeFtp) { return; } FtpWebRequest reqFTP; reqFTP = (FtpWebRequest)FtpWebRequest.Create(new Uri(uri)); reqFTP.Credentials = new NetworkCredential(_username, _password); reqFTP.KeepAlive = false; reqFTP.Method = WebRequestMethods.Ftp.DownloadFile; reqFTP.UseBinary = true; reqFTP.Proxy = null; reqFTP.UsePassive = true; reqFTP.EnableSsl = _useSSL; reqFTP.Timeout = TimeoutInSeconds * 1000; FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse(); Stream responseStream = response.GetResponseStream(); string destinationFileName = Path.Combine(destination.ForLoading.FullName, file); using (FileStream writeStream = new FileStream(destinationFileName, FileMode.Create)) { int Length = 2048; Byte[] buffer = new Byte[Length]; int bytesRead = responseStream.Read(buffer, 0, Length); int totalBytesReadSoFar = bytesRead; while (bytesRead > 0) { writeStream.Write(buffer, 0, bytesRead); bytesRead = responseStream.Read(buffer, 0, Length); //notify whoever is listening of how far along the process we are totalBytesReadSoFar += bytesRead; job.OnProgress(this, new ProgressEventArgs(destinationFileName, new ProgressMeasurement(totalBytesReadSoFar / 1024, ProgressType.Kilobytes), s.Elapsed)); } writeStream.Close(); } response.Close(); _filesRetrieved.Add(serverUri.ToString()); s.Stop(); }