public async Task DeleteNotContainedFiles(LastRunConfig lastRun, List <FileInfo> files) { var root = IRootHelper.GetRoot(); var currentFiles = files .Select(x => root.GetRealtive(x)) .ToList(); var remoteFilesToRemove = lastRun.Files.Except(currentFiles); var ftpClient = GetFtpClient(); foreach (var remoteFile in remoteFilesToRemove) { try { var remoteUrl = GetRemoteUrl(remoteFile); await ftpClient.DeleteFileAsync(remoteUrl); DidCleanup?.Invoke(this, remoteFile); } catch (FtpException) { Error?.Invoke(this, remoteFile); continue; } } }
private List <FileInfo> Pick(FilterConfig filter) { if (filter.Entries.Any(x => x.Contains("*"))) { throw new ConfigException($"Filter for {nameof(Pick)} must not use wildcard [*]."); } var root = IRootHelper.GetRoot(); var result = new List <FileInfo>(); foreach (var item in filter.Entries) { var isFile = !item.EndsWith("/"); var absolutePath = root.GetAbsoulte(item); if (isFile) { var file = new FileInfo(absolutePath); result.Add(file); continue; } var directory = new DirectoryInfo(absolutePath); result.AddRange(Pick(directory)); } return(result); }
private async Task WriteLastRunAsync(IFtpClient ftpClient, List <FileInfo> files) { var root = IRootHelper.GetRoot(); var lastRun = new LastRunConfig { Dt = DateTime.Now, Files = files .Select(x => root.GetRealtive(x)) .ToList() }; var json = lastRun.ToJsonWithTypeInformation(); await ftpClient.UploadStringAsync(_LastRunFileName, json); }
private async Task <List <FileInfo> > UploadFilesAsync(TargetFtpConfig targetFtp, List <FileInfo> files) { var result = new List <FileInfo>(); var worker = new FtpWorker(targetFtp); Container.Shared.Inject(worker); worker.Enqueue(files.ToArray()); var root = IRootHelper.GetRoot(); worker.FileWillUpload += (sender, a) => { lock (this) { IOutputConsole.ClearLine(); IOutputConsole.Write($"uploading {a.FilesLeft} {root.GetRealtive(a.File)}"); } }; worker.FileDidUpload += (sender, a) => { lock (result) { result.Add(a.File); } }; worker.FileError += (sender, a) => { lock (this) { IOutputConsole.ClearLine(); IOutputConsole.WriteLine($"error: {root.GetRealtive(a.File)}, [{a.Message}]"); } }; await worker.UploadAsync(); return(result); }
private List <FileInfo> GetNewFiles(LastRunConfig lastRun, List <FileInfo> files) { var result = new List <FileInfo>(); var root = IRootHelper.GetRoot(); var lastFiles = lastRun.Files.ToHashSet(); foreach (var file in files) { var relative = root.GetRealtive(file); if (lastFiles.Contains(relative)) { continue; } result.Add(file); } return(result); }
private List <FileInfo> CrawlDirectories(FilterConfig filter) { var root = IRootHelper.GetRoot(); return(CrawlDirectories(root, filter)); }
public override async Task Process(TargetCommand command) { foreach (var ftp in command.Target.Ftp) { var targetFtp = CreateTargetFtp(command, ftp); using (var ftpClient = IFtpFactory.Client(targetFtp)) { var lastRun = await ReadLastRunAsync(ftpClient); var modifiedSince = lastRun?.Dt; var localFiles = GetFiles(command.Target); var filesToUpload = localFiles; if (modifiedSince.HasValue) { filesToUpload = localFiles .Where(x => AppliesToModified(x, modifiedSince)) .ToList(); } if (lastRun != null) { var remoteFilesHash = lastRun .Files .ToHashSet(); var root = IRootHelper.GetRoot(); foreach (var file in localFiles) { var remotePath = root.GetRealtive(file); if (remoteFilesHash.Contains(remotePath)) { continue; } if (!filesToUpload.Any(x => x.FullName == file.FullName)) { filesToUpload.Add(file); } } } IOutputConsole.WriteLine($"Target [{command.Value}/{targetFtp.Name}]"); if (!filesToUpload.HasContent()) { PrintNoFiles(); } else { IOutputConsole.WriteLine($"found {filesToUpload.Count} files"); await UploadFilesAsync(targetFtp, filesToUpload); } if (!command.Target.CleanupDisable) { await CleanupAsync(targetFtp, lastRun, localFiles); } await UploadLastRunAsync(ftpClient, localFiles); PrintFinished(); if (command.Target.OK.HasContent()) { await IOkProcessor.Process(new OkCommand { Url = command.Target.OK }); } } } }