public async Task DeleteNotContainedFiles(LastRunConfig lastRun, List <FileInfo> files)
{
var root = IRootHelper.GetRoot();
var currentFiles =
files
.Select(x => root.GetRealtive(x))
.ToList();
var remoteFilesToRemove = lastRun.Files.Except(currentFiles);
var ftpClient = GetFtpClient();
foreach (var remoteFile in remoteFilesToRemove)
{
try
{
var remoteUrl = GetRemoteUrl(remoteFile);
await ftpClient.DeleteFileAsync(remoteUrl);
DidCleanup?.Invoke(this, remoteFile);
}
catch (FtpException)
{
Error?.Invoke(this, remoteFile);
continue;
}
}
}
private List <FileInfo> Pick(FilterConfig filter)
{
if (filter.Entries.Any(x => x.Contains("*")))
{
throw new ConfigException($"Filter for {nameof(Pick)} must not use wildcard [*].");
}
var root = IRootHelper.GetRoot();
var result = new List <FileInfo>();
foreach (var item in filter.Entries)
{
var isFile = !item.EndsWith("/");
var absolutePath = root.GetAbsoulte(item);
if (isFile)
{
var file = new FileInfo(absolutePath);
result.Add(file);
continue;
}
var directory = new DirectoryInfo(absolutePath);
result.AddRange(Pick(directory));
}
return(result);
}
private async Task WriteLastRunAsync(IFtpClient ftpClient, List <FileInfo> files)
{
var root = IRootHelper.GetRoot();
var lastRun = new LastRunConfig
{
Dt = DateTime.Now,
Files =
files
.Select(x => root.GetRealtive(x))
.ToList()
};
var json = lastRun.ToJsonWithTypeInformation();
await ftpClient.UploadStringAsync(_LastRunFileName, json);
}
private async Task <List <FileInfo> > UploadFilesAsync(TargetFtpConfig targetFtp, List <FileInfo> files)
{
var result = new List <FileInfo>();
var worker = new FtpWorker(targetFtp);
Container.Shared.Inject(worker);
worker.Enqueue(files.ToArray());
var root = IRootHelper.GetRoot();
worker.FileWillUpload += (sender, a) =>
{
lock (this)
{
IOutputConsole.ClearLine();
IOutputConsole.Write($"uploading {a.FilesLeft} {root.GetRealtive(a.File)}");
}
};
worker.FileDidUpload += (sender, a) =>
{
lock (result)
{
result.Add(a.File);
}
};
worker.FileError += (sender, a) =>
{
lock (this)
{
IOutputConsole.ClearLine();
IOutputConsole.WriteLine($"error: {root.GetRealtive(a.File)}, [{a.Message}]");
}
};
await worker.UploadAsync();
return(result);
}
private List <FileInfo> GetNewFiles(LastRunConfig lastRun, List <FileInfo> files)
{
var result = new List <FileInfo>();
var root = IRootHelper.GetRoot();
var lastFiles = lastRun.Files.ToHashSet();
foreach (var file in files)
{
var relative = root.GetRealtive(file);
if (lastFiles.Contains(relative))
{
continue;
}
result.Add(file);
}
return(result);
}
private List <FileInfo> CrawlDirectories(FilterConfig filter)
{
var root = IRootHelper.GetRoot();
return(CrawlDirectories(root, filter));
}
public override async Task Process(TargetCommand command)
{
foreach (var ftp in command.Target.Ftp)
{
var targetFtp = CreateTargetFtp(command, ftp);
using (var ftpClient = IFtpFactory.Client(targetFtp))
{
var lastRun = await ReadLastRunAsync(ftpClient);
var modifiedSince = lastRun?.Dt;
var localFiles = GetFiles(command.Target);
var filesToUpload = localFiles;
if (modifiedSince.HasValue)
{
filesToUpload =
localFiles
.Where(x => AppliesToModified(x, modifiedSince))
.ToList();
}
if (lastRun != null)
{
var remoteFilesHash =
lastRun
.Files
.ToHashSet();
var root = IRootHelper.GetRoot();
foreach (var file in localFiles)
{
var remotePath = root.GetRealtive(file);
if (remoteFilesHash.Contains(remotePath))
{
continue;
}
if (!filesToUpload.Any(x => x.FullName == file.FullName))
{
filesToUpload.Add(file);
}
}
}
IOutputConsole.WriteLine($"Target [{command.Value}/{targetFtp.Name}]");
if (!filesToUpload.HasContent())
{
PrintNoFiles();
}
else
{
IOutputConsole.WriteLine($"found {filesToUpload.Count} files");
await UploadFilesAsync(targetFtp, filesToUpload);
}
if (!command.Target.CleanupDisable)
{
await CleanupAsync(targetFtp, lastRun, localFiles);
}
await UploadLastRunAsync(ftpClient, localFiles);
PrintFinished();
if (command.Target.OK.HasContent())
{
await IOkProcessor.Process(new OkCommand { Url = command.Target.OK });
}
}
}
}
