private async Task <FtpSummary> DeleteFilesAsync(
RuleConfiguration ruleConfiguration,
ImmutableArray <FtpPath> fileSystemItems,
CancellationToken cancellationToken)
{
var deploymentChangeSummary = new FtpSummary();
foreach (var fileSystemItem in fileSystemItems
.Where(fileSystemItem => fileSystemItem.Type == FileSystemType.File))
{
if (ruleConfiguration.AppDataSkipDirectiveEnabled && fileSystemItem.IsAppDataDirectoryOrFile)
{
continue;
}
if (ruleConfiguration.Excludes.Any(value =>
fileSystemItem.Path.StartsWith(value, StringComparison.OrdinalIgnoreCase)))
{
deploymentChangeSummary.IgnoredFiles.Add(fileSystemItem.Path);
continue;
}
await DeleteFileAsync(fileSystemItem, cancellationToken);
deploymentChangeSummary.Deleted.Add(fileSystemItem.Path);
}
return(deploymentChangeSummary);
}
private async Task <FtpSummary> UploadDirectoryInternalAsync(
[NotNull] DirectoryInfo sourceDirectory,
[NotNull] DirectoryInfo baseDirectory,
FtpPath basePath,
CancellationToken cancellationToken)
{
var dir = basePath.Append(new FtpPath(PathHelper.RelativePath(sourceDirectory, baseDirectory),
FileSystemType.Directory));
var summary = new FtpSummary();
bool directoryExists = await DirectoryExistsAsync(dir, cancellationToken);
if (!directoryExists)
{
await CreateDirectoryAsync(dir, cancellationToken);
summary.CreatedDirectories.Add(dir.Path);
}
var uploadSummary = await UploadFilesAsync(sourceDirectory, baseDirectory, basePath, cancellationToken);
summary.Add(uploadSummary);
return(summary);
}
private async Task <IDeploymentChangeSummary> PublishInternalAsync(
[NotNull] RuleConfiguration ruleConfiguration,
[NotNull] DirectoryInfo sourceDirectory,
CancellationToken cancellationToken)
{
var deploymentChangeSummary = new FtpSummary();
var stopwatch = Stopwatch.StartNew();
try
{
var basePath = _ftpSettings.BasePath ?? FtpPath.Root;
if (!await DirectoryExistsAsync(basePath, cancellationToken))
{
await CreateDirectoryAsync(basePath, cancellationToken);
}
_logger.Debug("Listing files in remote path '{Path}'", basePath.Path);
var fileSystemItems =
await ListDirectoryAsync(basePath, cancellationToken);
var sourceFiles = sourceDirectory
.GetFiles("*", SearchOption.AllDirectories)
.Select(s =>
basePath.Append(new FtpPath(PathHelper.RelativePath(s, sourceDirectory), FileSystemType.File)))
.ToArray();
var filesToKeep = fileSystemItems
.Where(s => KeepFile(s, ruleConfiguration))
.Where(s => s.Type == FileSystemType.File)
.ToArray();
var filesToRemove = fileSystemItems
.Except(sourceFiles)
.Except(filesToKeep)
.Where(s => s.Type == FileSystemType.File)
.ToImmutableArray();
var updated = fileSystemItems.Except(filesToRemove)
.Where(s => s.Type == FileSystemType.File);
if (ruleConfiguration.AppOfflineEnabled)
{
using var tempFile = TempFile.CreateTempFile("App_Offline", ".htm");
var appOfflinePath = new FtpPath($"/{tempFile.File.Name}", FileSystemType.File);
var appOfflineFullPath =
(_ftpSettings.PublicRootPath ?? _ftpSettings.BasePath ?? FtpPath.Root).Append(
appOfflinePath);
await UploadFileAsync(appOfflineFullPath, tempFile.File, cancellationToken);
_logger.Debug("Uploaded file '{App_Offline}'", appOfflineFullPath.Path);
}
var deleteFiles = await DeleteFilesAsync(ruleConfiguration, filesToRemove, cancellationToken);
deploymentChangeSummary.Add(deleteFiles);
foreach (var ftpPath in filesToRemove)
{
deploymentChangeSummary.Deleted.Add(ftpPath.Path);
}
foreach (var ftpPath in updated)
{
deploymentChangeSummary.UpdatedFiles.Add(ftpPath.Path);
}
var uploadDirectoryAsync =
await UploadDirectoryAsync(ruleConfiguration,
sourceDirectory,
sourceDirectory,
basePath,
cancellationToken);
deploymentChangeSummary.Add(uploadDirectoryAsync);
if (ruleConfiguration.AppOfflineEnabled)
{
var appOfflineFiles = sourceFiles.Intersect(fileSystemItems)
.Where(file => file.Path != null &&
Path.GetFileName(file.Path)
.Equals(DeploymentConstants.AppOfflineHtm,
StringComparison.OrdinalIgnoreCase))
.Select(file => file.Path)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Select(file => new FtpPath(file, FileSystemType.File))
.ToArray();
foreach (var appOfflineFile in appOfflineFiles)
{
bool fileExists = await FileExistsAsync(appOfflineFile, cancellationToken);
if (fileExists)
{
await DeleteFileAsync(appOfflineFile, cancellationToken);
_logger.Debug("Deleted {App_Offline}", appOfflineFile.Path);
}
}
}
}
finally
{
stopwatch.Stop();
}
deploymentChangeSummary.TotalTime = stopwatch.Elapsed;
return(deploymentChangeSummary);
}
private async Task <FtpSummary> UploadFilesAsync(
DirectoryInfo sourceDirectory,
DirectoryInfo baseDirectory,
FtpPath basePath,
CancellationToken cancellationToken)
{
var summary = new FtpSummary();
string[] localPaths = sourceDirectory
.GetFiles()
.Select(f => f.FullName)
.ToArray();
int totalCount = localPaths.Length;
if (totalCount == 0)
{
return(summary);
}
var relativeDir = basePath.Append(new FtpPath(PathHelper.RelativePath(sourceDirectory, baseDirectory),
FileSystemType.Directory));
_logger.Verbose("Uploading {Files} files", localPaths.Length);
int batchSize = _ftpSettings.BatchSize;
var totalTime = Stopwatch.StartNew();
try
{
int batches = (int)Math.Ceiling(totalCount / (double)batchSize);
int uploaded = 0;
for (int i = 0; i < batches; i++)
{
bool batchSuccessful = false;
int batchNumber = i + 1;
string[] files = localPaths.Skip(i * batchSize).Take(batchSize).ToArray();
var stopwatch = new Stopwatch();
for (int j = 0; j < _ftpSettings.MaxAttempts; j++)
{
try
{
stopwatch.Restart();
int uploadedFiles = await _ftpClient.UploadFilesAsync(files,
relativeDir.Path,
FtpRemoteExists.Overwrite,
true,
FtpVerify.Delete | FtpVerify.Retry,
token : cancellationToken);
if (uploadedFiles == files.Length)
{
batchSuccessful = true;
break;
}
_logger.Warning(
"The expected number of uploaded files was {Expected} but result was {Actual}, retrying batch {Batch}",
files.Length,
uploadedFiles,
batchNumber);
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Warning(ex, "FTP ERROR in batch {Batch}", batchNumber);
}
finally
{
stopwatch.Stop();
}
}
if (!batchSuccessful)
{
string message = batches > 1
? $"The batch {batchNumber} failed"
: $"Failed to upload files {files}";
throw new InvalidOperationException(message);
}
uploaded += files.Length;
string elapsed = $"{stopwatch.Elapsed.TotalSeconds:F2}";
string percentage = $"{100 * uploaded / (double)totalCount:F1}";
string paddedPercentage = new string(' ', 5 - percentage.Length) + percentage;
double averageTime = totalTime.Elapsed.TotalSeconds / uploaded;
string average = averageTime.ToString("F2", CultureInfo.InvariantCulture);
string timeLeft = $"{(totalCount - uploaded) * averageTime:F2}s";
string paddedBatch =
$"{new string(' ', batches.ToString(CultureInfo.InvariantCulture).Length - batchNumber.ToString(CultureInfo.InvariantCulture).Length)}{batchNumber}";
string paddedUploaded =
$"{new string(' ', totalCount.ToString(CultureInfo.InvariantCulture).Length - uploaded.ToString(CultureInfo.InvariantCulture).Length)}{uploaded}";
string totalElapsed = totalTime.Elapsed.TotalSeconds.ToString("F2", CultureInfo.InvariantCulture);
if (batches > 1)
{
_logger.Information(
"Uploaded batch {BatchNumber} of {BatchCount} using batch size {Size}, {Uploaded}/{Total} {Percentage}%, average {Average}s per file, time left: ~{TimeLeft}, took {ElapsedTime}s, total time {TotalTime}s",
paddedBatch,
batches,
batchSize,
paddedUploaded,
totalCount,
paddedPercentage,
average,
timeLeft,
elapsed,
totalElapsed);
}
else
{
_logger.Information("Uploaded files {Files}, took {TotalElapsed}s", files, totalElapsed);
}
}
}
finally
{
totalTime.Stop();
}
foreach (string path in localPaths)
{
summary.CreatedFiles.Add(PathHelper.RelativePath(new FileInfo(path), baseDirectory));
}
foreach (var directoryInfo in sourceDirectory.GetDirectories())
{
var subSummary = await UploadFilesAsync(directoryInfo, baseDirectory, basePath, cancellationToken);
summary.Add(subSummary);
}
return(summary);
}
