private async Task <DeploySummary> UploadDirectoryInternalAsync(
[NotNull] DirectoryInfo sourceDirectory,
[NotNull] DirectoryInfo baseDirectory,
FtpPath basePath,
CancellationToken cancellationToken)
{
FtpPath dir = basePath.Append(new FtpPath(PathHelper.RelativePath(sourceDirectory, baseDirectory),
FileSystemType.Directory));
var summary = new DeploySummary();
bool directoryExists = await DirectoryExistsAsync(dir, cancellationToken);
if (!directoryExists)
{
await CreateDirectoryAsync(dir, cancellationToken);
summary.CreatedDirectories.Add(dir.Path);
}
DeploySummary uploadSummary =
await UploadFilesAsync(sourceDirectory, baseDirectory, basePath, cancellationToken);
summary.Add(uploadSummary);
return(summary);
}
public static string ToDisplayValue([NotNull] this DeploySummary summary)
{
if (summary is null)
{
throw new ArgumentNullException(nameof(summary));
}
return(JsonConvert.SerializeObject(summary, Formatting.Indented));
}
public async Task PublishFilesShouldSyncFiles()
{
var logger = Context.Logger;
var(source, deployTargetDirectory, temp) = TestDataHelper.CopyTestData(logger);
var ftpSettings = new FtpSettings(
new FtpPath("/", FileSystemType.Directory),
publicRootPath: new FtpPath("/", FileSystemType.Directory),
isSecure: false);
FtpHandler handler = await FtpHandler.Create(new Uri("ftp://127.0.0.1:30021"),
ftpSettings, new NetworkCredential("testuser", "testpw"), logger);
var sourceDirectory = new DirectoryInfo(source);
var ruleConfiguration = new RuleConfiguration {
AppOfflineEnabled = true
};
using var initialCancellationTokenSource =
new CancellationTokenSource(TimeSpan.FromSeconds(50));
DeploySummary initialSummary = await handler.PublishAsync(ruleConfiguration,
deployTargetDirectory,
initialCancellationTokenSource.Token);
logger.Information("Initial: {Initial}", initialSummary.ToDisplayValue());
using var cancellationTokenSource =
new CancellationTokenSource(TimeSpan.FromSeconds(50));
DeploySummary summary = await handler.PublishAsync(ruleConfiguration,
sourceDirectory,
cancellationTokenSource.Token);
logger.Information("Result: {Result}", summary.ToDisplayValue());
var fileSystemItems = await handler.ListDirectoryAsync(FtpPath.Root, cancellationTokenSource.Token);
foreach (FtpPath fileSystemItem in fileSystemItems)
{
logger.Information("{Item}", fileSystemItem.Path);
}
temp.Dispose();
}
private async Task <DeploySummary> PublishInternalAsync(
[NotNull] RuleConfiguration ruleConfiguration,
[NotNull] DirectoryInfo sourceDirectory,
CancellationToken cancellationToken)
{
var deploymentChangeSummary = new DeploySummary();
var stopwatch = Stopwatch.StartNew();
try
{
FtpPath basePath = _ftpSettings.BasePath ?? FtpPath.Root;
if (!await DirectoryExistsAsync(basePath, cancellationToken))
{
await CreateDirectoryAsync(basePath, cancellationToken);
}
_logger.Debug("Listing files in remote path '{Path}'", basePath.Path);
var fileSystemItems =
await ListDirectoryAsync(basePath, cancellationToken);
FtpPath[] sourceFiles = sourceDirectory
.GetFiles("*", SearchOption.AllDirectories)
.Select(s =>
basePath.Append(new FtpPath(PathHelper.RelativePath(s, sourceDirectory), FileSystemType.File)))
.ToArray();
FtpPath[] filesToKeep = fileSystemItems
.Where(s => KeepFile(s, ruleConfiguration))
.Where(s => s.Type == FileSystemType.File)
.ToArray();
var filesToRemove = fileSystemItems
.Except(sourceFiles)
.Except(filesToKeep)
.Where(s => s.Type == FileSystemType.File)
.ToImmutableArray();
IEnumerable <FtpPath> updated = fileSystemItems.Except(filesToRemove)
.Where(s => s.Type == FileSystemType.File);
if (ruleConfiguration.AppOfflineEnabled)
{
using var tempFile = TempFile.CreateTempFile("App_Offline", ".htm");
var appOfflinePath = new FtpPath($"/{tempFile.File.Name}", FileSystemType.File);
FtpPath appOfflineFullPath =
(_ftpSettings.PublicRootPath ?? _ftpSettings.BasePath ?? FtpPath.Root).Append(
appOfflinePath);
await UploadFileAsync(appOfflineFullPath, tempFile.File, cancellationToken);
_logger.Debug("Uploaded file '{App_Offline}'", appOfflineFullPath.Path);
}
DeploySummary deleteFiles = await DeleteFilesAsync(ruleConfiguration, filesToRemove, cancellationToken);
deploymentChangeSummary.Add(deleteFiles);
foreach (FtpPath ftpPath in filesToRemove)
{
deploymentChangeSummary.DeletedFiles.Add(ftpPath.Path);
}
foreach (FtpPath ftpPath in updated)
{
deploymentChangeSummary.UpdatedFiles.Add(ftpPath.Path);
}
DeploySummary uploadDirectoryAsync =
await UploadDirectoryAsync(ruleConfiguration,
sourceDirectory,
sourceDirectory,
basePath,
cancellationToken);
deploymentChangeSummary.Add(uploadDirectoryAsync);
if (ruleConfiguration.AppOfflineEnabled)
{
FtpPath[] appOfflineFiles = sourceFiles.Intersect(fileSystemItems)
.Where(file => file.Path is {} &&
Path.GetFileName(file.Path)
.Equals(DeploymentConstants.AppOfflineHtm,
StringComparison.OrdinalIgnoreCase))
.Select(file => file.Path)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Select(file => new FtpPath(file, FileSystemType.File))
.ToArray();
foreach (FtpPath appOfflineFile in appOfflineFiles)
{
bool fileExists = await FileExistsAsync(appOfflineFile, cancellationToken);
if (fileExists)
{
await DeleteFileAsync(appOfflineFile, cancellationToken);
_logger.Debug("Deleted {App_Offline}", appOfflineFile.Path);
}
}
}
private async Task <DeploySummary> UploadFilesAsync(
DirectoryInfo sourceDirectory,
DirectoryInfo baseDirectory,
FtpPath basePath,
CancellationToken cancellationToken)
{
var summary = new DeploySummary();
string[] localPaths = sourceDirectory
.GetFiles()
.Select(f => f.FullName)
.ToArray();
int totalCount = localPaths.Length;
if (totalCount == 0)
{
return(summary);
}
FtpPath relativeDir = basePath.Append(new FtpPath(PathHelper.RelativePath(sourceDirectory, baseDirectory),
FileSystemType.Directory));
_logger.Verbose("Uploading {Files} files", localPaths.Length);
int batchSize = _ftpSettings.BatchSize;
var totalTime = Stopwatch.StartNew();
try
{
int batches = (int)Math.Ceiling(totalCount / (double)batchSize);
int uploaded = 0;
for (int i = 0; i < batches; i++)
{
bool batchSuccessful = false;
int batchNumber = i + 1;
string[] files = localPaths.Skip(i * batchSize).Take(batchSize).ToArray();
var stopwatch = new Stopwatch();
for (int j = 0; j < _ftpSettings.MaxAttempts; j++)
{
try
{
stopwatch.Restart();
int uploadedFiles = await _ftpClient.UploadFilesAsync(files,
relativeDir.Path,
FtpRemoteExists.Overwrite,
true,
FtpVerify.Delete | FtpVerify.Retry,
token : cancellationToken);
if (uploadedFiles == files.Length)
{
batchSuccessful = true;
break;
}
_logger.Warning(
"The expected number of uploaded files was {Expected} but result was {Actual}, retrying batch {Batch}",
files.Length,
uploadedFiles,
batchNumber);
}
catch (Exception ex) when(!ex.IsFatal())
{
_logger.Warning(ex, "FTP ERROR in batch {Batch}", batchNumber);
}
finally
{
stopwatch.Stop();
}
}
if (!batchSuccessful)
{
string message = batches > 1
? $"The batch {batchNumber} failed"
: $"Failed to upload files {files}";
throw new InvalidOperationException(message);
}
uploaded += files.Length;
string elapsed = $"{stopwatch.Elapsed.TotalSeconds:F2}";
string percentage = $"{(100.0D * uploaded) / totalCount:F1}";
string paddedPercentage = new string(' ', 5 - percentage.Length) + percentage;
double averageTime = totalTime.Elapsed.TotalSeconds / uploaded;
string average = averageTime.ToString("F2", CultureInfo.InvariantCulture);
string timeLeft = $"{(totalCount - uploaded) * averageTime:F2}s";
string paddedBatch =
$"{new string(' ', batches.ToString(CultureInfo.InvariantCulture).Length - batchNumber.ToString(CultureInfo.InvariantCulture).Length)}{batchNumber}";
string paddedUploaded =
$"{new string(' ', totalCount.ToString(CultureInfo.InvariantCulture).Length - uploaded.ToString(CultureInfo.InvariantCulture).Length)}{uploaded}";
string totalElapsed = totalTime.Elapsed.TotalSeconds.ToString("F2", CultureInfo.InvariantCulture);
if (batches > 1)
{
_logger.Information(
"Uploaded batch {BatchNumber} of {BatchCount} using batch size {Size}, {Uploaded}/{Total} {Percentage}%, average {Average}s per file, time left: ~{TimeLeft}, took {ElapsedTime}s, total time {TotalTime}s",
paddedBatch,
batches,
batchSize,
paddedUploaded,
totalCount,
paddedPercentage,
average,
timeLeft,
elapsed,
totalElapsed);
}
else
{
_logger.Information("Uploaded files {Files}, took {TotalElapsed}s", files, totalElapsed);
}
}
}
finally
{
totalTime.Stop();
}
foreach (string path in localPaths)
{
summary.CreatedFiles.Add(PathHelper.RelativePath(new FileInfo(path), baseDirectory));
}
foreach (DirectoryInfo directoryInfo in sourceDirectory.GetDirectories())
{
DeploySummary subSummary =
await UploadFilesAsync(directoryInfo, baseDirectory, basePath, cancellationToken);
summary.Add(subSummary);
}
return(summary);
}
