private async Task UploadAndAssociateAsync(AssociationsStatus associateStatus, FileBlobDescriptor[] blobsForUpload)
{
int numMissing = associateStatus.Missing.Count();
m_logger.Info("Uploading {0} missing files.", numMissing);
#if DEBUG
// check that missing hashes reported in associateStatus match those provided in blobsForUpload
var providedHashes = new HashSet <BlobIdentifier>(blobsForUpload.Select(fb => fb.BlobIdentifier));
var notFoundMissingHashes = associateStatus.Missing.Where(b => !providedHashes.Contains(b)).ToList();
if (notFoundMissingHashes.Any())
{
throw new DaemonException("This many hashes not found in blobs to upload: " + notFoundMissingHashes.Count());
}
#endif
var startTime = DateTime.UtcNow;
await m_dropClient.UploadAndAssociateAsync(
DropName,
blobsForUpload.ToList(),
abortIfAlreadyExists : false,
firstAssociationStatus : associateStatus,
cancellationToken : Token).ConfigureAwait(false);
Interlocked.Add(ref Stats.TotalUploadTimeMs, ElapsedMillis(startTime));
Interlocked.Add(ref Stats.NumFilesUploaded, numMissing);
}
/// <inheritdoc />
public Task UploadAndAssociateAsync(string dropName, List<FileBlobDescriptor> preComputedBlobIds, bool abortIfAlreadyExists, AssociationsStatus firstAssociationStatus, CancellationToken cancellationToken)
{
return RetryAsync(
nameof(IDropServiceClient.UploadAndAssociateAsync),
(client, ct) => client.UploadAndAssociateAsync(dropName, preComputedBlobIds, abortIfAlreadyExists, firstAssociationStatus, ct),
cancellationToken);
}
Task IDropServiceClient.UploadAndAssociateAsync(string dropName, List <FileBlobDescriptor> preComputedBlobIds, bool abortIfAlreadyExists, AssociationsStatus firstAssociationStatus, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
/// <summary>
/// Sets completion results for <see cref="AddFileItem"/>s that were successfully associated
/// (as indicated by <paramref name="associateStatus"/>), and returns <see cref="AddFileItem"/>s
/// for those that are missing (<see cref="AssociationsStatus.Missing"/>).
/// </summary>
private static async Task <IReadOnlyList <AddFileItem> > SetResultForAssociatedNonMissingItemsAsync(AddFileItem[] batch, AssociationsStatus associateStatus, bool chunkDedup, CancellationToken cancellationToken)
{
var missingItems = new List <AddFileItem>();
var missingBlobIds = new HashSet <BlobIdentifier>(associateStatus.Missing);
foreach (AddFileItem item in batch)
{
var itemFileBlobDescriptor = await item.FileBlobDescriptorForAssociateAsync(chunkDedup, cancellationToken);
if (!missingBlobIds.Contains(itemFileBlobDescriptor.BlobIdentifier))
{
item.TaskSource.SetResult(AddFileResult.Associated);
}
else
{
missingItems.Add(item);
}
}
return(missingItems);
}
/// <summary>
/// Implements 'drop addfile' by first calling <see cref="IDropServiceClient.AssociateAsync"/>,
/// and then <see cref="IDropServiceClient.UploadAndAssociateAsync"/>.
/// </summary>
/// <remarks>
/// This method is called concurrently.
/// </remarks>
private async Task ProcessAddFilesAsync(AddFileItem[] batch)
{
Interlocked.Exchange(ref m_lastTimeProcessAddFileRanInTicks, DateTime.UtcNow.Ticks);
int batchLength = batch.Length;
if (batchLength == 0)
{
return;
}
Interlocked.Increment(ref Stats.NumBatches);
if (batchLength == m_config.BatchSize)
{
Interlocked.Increment(ref Stats.NumCompleteBatches);
}
else
{
Interlocked.Increment(ref Stats.NumIncompleteBatches);
}
m_logger.Info("Processing a batch of {0} drop files.", batchLength);
try
{
// compute blobs for associate
var startTime = DateTime.UtcNow;
FileBlobDescriptor[] blobsForAssociate = await Task.WhenAll(batch.Select(item => item.FileBlobDescriptorForAssociateAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForAssociateMs, ElapsedMillis(startTime));
// run 'Associate' on all items from the batch; the result will indicate which files were associated and which need to be uploaded
AssociationsStatus associateStatus = await AssociateAsync(blobsForAssociate);
IReadOnlyList <AddFileItem> itemsLeftToUpload = await SetResultForAssociatedNonMissingItemsAsync(batch, associateStatus, m_config.EnableChunkDedup, Token);
// compute blobs for upload
startTime = DateTime.UtcNow;
FileBlobDescriptor[] blobsForUpload = await Task.WhenAll(itemsLeftToUpload.Select(item => item.FileBlobDescriptorForUploadAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForUploadMs, ElapsedMillis(startTime));
// run 'UploadAndAssociate' for the missing files.
await UploadAndAssociateAsync(associateStatus, blobsForUpload);
SetResultForUploadedMissingItems(itemsLeftToUpload);
Interlocked.Add(ref Stats.TotalUploadSizeMb, blobsForUpload.Sum(b => b.FileSize ?? 0) >> 20);
m_logger.Info("Done processing AddFile batch.");
}
catch (Exception e)
{
foreach (AddFileItem item in batch)
{
if (!item.TaskSource.Task.IsCompleted)
{
item.TaskSource.SetException(e);
}
}
}
}
/// <summary>
/// Implements 'drop addfile' by first calling <see cref="IDropServiceClient.AssociateAsync"/>,
/// and then <see cref="IDropServiceClient.UploadAndAssociateAsync"/>.
/// </summary>
/// <remarks>
/// This method is called concurrently.
/// </remarks>
private async Task ProcessAddFilesAsync(AddFileItem[] batch)
{
Interlocked.Exchange(ref m_lastTimeProcessAddFileRanInTicks, DateTime.UtcNow.Ticks);
if (batch.Length == 0)
{
return;
}
Interlocked.Increment(ref Stats.NumBatches);
if (batch.Length == m_config.BatchSize)
{
Interlocked.Increment(ref Stats.NumCompleteBatches);
}
else
{
Interlocked.Increment(ref Stats.NumIncompleteBatches);
}
FileBlobDescriptor[] blobsForAssociate = new FileBlobDescriptor[0];
try
{
var dedupedBatch = SkipFilesWithTheSameDropPathAndContent(batch);
var numSkipped = batch.Length - dedupedBatch.Length;
m_logger.Info("Processing a batch of {0} drop files after skipping {1} files.", dedupedBatch.Length, numSkipped);
Task <HashSet <string> > registerFilesForBuildManifestTask = null;
// Register files for Build Manifest
if (m_dropDaemon.DropConfig.EnableBuildManifestCreation)
{
BuildManifestEntry[] buildManifestEntries = dedupedBatch
.Where(dropItem => dropItem.BlobIdentifier != null)
.Select(dropItem => new BuildManifestEntry(dropItem.RelativeDropFilePath, dropItem.BlobIdentifier.ToContentHash(), dropItem.FullFilePath))
.ToArray();
registerFilesForBuildManifestTask = Task.Run(() => RegisterFilesForBuildManifestAsync(buildManifestEntries));
}
// compute blobs for associate
var startTime = DateTime.UtcNow;
blobsForAssociate = await Task.WhenAll(dedupedBatch.Select(item => item.FileBlobDescriptorForAssociateAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForAssociateMs, ElapsedMillis(startTime));
// run 'Associate' on all items from the batch; the result will indicate which files were associated and which need to be uploaded
AssociationsStatus associateStatus = await AssociateAsync(blobsForAssociate);
IReadOnlyList <AddFileItem> itemsLeftToUpload = await SetResultForAssociatedNonMissingItemsAsync(dedupedBatch, associateStatus, m_config.EnableChunkDedup, Token);
// compute blobs for upload
startTime = DateTime.UtcNow;
FileBlobDescriptor[] blobsForUpload = await Task.WhenAll(itemsLeftToUpload.Select(item => item.FileBlobDescriptorForUploadAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForUploadMs, ElapsedMillis(startTime));
// run 'UploadAndAssociate' for the missing files.
await UploadAndAssociateAsync(associateStatus, blobsForUpload);
SetResultForUploadedMissingItems(itemsLeftToUpload);
Interlocked.Add(ref Stats.TotalUploadSizeMb, blobsForUpload.Sum(b => b.FileSize ?? 0) >> 20);
foreach (var file in dedupedBatch)
{
RegisterFileForBuildManifestResult result = registerFilesForBuildManifestTask == null
? RegisterFileForBuildManifestResult.Skipped
: (await registerFilesForBuildManifestTask).Contains(file.FullFilePath)
? RegisterFileForBuildManifestResult.Failed
: RegisterFileForBuildManifestResult.Registered;
file.BuildManifestTaskSource.TrySetResult(result);
}
m_logger.Info("Done processing AddFile batch.");
}
catch (Exception e)
{
m_logger.Verbose($"Failed ProcessAddFilesAsync (batch size:{batch.Length}, blobsForAssociate size:{blobsForAssociate.Length}){Environment.NewLine}"
+ string.Join(
Environment.NewLine,
batch.Select(item => $"'{item.FullFilePath}', '{item.RelativeDropFilePath}', BlobId:'{item.BlobIdentifier?.ToString() ?? ""}', Task.IsCompleted:{item.DropResultTaskSource.Task.IsCompleted}")));
foreach (AddFileItem item in batch)
{
if (!item.DropResultTaskSource.Task.IsCompleted)
{
item.DropResultTaskSource.SetException(e);
}
// No exceptions are thrown by RegisterFilesForBuildManifestAsync
item.BuildManifestTaskSource.TrySetResult(RegisterFileForBuildManifestResult.Skipped);
}
}
}
/// <summary>
/// Implements 'drop addfile' by first calling <see cref="IDropServiceClient.AssociateAsync"/>,
/// and then <see cref="IDropServiceClient.UploadAndAssociateAsync"/>.
/// </summary>
/// <remarks>
/// This method is called concurrently.
/// </remarks>
private async Task ProcessAddFilesAsync(AddFileItem[] batch)
{
Interlocked.Exchange(ref m_lastTimeProcessAddFileRanInTicks, DateTime.UtcNow.Ticks);
int batchLength = batch.Length;
if (batchLength == 0)
{
return;
}
Interlocked.Increment(ref Stats.NumBatches);
if (batchLength == m_config.BatchSize)
{
Interlocked.Increment(ref Stats.NumCompleteBatches);
}
else
{
Interlocked.Increment(ref Stats.NumIncompleteBatches);
}
m_logger.Info("Processing a batch of {0} drop files.", batchLength);
FileBlobDescriptor[] blobsForAssociate = new FileBlobDescriptor[0];
try
{
var dedupedBatch = SkipFilesWithTheSameDropPathAndContent(batch);
// compute blobs for associate
var startTime = DateTime.UtcNow;
blobsForAssociate = await Task.WhenAll(dedupedBatch.Select(item => item.FileBlobDescriptorForAssociateAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForAssociateMs, ElapsedMillis(startTime));
// run 'Associate' on all items from the batch; the result will indicate which files were associated and which need to be uploaded
AssociationsStatus associateStatus = await AssociateAsync(blobsForAssociate);
IReadOnlyList <AddFileItem> itemsLeftToUpload = await SetResultForAssociatedNonMissingItemsAsync(dedupedBatch, associateStatus, m_config.EnableChunkDedup, Token);
// compute blobs for upload
startTime = DateTime.UtcNow;
FileBlobDescriptor[] blobsForUpload = await Task.WhenAll(itemsLeftToUpload.Select(item => item.FileBlobDescriptorForUploadAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForUploadMs, ElapsedMillis(startTime));
// run 'UploadAndAssociate' for the missing files.
await UploadAndAssociateAsync(associateStatus, blobsForUpload);
SetResultForUploadedMissingItems(itemsLeftToUpload);
Interlocked.Add(ref Stats.TotalUploadSizeMb, blobsForUpload.Sum(b => b.FileSize ?? 0) >> 20);
m_logger.Info("Done processing AddFile batch.");
}
catch (Exception e)
{
m_logger.Verbose($"Failed ProcessAddFilesAsync (batch size:{batch.Length}, blobsForAssociate size:{blobsForAssociate.Length}){Environment.NewLine}"
+ string.Join(
Environment.NewLine,
batch.Select(item => $"'{item.FullFilePath}', '{item.RelativeDropFilePath}', BlobId:'{item.BlobIdentifier?.ToString() ?? ""}', Task.IsCompleted:{item.TaskSource.Task.IsCompleted}")));
foreach (AddFileItem item in batch)
{
if (!item.TaskSource.Task.IsCompleted)
{
item.TaskSource.SetException(e);
}
}
}
}
/// <summary>
/// Sets completion results for <see cref="AddFileItem"/>s that were successfully associated
/// (as indicated by <paramref name="associateStatus"/>), and returns <see cref="AddFileItem"/>s
/// for those that are missing (<see cref="AssociationsStatus.Missing"/>).
/// </summary>
private async Task <IReadOnlyList <AddFileItem> > SetResultForAssociatedNonMissingItemsAsync(AddFileItem[] batch, AssociationsStatus associateStatus, bool chunkDedup, CancellationToken cancellationToken)
{
var missingItems = new List <AddFileItem>();
var missingBlobIds = new HashSet <BlobIdentifier>(associateStatus.Missing);
long totalSizeOfAssociatedFiles = 0;
foreach (AddFileItem item in batch)
{
var itemFileBlobDescriptor = await item.FileBlobDescriptorForAssociateAsync(chunkDedup, cancellationToken);
if (!missingBlobIds.Contains(itemFileBlobDescriptor.BlobIdentifier))
{
item.DropResultTaskSource.SetResult(AddFileResult.Associated);
totalSizeOfAssociatedFiles += item.PrecomputedFileLength;
}
else
{
missingItems.Add(item);
}
}
Interlocked.Add(ref Stats.TotalAssociateSizeBytes, totalSizeOfAssociatedFiles);
return(missingItems);
}
