public void StartProcessingBatchesIfNoFailures_WhenProcessingQueueContainsCompletedTasks_StartsNewBatch()
{
var commitItemBatch0 = new CatalogCommitItemBatch(new[] { _commitItem0 }, _packageIdentitya.Id);
var commitItemBatch1 = new CatalogCommitItemBatch(new[] { _commitItem1 }, _packageIdentityb.Id);
var completedTask = new CatalogCommitItemBatchTask(commitItemBatch0, Task.CompletedTask);
var unprocessedBatches = new List <CatalogCommitItemBatch>()
{
commitItemBatch1
};
var processingBatches = new List <CatalogCommitItemBatchTask>()
{
completedTask
};
CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
unprocessedBatches,
processingBatches,
_maxConcurrentBatches,
NoOpProcessBatchAsync,
CancellationToken.None);
Assert.Empty(unprocessedBatches);
Assert.Equal(2, processingBatches.Count);
}
public void StartProcessingBatchesIfNoFailures_WhenMaxConcurrencyLimitHit_DoesNotStartNewBatch()
{
var commitItemBatch0 = new CatalogCommitItemBatch(new[] { _commitItem0 }, _packageIdentitya.Id);
var commitItemBatch1 = new CatalogCommitItemBatch(new[] { _commitItem2 }, _packageIdentityc.Id);
using (var cancellationTokenSource = new CancellationTokenSource())
{
var inProcessTask = new CatalogCommitItemBatchTask(
commitItemBatch0,
Task.Delay(TimeSpan.FromMilliseconds(-1), cancellationTokenSource.Token));
var unprocessedBatches = new List <CatalogCommitItemBatch>()
{
commitItemBatch1
};
var processingBatches = new List <CatalogCommitItemBatchTask>()
{
inProcessTask
};
const int maxConcurrentBatches = 1;
CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
NoOpProcessBatchAsync,
CancellationToken.None);
Assert.Single(unprocessedBatches);
Assert.Single(processingBatches);
}
}
protected override Task <IEnumerable <CatalogCommitItemBatch> > CreateBatchesAsync(
IEnumerable <CatalogCommitItem> catalogItems)
{
// Grouping batches by commit is slow if it contains
// the same package registration id over and over again.
// This happens when, for example, a package publish "wave"
// occurs.
//
// If one package registration id is part of 20 batches,
// we'll have to process all registration leafs 20 times.
// It would be better to process these leafs only once.
//
// So let's batch by package registration id here,
// ensuring we never write a commit timestamp to the cursor
// that is higher than the last item currently processed.
//
// So, group by id, then make sure the batch key is the
// *lowest* timestamp of all commits in that batch.
// This ensures that on retries, we will retry
// from the correct location (even though we may have
// a little rework).
var batches = CatalogCommitUtilities.CreateCommitItemBatches(catalogItems, GetKey);
return(Task.FromResult(batches));
}
public void GetPackageIdKey_WhenItemIsNull_Throws()
{
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.GetPackageIdKey(item: null));
Assert.Equal("item", exception.ParamName);
}
public void StartProcessingBatchesIfNoFailures_WhenNoBatchIsCancelled_DoesNotStartNewBatch()
{
var commitItemBatch = new CatalogCommitItemBatch(
new[] { _commitItem0, _commitItem1 },
_packageIdentitya.Id);
var cancelledBatchTask = new CatalogCommitItemBatchTask(
commitItemBatch,
Task.FromCanceled(new CancellationToken(canceled: true)));
var unprocessedBatches = new List <CatalogCommitItemBatch>()
{
commitItemBatch
};
var processingBatches = new List <CatalogCommitItemBatchTask>()
{
cancelledBatchTask
};
CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
unprocessedBatches,
processingBatches,
_maxConcurrentBatches,
NoOpProcessBatchAsync,
CancellationToken.None);
Assert.Single(unprocessedBatches);
Assert.Single(processingBatches);
}
public void StartProcessingBatchesIfNoFailures_WhenAnyBatchIsFailed_DoesNotStartNewBatch()
{
var commitItemBatch = new CatalogCommitItemBatch(
new[] { _commitItem0, _commitItem1 },
_packageIdentitya.Id);
var failedBatchTask = new CatalogCommitItemBatchTask(commitItemBatch, FailedTask);
var unprocessedBatches = new List <CatalogCommitItemBatch>()
{
commitItemBatch
};
var processingBatches = new List <CatalogCommitItemBatchTask>()
{
failedBatchTask
};
CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
unprocessedBatches,
processingBatches,
_maxConcurrentBatches,
NoOpProcessBatchAsync,
CancellationToken.None);
Assert.Equal(1, unprocessedBatches.Count);
Assert.Equal(1, processingBatches.Count);
}
public void CreateCommitItemBatches_WhenGetCatalogCommitItemKeyIsNull_Throws()
{
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
Enumerable.Empty <CatalogCommitItem>(),
getCatalogCommitItemKey: null));
Assert.Equal("getCatalogCommitItemKey", exception.ParamName);
}
public void GetPackageIdKey_WhenPackageIdVariesInCase_ReturnsLowerCase(string packageId)
{
var commitItem = TestUtility.CreateCatalogCommitItem(
DateTime.UtcNow,
new PackageIdentity(packageId, new NuGetVersion("1.0.0")));
var key = CatalogCommitUtilities.GetPackageIdKey(commitItem);
Assert.Equal(packageId.ToLowerInvariant(), key);
}
protected override Task <IEnumerable <CatalogCommitItemBatch> > CreateBatchesAsync(
IEnumerable <CatalogCommitItem> catalogItems)
{
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
catalogItems,
CatalogCommitUtilities.GetPackageIdKey);
return(Task.FromResult(batches));
}
public void CreateCommitItemBatches_WhenCatalogItemsIsNull_Throws()
{
IEnumerable <CatalogCommitItem> catalogItems = null;
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
catalogItems,
CatalogCommitUtilities.GetPackageIdKey));
Assert.Equal("catalogItems", exception.ParamName);
}
public void StartProcessingBatchesIfNoFailures_WhenMaxConcurrentBatchesIsLessThanOne_Throws()
{
const int maxConcurrentBatches = 0;
var exception = Assert.Throws <ArgumentOutOfRangeException>(
() => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
new List <CatalogCommitItemBatch>(),
new List <CatalogCommitItemBatchTask>(),
maxConcurrentBatches,
NoOpProcessBatchAsync,
CancellationToken.None));
Assert.Equal("maxConcurrentBatches", exception.ParamName);
}
public void StartProcessingBatchesIfNoFailures_WhenProcessCommitItemBatchAsyncIsNull_Throws()
{
const ProcessCommitItemBatchAsync processCommitItemBatchAsync = null;
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
new List <CatalogCommitItemBatch>(),
new List <CatalogCommitItemBatchTask>(),
_maxConcurrentBatches,
processCommitItemBatchAsync,
CancellationToken.None));
Assert.Equal("processCommitItemBatchAsync", exception.ParamName);
}
protected override Task <bool> FetchAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
CancellationToken cancellationToken)
{
return(CatalogCommitUtilities.ProcessCatalogCommitsAsync(
client,
front,
back,
FetchCatalogCommitsAsync,
CreateBatchesAsync,
ProcessBatchAsync,
_maxConcurrentBatches,
_logger,
cancellationToken));
}
public void CreateCommitItemBatches_WhenPackageIdsVaryInCasing_GroupsUsingProvidedGetKeyFunction()
{
var now = DateTime.UtcNow;
var packageIdentityA0 = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0"));
var packageIdentityA1 = new PackageIdentity(id: "A", version: new NuGetVersion("2.0.0"));
var packageIdentityA2 = new PackageIdentity(id: "A", version: new NuGetVersion("3.0.0"));
var packageIdentityB0 = new PackageIdentity(id: "b", version: new NuGetVersion("1.0.0"));
var packageIdentityB1 = new PackageIdentity(id: "B", version: new NuGetVersion("2.0.0"));
var commitId0 = Guid.NewGuid().ToString();
var commitId1 = Guid.NewGuid().ToString();
var commitId2 = Guid.NewGuid().ToString();
var commitItem0 = TestUtility.CreateCatalogCommitItem(now, packageIdentityA0, commitId0);
var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityA1, commitId1);
var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), packageIdentityA2, commitId2);
var commitItem3 = TestUtility.CreateCatalogCommitItem(now, packageIdentityB0, commitId0);
var commitItem4 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityB1, commitId1);
// not in alphanumeric or chronological order
var commitItems = new[] { commitItem4, commitItem2, commitItem0, commitItem3, commitItem1 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitItem3.CommitTimeStamp, batch.CommitTimeStamp);
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem3)),
commit => Assert.True(ReferenceEquals(commit, commitItem4)));
},
batch =>
{
Assert.Equal(commitItem0.CommitTimeStamp, batch.CommitTimeStamp);
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem0)),
commit => Assert.True(ReferenceEquals(commit, commitItem1)),
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
});
}
public void CreateCommitItemBatches_WhenCommitItemsContainMultipleCommitsForSamePackageIdentity_ReturnsOnlyLastCommitForEachPackageIdentity()
{
var now = DateTime.UtcNow;
var commitItem0 = TestUtility.CreateCatalogCommitItem(now, _packageIdentitya);
var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), _packageIdentitya);
var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), _packageIdentitya);
var commitItems = new[] { commitItem0, commitItem1, commitItem2 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(commitItems, CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitItem2.CommitTimeStamp, batch.CommitTimeStamp);
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
});
}
public void StartProcessingBatchesIfNoFailures_WhenCanStartNewBatch_StartsNewBatch()
{
var commitItemBatch = new CatalogCommitItemBatch(new[] { _commitItem0 }, _packageIdentitya.Id);
var unprocessedBatches = new List <CatalogCommitItemBatch>()
{
commitItemBatch
};
var processingBatches = new List <CatalogCommitItemBatchTask>();
CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
unprocessedBatches,
processingBatches,
_maxConcurrentBatches,
NoOpProcessBatchAsync,
CancellationToken.None);
Assert.Empty(unprocessedBatches);
Assert.Single(processingBatches);
}
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitId_Throws()
{
var commitTimeStamp = DateTime.UtcNow;
var context = TestUtility.CreateCatalogContextJObject();
var commitItem0 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentitya));
var commitItem1 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentityb));
var commitItems = new[] { commitItem0, commitItem1 };
var exception = Assert.Throws <ArgumentException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey));
Assert.Equal("catalogItems", exception.ParamName);
Assert.StartsWith("Multiple commits exist with the same commit timestamp but different commit ID's: " +
$"{{ CommitId = {commitItem0.CommitId}, CommitTimeStamp = {commitItem0.CommitTimeStamp.ToString("O")} }}, " +
$"{{ CommitId = {commitItem1.CommitId}, CommitTimeStamp = {commitItem1.CommitTimeStamp.ToString("O")} }}.",
exception.Message);
}
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitIdAndLaterCommitExists_DoesNotThrow()
{
var commitTimeStamp0 = DateTime.UtcNow;
var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1);
var context = TestUtility.CreateCatalogContextJObject();
var commitItem0 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentitya));
var commitItem1 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentityb));
var commitItem2 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp1, _packageIdentitya));
var commitItems = new[] { commitItem0, commitItem1, commitItem2 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitTimeStamp1, batch.CommitTimeStamp.ToUniversalTime());
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
},
batch =>
{
Assert.Equal(commitTimeStamp0, batch.CommitTimeStamp.ToUniversalTime());
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem1)));
});
}
protected override string GetKey(CatalogCommitItem item)
{
return(CatalogCommitUtilities.GetPackageIdKey(item));
}
