protected override Task <IEnumerable <CatalogCommitItemBatch> > CreateBatchesAsync(
IEnumerable <CatalogCommitItem> catalogItems)
{
// Grouping batches by commit is slow if it contains
// the same package registration id over and over again.
// This happens when, for example, a package publish "wave"
// occurs.
//
// If one package registration id is part of 20 batches,
// we'll have to process all registration leafs 20 times.
// It would be better to process these leafs only once.
//
// So let's batch by package registration id here,
// ensuring we never write a commit timestamp to the cursor
// that is higher than the last item currently processed.
//
// So, group by id, then make sure the batch key is the
// *lowest* timestamp of all commits in that batch.
// This ensures that on retries, we will retry
// from the correct location (even though we may have
// a little rework).
var batches = CatalogCommitUtilities.CreateCommitItemBatches(catalogItems, GetKey);
return(Task.FromResult(batches));
}
public void CreateCommitItemBatches_WhenGetCatalogCommitItemKeyIsNull_Throws()
{
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
Enumerable.Empty <CatalogCommitItem>(),
getCatalogCommitItemKey: null));
Assert.Equal("getCatalogCommitItemKey", exception.ParamName);
}
protected override Task <IEnumerable <CatalogCommitItemBatch> > CreateBatchesAsync(
IEnumerable <CatalogCommitItem> catalogItems)
{
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
catalogItems,
CatalogCommitUtilities.GetPackageIdKey);
return(Task.FromResult(batches));
}
public void CreateCommitItemBatches_WhenCatalogItemsIsNull_Throws()
{
IEnumerable <CatalogCommitItem> catalogItems = null;
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
catalogItems,
CatalogCommitUtilities.GetPackageIdKey));
Assert.Equal("catalogItems", exception.ParamName);
}
public void CreateCommitItemBatches_WhenPackageIdsVaryInCasing_GroupsUsingProvidedGetKeyFunction()
{
var now = DateTime.UtcNow;
var packageIdentityA0 = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0"));
var packageIdentityA1 = new PackageIdentity(id: "A", version: new NuGetVersion("2.0.0"));
var packageIdentityA2 = new PackageIdentity(id: "A", version: new NuGetVersion("3.0.0"));
var packageIdentityB0 = new PackageIdentity(id: "b", version: new NuGetVersion("1.0.0"));
var packageIdentityB1 = new PackageIdentity(id: "B", version: new NuGetVersion("2.0.0"));
var commitId0 = Guid.NewGuid().ToString();
var commitId1 = Guid.NewGuid().ToString();
var commitId2 = Guid.NewGuid().ToString();
var commitItem0 = TestUtility.CreateCatalogCommitItem(now, packageIdentityA0, commitId0);
var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityA1, commitId1);
var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), packageIdentityA2, commitId2);
var commitItem3 = TestUtility.CreateCatalogCommitItem(now, packageIdentityB0, commitId0);
var commitItem4 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityB1, commitId1);
// not in alphanumeric or chronological order
var commitItems = new[] { commitItem4, commitItem2, commitItem0, commitItem3, commitItem1 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitItem3.CommitTimeStamp, batch.CommitTimeStamp);
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem3)),
commit => Assert.True(ReferenceEquals(commit, commitItem4)));
},
batch =>
{
Assert.Equal(commitItem0.CommitTimeStamp, batch.CommitTimeStamp);
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem0)),
commit => Assert.True(ReferenceEquals(commit, commitItem1)),
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
});
}
public void CreateCommitItemBatches_WhenCommitItemsContainMultipleCommitsForSamePackageIdentity_ReturnsOnlyLastCommitForEachPackageIdentity()
{
var now = DateTime.UtcNow;
var commitItem0 = TestUtility.CreateCatalogCommitItem(now, _packageIdentitya);
var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), _packageIdentitya);
var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), _packageIdentitya);
var commitItems = new[] { commitItem0, commitItem1, commitItem2 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(commitItems, CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitItem2.CommitTimeStamp, batch.CommitTimeStamp);
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
});
}
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitId_Throws()
{
var commitTimeStamp = DateTime.UtcNow;
var context = TestUtility.CreateCatalogContextJObject();
var commitItem0 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentitya));
var commitItem1 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentityb));
var commitItems = new[] { commitItem0, commitItem1 };
var exception = Assert.Throws <ArgumentException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey));
Assert.Equal("catalogItems", exception.ParamName);
Assert.StartsWith("Multiple commits exist with the same commit timestamp but different commit ID's: " +
$"{{ CommitId = {commitItem0.CommitId}, CommitTimeStamp = {commitItem0.CommitTimeStamp.ToString("O")} }}, " +
$"{{ CommitId = {commitItem1.CommitId}, CommitTimeStamp = {commitItem1.CommitTimeStamp.ToString("O")} }}.",
exception.Message);
}
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitIdAndLaterCommitExists_DoesNotThrow()
{
var commitTimeStamp0 = DateTime.UtcNow;
var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1);
var context = TestUtility.CreateCatalogContextJObject();
var commitItem0 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentitya));
var commitItem1 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentityb));
var commitItem2 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp1, _packageIdentitya));
var commitItems = new[] { commitItem0, commitItem1, commitItem2 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitTimeStamp1, batch.CommitTimeStamp.ToUniversalTime());
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
},
batch =>
{
Assert.Equal(commitTimeStamp0, batch.CommitTimeStamp.ToUniversalTime());
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem1)));
});
}