protected override Task <IEnumerable <CatalogCommitItemBatch> > CreateBatchesAsync( IEnumerable <CatalogCommitItem> catalogItems) { // Grouping batches by commit is slow if it contains // the same package registration id over and over again. // This happens when, for example, a package publish "wave" // occurs. // // If one package registration id is part of 20 batches, // we'll have to process all registration leafs 20 times. // It would be better to process these leafs only once. // // So let's batch by package registration id here, // ensuring we never write a commit timestamp to the cursor // that is higher than the last item currently processed. // // So, group by id, then make sure the batch key is the // *lowest* timestamp of all commits in that batch. // This ensures that on retries, we will retry // from the correct location (even though we may have // a little rework). var batches = CatalogCommitUtilities.CreateCommitItemBatches(catalogItems, GetKey); return(Task.FromResult(batches)); }
public void CreateCommitItemBatches_WhenGetCatalogCommitItemKeyIsNull_Throws() { var exception = Assert.Throws <ArgumentNullException>( () => CatalogCommitUtilities.CreateCommitItemBatches( Enumerable.Empty <CatalogCommitItem>(), getCatalogCommitItemKey: null)); Assert.Equal("getCatalogCommitItemKey", exception.ParamName); }
protected override Task <IEnumerable <CatalogCommitItemBatch> > CreateBatchesAsync( IEnumerable <CatalogCommitItem> catalogItems) { var batches = CatalogCommitUtilities.CreateCommitItemBatches( catalogItems, CatalogCommitUtilities.GetPackageIdKey); return(Task.FromResult(batches)); }
public void CreateCommitItemBatches_WhenCatalogItemsIsNull_Throws() { IEnumerable <CatalogCommitItem> catalogItems = null; var exception = Assert.Throws <ArgumentNullException>( () => CatalogCommitUtilities.CreateCommitItemBatches( catalogItems, CatalogCommitUtilities.GetPackageIdKey)); Assert.Equal("catalogItems", exception.ParamName); }
public void CreateCommitItemBatches_WhenPackageIdsVaryInCasing_GroupsUsingProvidedGetKeyFunction() { var now = DateTime.UtcNow; var packageIdentityA0 = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0")); var packageIdentityA1 = new PackageIdentity(id: "A", version: new NuGetVersion("2.0.0")); var packageIdentityA2 = new PackageIdentity(id: "A", version: new NuGetVersion("3.0.0")); var packageIdentityB0 = new PackageIdentity(id: "b", version: new NuGetVersion("1.0.0")); var packageIdentityB1 = new PackageIdentity(id: "B", version: new NuGetVersion("2.0.0")); var commitId0 = Guid.NewGuid().ToString(); var commitId1 = Guid.NewGuid().ToString(); var commitId2 = Guid.NewGuid().ToString(); var commitItem0 = TestUtility.CreateCatalogCommitItem(now, packageIdentityA0, commitId0); var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityA1, commitId1); var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), packageIdentityA2, commitId2); var commitItem3 = TestUtility.CreateCatalogCommitItem(now, packageIdentityB0, commitId0); var commitItem4 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityB1, commitId1); // not in alphanumeric or chronological order var commitItems = new[] { commitItem4, commitItem2, commitItem0, commitItem3, commitItem1 }; var batches = CatalogCommitUtilities.CreateCommitItemBatches( commitItems, CatalogCommitUtilities.GetPackageIdKey); Assert.Collection( batches, batch => { Assert.Equal(commitItem3.CommitTimeStamp, batch.CommitTimeStamp); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem3)), commit => Assert.True(ReferenceEquals(commit, commitItem4))); }, batch => { Assert.Equal(commitItem0.CommitTimeStamp, batch.CommitTimeStamp); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem0)), commit => Assert.True(ReferenceEquals(commit, commitItem1)), commit => Assert.True(ReferenceEquals(commit, commitItem2))); }); }
public void CreateCommitItemBatches_WhenCommitItemsContainMultipleCommitsForSamePackageIdentity_ReturnsOnlyLastCommitForEachPackageIdentity() { var now = DateTime.UtcNow; var commitItem0 = TestUtility.CreateCatalogCommitItem(now, _packageIdentitya); var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), _packageIdentitya); var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), _packageIdentitya); var commitItems = new[] { commitItem0, commitItem1, commitItem2 }; var batches = CatalogCommitUtilities.CreateCommitItemBatches(commitItems, CatalogCommitUtilities.GetPackageIdKey); Assert.Collection( batches, batch => { Assert.Equal(commitItem2.CommitTimeStamp, batch.CommitTimeStamp); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem2))); }); }
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitId_Throws() { var commitTimeStamp = DateTime.UtcNow; var context = TestUtility.CreateCatalogContextJObject(); var commitItem0 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentitya)); var commitItem1 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentityb)); var commitItems = new[] { commitItem0, commitItem1 }; var exception = Assert.Throws <ArgumentException>( () => CatalogCommitUtilities.CreateCommitItemBatches( commitItems, CatalogCommitUtilities.GetPackageIdKey)); Assert.Equal("catalogItems", exception.ParamName); Assert.StartsWith("Multiple commits exist with the same commit timestamp but different commit ID's: " + $"{{ CommitId = {commitItem0.CommitId}, CommitTimeStamp = {commitItem0.CommitTimeStamp.ToString("O")} }}, " + $"{{ CommitId = {commitItem1.CommitId}, CommitTimeStamp = {commitItem1.CommitTimeStamp.ToString("O")} }}.", exception.Message); }
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitIdAndLaterCommitExists_DoesNotThrow() { var commitTimeStamp0 = DateTime.UtcNow; var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1); var context = TestUtility.CreateCatalogContextJObject(); var commitItem0 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentitya)); var commitItem1 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentityb)); var commitItem2 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp1, _packageIdentitya)); var commitItems = new[] { commitItem0, commitItem1, commitItem2 }; var batches = CatalogCommitUtilities.CreateCommitItemBatches( commitItems, CatalogCommitUtilities.GetPackageIdKey); Assert.Collection( batches, batch => { Assert.Equal(commitTimeStamp1, batch.CommitTimeStamp.ToUniversalTime()); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem2))); }, batch => { Assert.Equal(commitTimeStamp0, batch.CommitTimeStamp.ToUniversalTime()); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem1))); }); }