public async Task GetCatalogPropertiesAsync_ThrowsIfCancelled()
{
await Assert.ThrowsAsync <OperationCanceledException>(
() => FeedHelpers.GetCatalogPropertiesAsync(
Mock.Of <IStorage>(),
Mock.Of <ITelemetryService>(),
new CancellationToken(canceled: true)));
}
private Task <SortedList <DateTime, IList <FeedPackageDetails> > > TestGetPackagesInOrder(
IEnumerable <ODataPackage> oDataPackages,
Func <FeedPackageDetails, DateTime> keyDateFunc)
{
return(FeedHelpers.GetPackagesInOrder(
new HttpClient(GetMessageHandlerForPackages(oDataPackages)),
new Uri(_baseUri + "/test"),
keyDateFunc));
}
public async Task GetCatalogPropertiesAsync_ThrowsForNullTelemetryService()
{
var exception = await Assert.ThrowsAsync <ArgumentNullException>(
() => FeedHelpers.GetCatalogPropertiesAsync(
storage: Mock.Of <IStorage>(),
telemetryService: null,
cancellationToken: CancellationToken.None));
Assert.Equal("telemetryService", exception.ParamName);
}
private static async Task VerifyPropertyAsync(string json, Action <CatalogProperties> propertyVerifier)
{
var storage = CreateStorageMock(json);
var catalogProperties = await FeedHelpers.GetCatalogPropertiesAsync(
storage.Object,
Mock.Of <ITelemetryService>(),
CancellationToken.None);
Assert.NotNull(catalogProperties);
propertyVerifier(catalogProperties);
storage.Verify();
}
public async Task <ActionResult> GetFeed(int id)
{
var feedDto = await _dbContext.RssFeeds
.Include(f => f.MediaFiles)
.FirstOrDefaultAsync(f => f.Id == id);
var feed = FeedHelpers.GetFeed(feedDto, id => Url.Action("Get", "Files", new { id }));
var stream = new MemoryStream();
FeedHelpers.Serialize(feed, stream);
return(File(stream.ToArray(), "application/rss+xml; charset=utf-8"));
}
public async Task GetCatalogPropertiesAsync_ReturnsNullPropertiesIfPropertiesMissing()
{
var storage = CreateStorageMock(json: "{}");
var catalogProperties = await FeedHelpers.GetCatalogPropertiesAsync(
storage.Object,
Mock.Of <ITelemetryService>(),
CancellationToken.None);
Assert.NotNull(catalogProperties);
Assert.Null(catalogProperties.LastCreated);
Assert.Null(catalogProperties.LastDeleted);
Assert.Null(catalogProperties.LastEdited);
storage.Verify();
}
public async Task DownloadMetadata2CatalogAsync_WhenCancellationTokenIsCancelled_Throws()
{
await Assert.ThrowsAsync <OperationCanceledException>(
() => FeedHelpers.DownloadMetadata2CatalogAsync(
Mock.Of <IPackageCatalogItemCreator>(),
new SortedList <DateTime, IList <FeedPackageDetails> >(),
Mock.Of <IStorage>(),
DateTime.MinValue,
DateTime.MinValue,
DateTime.MinValue,
maxDegreeOfParallelism: 1,
createdPackages: null,
updateCreatedFromEdited: false,
cancellationToken: new CancellationToken(canceled: true),
telemetryService: Mock.Of <ITelemetryService>(),
logger: Mock.Of <ILogger>()));
}
internal Task <DateTime> DownloadMetadata2CatalogAsync()
{
const int maxDegreeOfParallelism = 1;
return(FeedHelpers.DownloadMetadata2CatalogAsync(
PackageCatalogItemCreator.Object,
Packages,
Storage.Object,
LastCreated,
LastEdited,
LastDeleted,
maxDegreeOfParallelism,
CreatedPackages,
UpdateCreatedFromEdited,
CancellationToken.None,
TelemetryService.Object,
Logger.Object));
}
public async Task DownloadMetadata2CatalogAsync_WhenLoggerIsNull_Throws()
{
var exception = await Assert.ThrowsAsync <ArgumentNullException>(
() => FeedHelpers.DownloadMetadata2CatalogAsync(
Mock.Of <IPackageCatalogItemCreator>(),
new SortedList <DateTime, IList <FeedPackageDetails> >(),
Mock.Of <IStorage>(),
DateTime.MinValue,
DateTime.MinValue,
DateTime.MinValue,
maxDegreeOfParallelism: 1,
createdPackages: null,
updateCreatedFromEdited: false,
cancellationToken: CancellationToken.None,
telemetryService: Mock.Of <ITelemetryService>(),
logger: null));
Assert.Equal("logger", exception.ParamName);
}
protected override async Task RunInternalAsync(CancellationToken cancellationToken)
{
var timeout = TimeSpan.FromSeconds(300);
using (var client = CreateHttpClient())
{
client.Timeout = timeout;
// if the version is specified a single package is processed otherwise all the packages corresponding to that id are processed
var uri = (_version == null) ? MakePackageUri(_gallery, _id) : MakePackageUri(_gallery, _id, _version);
var packages = await FeedHelpers.GetPackagesInOrder(client, uri, package => package.CreatedDate);
Logger.LogInformation($"Downloading {packages.Select(t => t.Value.Count).Sum()} packages");
// the idea here is to leave the lastCreated, lastEdited and lastDeleted values exactly as they were
var catalogProperties = await CatalogProperties.ReadAsync(_storage, TelemetryService, cancellationToken);
var lastCreated = catalogProperties.LastCreated ?? DateTime.MinValue.ToUniversalTime();
var lastEdited = catalogProperties.LastEdited ?? DateTime.MinValue.ToUniversalTime();
var lastDeleted = catalogProperties.LastDeleted ?? DateTime.MinValue.ToUniversalTime();
var packageCatalogItemCreator = PackageCatalogItemCreator.Create(
client,
TelemetryService,
Logger,
storage: null);
await FeedHelpers.DownloadMetadata2CatalogAsync(
packageCatalogItemCreator,
packages,
_storage,
lastCreated,
lastEdited,
lastDeleted,
MaxDegreeOfParallelism,
createdPackages : null,
updateCreatedFromEdited : false,
cancellationToken : cancellationToken,
telemetryService : TelemetryService,
logger : Logger);
}
}
public async Task DownloadMetadata2CatalogAsync_WhenMaxDegreeOfParallelismIsOutOfRange_Throws(int maxDegreeOfParallelism)
{
var exception = await Assert.ThrowsAsync <ArgumentOutOfRangeException>(
() => FeedHelpers.DownloadMetadata2CatalogAsync(
Mock.Of <IPackageCatalogItemCreator>(),
new SortedList <DateTime, IList <FeedPackageDetails> >(),
Mock.Of <IStorage>(),
DateTime.UtcNow,
DateTime.UtcNow,
DateTime.UtcNow,
maxDegreeOfParallelism,
createdPackages: false,
updateCreatedFromEdited: false,
cancellationToken: CancellationToken.None,
telemetryService: Mock.Of <ITelemetryService>(),
logger: Mock.Of <ILogger>()));
Assert.Equal("maxDegreeOfParallelism", exception.ParamName);
Assert.StartsWith($"The argument must be within the range from 1 (inclusive) to {int.MaxValue} (inclusive).", exception.Message);
}
public async Task GetCatalogPropertiesAsync_ReturnsAllPropertiesIfAllPropertiesSet()
{
var datetimeOffset = CreateDateTimeOffset(TimeSpan.Zero);
var json = $"{{\"nuget:lastCreated\":\"{(datetimeOffset.ToString("O"))}\"," +
$"\"nuget:lastDeleted\":\"{(datetimeOffset.ToString("O"))}\"," +
$"\"nuget:lastEdited\":\"{(datetimeOffset.ToString("O"))}\"}}";
var storage = CreateStorageMock(json);
var catalogProperties = await FeedHelpers.GetCatalogPropertiesAsync(
storage.Object,
Mock.Of <ITelemetryService>(),
CancellationToken.None);
Assert.NotNull(catalogProperties);
Assert.NotNull(catalogProperties.LastCreated);
Assert.NotNull(catalogProperties.LastDeleted);
Assert.NotNull(catalogProperties.LastEdited);
Assert.Equal(datetimeOffset, catalogProperties.LastCreated.Value);
Assert.Equal(datetimeOffset, catalogProperties.LastDeleted.Value);
Assert.Equal(datetimeOffset, catalogProperties.LastEdited.Value);
storage.Verify();
}
/// <summary>
/// Parses the feed for the package specified by the <see cref="ValidationContext"/> and returns a <see cref="PackageTimestampMetadata"/>.
/// If the package is missing from the feed, returns the package's deletion audit record timestamp.
/// </summary>
public async Task <PackageTimestampMetadata> GetAsync(ValidationContext context)
{
var feedPackageDetails = await FeedHelpers.GetPackage(
context.Client,
_source,
context.Package.Id,
context.Package.Version.ToString());
if (feedPackageDetails != null)
{
return(PackageTimestampMetadata.CreateForPackageExistingOnFeed(
feedPackageDetails.CreatedDate,
feedPackageDetails.LastEditedDate));
}
DateTime?deleted = null;
if (context.DeletionAuditEntries.Any())
{
deleted = context.DeletionAuditEntries.Max(entry => entry.TimestampUtc);
}
return(PackageTimestampMetadata.CreateForPackageMissingFromFeed(deleted));
}
// Wrapper function for CatalogUtility.CreateHttpClient
// Overriden by NgTests.TestableDb2CatalogJob
protected virtual HttpClient CreateHttpClient()
{
return(FeedHelpers.CreateHttpClient(CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, Verbose)));
}
private async Task <Uri> ExecuteFeedToCatalogAsync(PerBatchContext context, IReadOnlyList <PerPackageContext> packageContexts)
{
var serviceProvider = GetServiceProvider(
context,
context.Global.CatalogContainerName,
context.Worker.CatalogStoragePath);
var now = DateTime.UtcNow;
var offset = 0;
var packages = new SortedList <DateTime, IList <FeedPackageDetails> >();
var maxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit;
foreach (var packageContext in packageContexts)
{
// These timestamps don't matter too much since the order that items are processed within a catalog
// commit is not defined. This is just a convenient way to get a bunch of unique timestamps to ease
// debugging.
var key = now.AddSeconds(offset--);
var published = now.AddSeconds(offset--);
var lastEdited = now.AddSeconds(offset--);
var created = now.AddSeconds(offset--);
packages.Add(key, new List <FeedPackageDetails>
{
new FeedPackageDetails(
packageContext.PackageUri,
created,
lastEdited,
published,
packageContext.PackageId,
packageContext.PackageVersion)
});
}
var storage = serviceProvider.GetRequiredService <IStorage>();
var createdPackages = true;
var updateCreatedFromEdited = false;
using (var httpClient = serviceProvider.GetRequiredService <HttpClient>())
{
var telemetryService = serviceProvider.GetRequiredService <ITelemetryService>();
var logger = serviceProvider.GetRequiredService <ILogger>();
var packageCatalogItemCreator = PackageCatalogItemCreator.Create(
httpClient,
telemetryService,
logger,
storage: null);
await FeedHelpers.DownloadMetadata2CatalogAsync(
packageCatalogItemCreator,
packages,
storage,
now,
now,
now,
maxDegreeOfParallelism,
createdPackages,
updateCreatedFromEdited,
CancellationToken.None,
telemetryService,
logger);
}
return(storage.ResolveUri("index.json"));
}
private static Task <SortedList <DateTime, IList <FeedPackageDetails> > > GetEditedPackages(HttpClient client, string source, DateTime since, int top)
{
return(FeedHelpers.GetPackagesInOrder(client, MakeLastEditedUri(source, since, top), package => package.LastEditedDate));
}
protected override async Task RunInternalAsync(CancellationToken cancellationToken)
{
using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri))
using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds))
using (var client = CreateHttpClient())
{
uint packagesDeleted;
uint packagesCreated;
uint packagesEdited;
client.Timeout = Timeout;
var packageCatalogItemCreator = PackageCatalogItemCreator.Create(
client,
TelemetryService,
Logger,
PreferredPackageSourceStorage);
do
{
packagesDeleted = 0;
packagesCreated = 0;
packagesEdited = 0;
// baseline timestamps
var catalogProperties = await FeedHelpers.GetCatalogPropertiesAsync(CatalogStorage, TelemetryService, cancellationToken);
var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc);
var lastEdited = catalogProperties.LastEdited ?? lastCreated;
var lastDeleted = catalogProperties.LastDeleted ?? lastCreated;
if (lastDeleted == Constants.DateTimeMinValueUtc)
{
lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated;
}
try
{
if (lastDeleted > Constants.DateTimeMinValueUtc)
{
using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds))
{
Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O"));
var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted);
packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count();
Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted);
// We want to ensure a commit only contains each package once at most.
// Therefore we segment by package id + version.
var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages);
foreach (var deletedPackagesSegment in deletedPackagesSegments)
{
lastDeleted = await Deletes2Catalog(
deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken);
// Wait for one second to ensure the next catalog commit gets a new timestamp
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
}
if (!SkipCreatedPackagesProcessing)
{
using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds))
{
Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O"));
var createdPackages = await GetCreatedPackages(client, Gallery, lastCreated, Top);
packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count();
Logger.LogInformation("FEED CreatedPackages: {CreatedPackagesCount}", packagesCreated);
lastCreated = await FeedHelpers.DownloadMetadata2CatalogAsync(
packageCatalogItemCreator,
createdPackages,
CatalogStorage,
lastCreated,
lastEdited,
lastDeleted,
MaxDegreeOfParallelism,
createdPackages : true,
updateCreatedFromEdited : false,
cancellationToken : cancellationToken,
telemetryService : TelemetryService,
logger : Logger);
}
}
using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds))
{
Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O"));
var editedPackages = await GetEditedPackages(client, Gallery, lastEdited, Top);
packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count();
Logger.LogInformation("FEED EditedPackages: {EditedPackagesCount}", packagesEdited);
lastEdited = await FeedHelpers.DownloadMetadata2CatalogAsync(
packageCatalogItemCreator,
editedPackages,
CatalogStorage,
lastCreated,
lastEdited,
lastDeleted,
MaxDegreeOfParallelism,
createdPackages : false,
updateCreatedFromEdited : SkipCreatedPackagesProcessing,
cancellationToken : cancellationToken,
telemetryService : TelemetryService,
logger : Logger);
}
}
finally
{
TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted);
if (!SkipCreatedPackagesProcessing)
{
TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated);
}
TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited);
}
} while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0);
}
}
private Task <IList <FeedPackageDetails> > TestGetPackagesAsync(IEnumerable <ODataPackage> oDataPackages)
{
return(FeedHelpers.GetPackages(
new HttpClient(GetMessageHandlerForPackages(oDataPackages)),
new Uri(_baseUri + "/test")));
}
