protected override async Task <bool> OnProcessBatchAsync(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken)
{
JObject catalogIndex = null;
if (_baseAddress != null)
{
var stopwatch = Stopwatch.StartNew();
catalogIndex = await client.GetJObjectAsync(Index, cancellationToken);
_telemetryService.TrackCatalogIndexReadDuration(stopwatch.Elapsed, Index);
}
IEnumerable <JObject> catalogItems = await FetchCatalogItemsAsync(client, items, cancellationToken);
var numDocs = _indexWriter.NumDocs();
_logger.LogInformation(string.Format("Index contains {0} documents.", _indexWriter.NumDocs()));
ProcessCatalogIndex(_indexWriter, catalogIndex, _baseAddress);
ProcessCatalogItems(_indexWriter, catalogItems, _baseAddress);
var docsDifference = _indexWriter.NumDocs() - numDocs;
UpdateCommitMetadata(commitTimeStamp, docsDifference);
_logger.LogInformation(string.Format("Processed catalog items. Index now contains {0} documents. (total uncommitted {1}, batch {2})",
_indexWriter.NumDocs(), _metadataForNextCommit.Count, docsDifference));
if (_commitEachBatch || isLastBatch)
{
EnsureCommitted();
}
return(true);
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IList <JObject> items, JObject context)
{
List <Task <JObject> > tasks = new List <Task <JObject> >();
foreach (JObject item in items)
{
Uri itemUri = item["@id"].ToObject <Uri>();
tasks.Add(client.GetJObjectAsync(itemUri));
}
await Task.WhenAll(tasks.ToArray());
foreach (Task <JObject> task in tasks)
{
JObject obj = task.Result;
string id = obj["id"].ToString();
string version = obj["version"].ToString();
HashSet <string> versions;
if (!Result.TryGetValue(id, out versions))
{
versions = new HashSet <string>();
Result.Add(id, versions);
}
versions.Add(version);
}
return(true);
}
private void EnqueueBatchesIfNoFailures(
CollectorHttpClient client,
JToken context,
SortedDictionary <DateTime, CommitBatchTasks> commitBatchTasksMap,
Queue <CatalogItemBatch> unprocessedBatches,
Queue <BatchTask> processingBatches,
CatalogItemBatch lastBatch,
CancellationToken cancellationToken)
{
var hasAnyBatchFailed = processingBatches.Any(batch => batch.Task.IsFaulted || batch.Task.IsCanceled);
if (hasAnyBatchFailed)
{
return;
}
var batchesToEnqueue = Math.Min(
_maxConcurrentBatches - processingBatches.Count(batch => !batch.Task.IsCompleted),
unprocessedBatches.Count);
for (var i = 0; i < batchesToEnqueue; ++i)
{
var batch = unprocessedBatches.Dequeue();
var batchItem = batch.Items.First();
var packageId = GetKey(batchItem.Value);
var batchTask = commitBatchTasksMap[batchItem.CommitTimeStamp].BatchTasks
.Single(bt => bt.PackageId == packageId);
batchTask.Task = ProcessBatchAsync(client, context, packageId, batch, lastBatch, cancellationToken);
processingBatches.Enqueue(batchTask);
}
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject item in items)
{
Uri itemUri = item["@id"].ToObject<Uri>();
tasks.Add(client.GetJObjectAsync(itemUri, cancellationToken));
}
await Task.WhenAll(tasks.ToArray());
foreach (Task<JObject> task in tasks)
{
JObject obj = task.Result;
string id = obj["id"].ToString();
string version = obj["version"].ToString();
HashSet<string> versions;
if (!Result.TryGetValue(id, out versions))
{
versions = new HashSet<string>();
Result.Add(id, versions);
}
versions.Add(version);
}
return true;
}
public void Client_ReturnsCorrectValue()
{
var client = new CollectorHttpClient();
var context = CreateContext(client: client);
Assert.Same(client, context.Client);
}
private async Task ProcessBatchAsync(
CollectorHttpClient client,
JToken context,
string packageId,
CatalogItemBatch batch,
CatalogItemBatch lastBatch,
CancellationToken cancellationToken)
{
await Task.Yield();
using (_telemetryService.TrackDuration(
TelemetryConstants.ProcessBatchSeconds,
new Dictionary <string, string>()
{
{ TelemetryConstants.Id, packageId },
{ TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() }
}))
{
await OnProcessBatchAsync(
client,
batch.Items.Select(item => item.Value),
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
}
}
protected ValidationContext CreateValidationContext(string packageResource = null)
{
// Add the package
if (packageResource != null)
{
var resourceStream = File.OpenRead(packageResource);
_mockServer.SetAction(
$"/packages/testpackage/1.0.0/testpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(resourceStream)
}));
}
var httpClient = new CollectorHttpClient(_mockServer);
// Mock V2 feed response for the package's Created/LastEdited timestamps. These timestamps must match
// the mocked catalog entry's timestamps.
var timestamp = PackageTimestampMetadata.CreateForPackageExistingOnFeed(created: PackageCreationTime, lastEdited: PackageCreationTime);
var timestampMetadataResource = new Mock <IPackageTimestampMetadataResource>();
timestampMetadataResource.Setup(t => t.GetAsync(It.IsAny <ValidationContext>()))
.ReturnsAsync(timestamp);
return(ValidationContextStub.Create(
PackageIdentity,
_catalogEntries,
client: httpClient,
timestampMetadataResource: timestampMetadataResource.Object));
}
protected ValidationContext CreateValidationContext(string packageResource = null)
{
// Add the package
if (packageResource != null)
{
var packageId = PackageIdentity.Id.ToLowerInvariant();
var packageVersion = PackageIdentity.Version.ToNormalizedString().ToLowerInvariant();
var relativeUrl = $"/packages/{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg";
var bytes = File.ReadAllBytes(packageResource);
_mockServer.SetAction(
relativeUrl,
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(bytes)
}));
}
var httpClient = new CollectorHttpClient(_mockServer);
// Mock V2 feed response for the package's Created/LastEdited timestamps. These timestamps must match
// the mocked catalog entry's timestamps.
var timestamp = PackageTimestampMetadata.CreateForExistingPackage(created: PackageCreationTime, lastEdited: PackageCreationTime);
var timestampMetadataResource = new Mock <IPackageTimestampMetadataResource>();
timestampMetadataResource.Setup(t => t.GetAsync(It.IsAny <ValidationContext>()))
.ReturnsAsync(timestamp);
return(ValidationContextStub.Create(
PackageIdentity,
_catalogEntries,
client: httpClient,
timestampMetadataResource: timestampMetadataResource.Object));
}
protected ValidationContext CreateValidationContext(string packageResource = null)
{
// Add the package
if (packageResource != null)
{
var resourceStream = File.OpenRead(packageResource);
_mockServer.SetAction(
$"/packages/testpackage/1.0.0/testpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(resourceStream)
}));
}
// Create the validation context.
var httpClient = new CollectorHttpClient(_mockServer);
return(new ValidationContext(
new PackageIdentity(PackageId, PackageNuGetVersion),
_catalogEntries,
new DeletionAuditEntry[0],
httpClient,
CancellationToken.None));
}
public CollectorHttpClientTests()
{
_telemetryService = new Mock <ITelemetryService>();
_handler = new MockServerHttpClientHandler();
_target = new CollectorHttpClient(_handler);
}
protected override async Task<bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken)
{
await front.Load(cancellationToken);
DateTime frontDateTime = front.Value;
JObject root = await client.GetJObjectAsync(Index, cancellationToken);
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject rootItem in root["items"])
{
DateTime pageTimeStamp = rootItem["commitTimeStamp"].ToObject<DateTime>();
if (pageTimeStamp > frontDateTime)
{
int count = int.Parse(rootItem["count"].ToString());
Total += count;
front.Value = pageTimeStamp;
await front.Save(cancellationToken);
}
}
return true;
}
public async Task ValidateAsync_WhenRequirePackageSignatureIsTrue_RequiresPackageSignatureFile(
bool includePackageSignatureFile)
{
var storage = new MemoryStorage(_baseUri);
var now = GetDateTime();
var storageContent = CreateStorageContent(now, includePackageSignatureFile);
storage.Content.TryAdd(_packageDetailsUri, storageContent);
using (var clientHandler = new MockServerHttpClientHandler())
using (CollectorHttpClient client = await CreateCollectorHttpClientStubAsync(clientHandler, storage))
{
ValidationContext context = CreateContext(client, now);
CatalogAggregateValidator validator = CreateValidator(context, requirePackageSignature: true);
AggregateValidationResult result = await validator.ValidateAsync(context);
Assert.Single(result.ValidationResults);
var expectedResult = includePackageSignatureFile ? TestResult.Pass : TestResult.Fail;
var actualResult = result.ValidationResults.Single();
Assert.Equal(expectedResult, actualResult.Result);
if (includePackageSignatureFile)
{
Assert.Null(actualResult.Exception);
}
else
{
Assert.IsType <MissingPackageSignatureFileException>(actualResult.Exception);
}
}
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IList <JObject> items, JObject context)
{
foreach (JObject item in items)
{
string id = item["nuget:id"].ToString();
string version = item["nuget:version"].ToString();
//Console.WriteLine("{0}/{1}", id, version);
if (_version != null)
{
if (id.Equals(_id, StringComparison.InvariantCultureIgnoreCase) && version.Equals(_version, StringComparison.InvariantCultureIgnoreCase))
{
PackageDetails = await client.GetJObjectAsync(new Uri(item["@id"].ToString()));
return(false);
}
}
else
{
if (id.Equals(_id, StringComparison.InvariantCultureIgnoreCase))
{
PackageDetails = await client.GetJObjectAsync(new Uri(item["@id"].ToString()));
return(false);
}
}
}
return(true);
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
foreach (JObject item in items)
{
string id = item["nuget:id"].ToString();
string version = item["nuget:version"].ToString();
//Console.WriteLine("{0}/{1}", id, version);
if (_version != null)
{
if (id.Equals(_id, StringComparison.InvariantCultureIgnoreCase) && version.Equals(_version, StringComparison.InvariantCultureIgnoreCase))
{
PackageDetails = await client.GetJObjectAsync(new Uri(item["@id"].ToString()), cancellationToken);
return false;
}
}
else
{
if (id.Equals(_id, StringComparison.InvariantCultureIgnoreCase))
{
PackageDetails = await client.GetJObjectAsync(new Uri(item["@id"].ToString()), cancellationToken);
return false;
}
}
}
return true;
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp)
{
foreach (JToken item in items)
{
string id = item["nuget:id"].ToString().ToLowerInvariant();
string version = item["nuget:version"].ToString().ToLowerInvariant();
Storage storage = _storageFactory.Create(id);
string nuspec = await LoadNuspec(id, version);
if (nuspec != null)
{
await SaveNuspec(storage, id, version, nuspec);
await CopyNupkg(storage, id, version);
await UpdateMetadata(storage, version);
Trace.TraceInformation("commit: {0}/{1}", id, version);
}
else
{
Trace.TraceInformation("no nuspec available for {0}/{1} skipping", id, version);
}
}
return(true);
}
protected override async Task <bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back)
{
await front.Load();
DateTime frontDateTime = front.Value;
JObject root = await client.GetJObjectAsync(Index);
List <Task <JObject> > tasks = new List <Task <JObject> >();
foreach (JObject rootItem in root["items"])
{
DateTime pageTimeStamp = rootItem["commitTimeStamp"].ToObject <DateTime>();
if (pageTimeStamp > frontDateTime)
{
int count = int.Parse(rootItem["count"].ToString());
Total += count;
front.Value = pageTimeStamp;
await front.Save();
}
}
return(true);
}
protected override async Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last)
{
CollectorCursor cursor = last;
DateTime lastDateTime = (DateTime)cursor;
JObject root = await client.GetJObjectAsync(index);
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject rootItem in root["items"])
{
DateTime pageTimeStamp = rootItem["commitTimestamp"].ToObject<DateTime>();
if (pageTimeStamp > lastDateTime)
{
cursor = pageTimeStamp;
int count = int.Parse(rootItem["count"].ToString());
Total += count;
}
}
return cursor;
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
var gallery = arguments.GetOrThrow <string>(Arguments.Gallery);
var index = arguments.GetOrThrow <string>(Arguments.Index);
var packageBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
var source = arguments.GetOrThrow <string>(Arguments.Source);
var requireSignature = arguments.GetOrDefault(Arguments.RequireSignature, false);
var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
CommandHelpers.AssertAzureStorage(arguments);
var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose);
var endpointInputs = CommandHelpers.GetEndpointFactoryInputs(arguments);
var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);
Logger.LogInformation(
"CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} endpoints: {Endpoints}",
gallery, index, monitoringStorageFactory, auditingStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name)));
_packageValidator = new PackageValidatorFactory(LoggerFactory)
.Create(gallery, index, packageBaseAddress, auditingStorageFactory, endpointInputs, messageHandlerFactory, requireSignature, verbose);
_queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version);
_statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);
_notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger <LoggerMonitoringNotificationService>());
_regResource = Repository.Factory.GetCoreV3(index).GetResource <RegistrationResourceV3>(cancellationToken);
_client = new CollectorHttpClient(messageHandlerFactory());
}
protected override async Task <bool> ProcessBatch(CollectorHttpClient client, IList <Newtonsoft.Json.Linq.JObject> items, Newtonsoft.Json.Linq.JObject context)
{
_cache = new NuGet3.Client.Core.JsonLdPageCache(client);
PerfEventTracker perfTracker = new PerfEventTracker();
TextWriter log = Console.Out;
Task <Document>[] packages = items.Select(x => MakePackage(client, x)).Where(x => x != null).ToArray();
await Task.WhenAll(packages);
foreach (Task <Document> pkg in packages)
{
if (pkg != null && pkg.Result != null)
{
Console.WriteLine("Package: {0}", pkg.Result.Get("Id"));
}
}
using (IndexWriter indexWriter = CreateIndexWriter(_directory, create: false))
{
foreach (Document doc in packages.Select(x => x.Result))
{
if (doc != null)
{
Console.WriteLine("Index document: {0}", doc.Get("Id"));
indexWriter.AddDocument(doc);
}
}
indexWriter.Commit();
}
return(true);
}
public GraphStrike(LightningJob job, CollectorHttpClient collectorHttpClient, string packageId)
{
_collectorHttpClient = collectorHttpClient;
_packageId = packageId;
_job = job;
_sortedGraphs = new Dictionary <string, IGraph>();
}
protected override async Task <bool> OnProcessBatchAsync(
CollectorHttpClient client,
IEnumerable <JToken> items,
JToken context,
DateTime commitTimeStamp,
bool isLastBatch,
CancellationToken cancellationToken)
{
var catalogEntries = items.Select(
item => new CatalogEntry(
item["nuget:id"].ToString().ToLowerInvariant(),
NuGetVersionUtility.NormalizeVersion(item["nuget:version"].ToString()).ToLowerInvariant(),
item["@type"].ToString().Replace("nuget:", Schema.Prefixes.NuGet),
item))
.ToList();
// Sanity check: a single catalog batch should not contain multiple entries for the same package ID and version.
AssertNoMultipleEntriesForSamePackageIdentity(commitTimeStamp, catalogEntries);
// Process .nupkg/.nuspec adds and deletes.
var processedCatalogEntries = await ProcessCatalogEntriesAsync(client, catalogEntries, cancellationToken);
// Update the package version index with adds and deletes.
await UpdatePackageVersionIndexAsync(processedCatalogEntries, cancellationToken);
return(true);
}
public static async Task <PackageTimestampMetadata> FromCatalogEntry(
CollectorHttpClient client,
CatalogIndexEntry catalogEntry)
{
var catalogLeaf = await client.GetJObjectAsync(catalogEntry.Uri);
try
{
if (catalogEntry.IsDelete)
{
// On the catalog page for a delete, the published value is the timestamp the package was deleted from the audit records.
var deleted = catalogLeaf.GetValue("published").Value <DateTimeOffset>();
return(CreateForPackageMissingFromFeed(deleted.DateTime));
}
else
{
var created = catalogLeaf.GetValue("created").Value <DateTimeOffset>();
var lastEdited = catalogLeaf.GetValue("lastEdited").Value <DateTimeOffset>();
return(CreateForPackageExistingOnFeed(created.DateTime, lastEdited.DateTime));
}
}
catch (Exception e)
{
throw new ArgumentException("Failed to create PackageTimestampMetadata from CatalogIndexEntry!", e);
}
}
protected override Task ProcessSortedBatchAsync(
CollectorHttpClient client,
KeyValuePair <FeedPackageIdentity, IList <JObject> > sortedBatch,
JToken context,
CancellationToken cancellationToken)
{
return(OverridableProcessSortedBatch(sortedBatch));
}
protected override Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
foreach (JObject item in items)
{
Console.WriteLine("{0} {1}", _name, item["@id"]);
}
return Task.FromResult(true);
}
protected override Task <bool> OnProcessBatch(CollectorHttpClient client, IList <JObject> items, JObject context)
{
foreach (JObject item in items)
{
Console.WriteLine("{0} {1}", _name, item["@id"]);
}
return(Task.FromResult(true));
}
protected override Task <bool> OnProcessBatch(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp)
{
foreach (JObject item in items)
{
Console.WriteLine("{0} {1}", _name, item["@id"].ToString());
}
return(Task.FromResult(true));
}
public static async Task <PackageTimestampMetadata> FromCatalogEntries(CollectorHttpClient client,
IEnumerable <CatalogIndexEntry> catalogEntries)
{
var packageTimestampMetadatas =
await Task.WhenAll(catalogEntries.Select(entry => FromCatalogEntry(client, entry)));
var maxTimestamp = packageTimestampMetadatas.Where(p => p != null).Max(p => p.Last);
return(packageTimestampMetadatas.FirstOrDefault(p => p.Last == maxTimestamp));
}
private static Task NoOpProcessBatchAsync(
CollectorHttpClient client,
JToken context,
string packageId,
CatalogCommitItemBatch batch,
CatalogCommitItemBatch lastBatch,
CancellationToken cancellationToken)
{
return(Task.CompletedTask);
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp, CancellationToken cancellationToken)
{
IEnumerable <JObject> catalogItems = await FetchCatalogItems(client, items, cancellationToken);
foreach (var entry in catalogItems)
{
Console.WriteLine(entry["id"]);
}
return(await Task.FromResult(true));
}
private ValidationContextStub(
PackageIdentity package,
IEnumerable <CatalogIndexEntry> entries,
IEnumerable <DeletionAuditEntry> deletionAuditEntries,
ValidationSourceRepositories sourceRepositories,
CollectorHttpClient client,
CancellationToken token,
ILogger <ValidationContext> logger)
: base(package, entries, deletionAuditEntries, sourceRepositories, client, token, logger)
{
}
public ValidationContext(
PackageIdentity package,
IEnumerable <CatalogIndexEntry> entries,
IEnumerable <DeletionAuditEntry> deletionAuditEntries,
IReadOnlyDictionary <FeedType, SourceRepository> feedToSource,
CollectorHttpClient client,
CancellationToken token,
ILogger <ValidationContext> logger)
{
if (entries == null)
{
throw new ArgumentNullException(nameof(entries));
}
if (deletionAuditEntries == null)
{
throw new ArgumentNullException(nameof(deletionAuditEntries));
}
if (feedToSource == null)
{
throw new ArgumentNullException(nameof(feedToSource));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
Package = package ?? throw new ArgumentNullException(nameof(package));
Entries = entries.ToList();
DeletionAuditEntries = deletionAuditEntries.ToList();
Client = client ?? throw new ArgumentNullException(nameof(client));
CancellationToken = token;
_timestampMetadataResourceV2 = feedToSource[FeedType.HttpV2].GetResource <IPackageTimestampMetadataResource>();
_v2Resource = feedToSource[FeedType.HttpV2].GetResource <IPackageRegistrationMetadataResource>();
_v3Resource = feedToSource[FeedType.HttpV3].GetResource <IPackageRegistrationMetadataResource>();
var commonLogger = logger.AsCommon();
_v2Index = new Lazy <Task <PackageRegistrationIndexMetadata> >(
() => _v2Resource.GetIndexAsync(Package, commonLogger, CancellationToken));
_v3Index = new Lazy <Task <PackageRegistrationIndexMetadata> >(
() => _v3Resource.GetIndexAsync(Package, commonLogger, CancellationToken));
_v2Leaf = new Lazy <Task <PackageRegistrationLeafMetadata> >(
() => _v2Resource.GetLeafAsync(Package, commonLogger, CancellationToken));
_v3Leaf = new Lazy <Task <PackageRegistrationLeafMetadata> >(
() => _v3Resource.GetLeafAsync(Package, commonLogger, CancellationToken));
_timestampMetadataV2 = new Lazy <Task <PackageTimestampMetadata> >(
() => _timestampMetadataResourceV2.GetAsync(this));
}
internal static ValidationContextStub Create(
PackageIdentity package = null,
IEnumerable <CatalogIndexEntry> entries = null,
IEnumerable <DeletionAuditEntry> deletionAuditEntries = null,
Dictionary <FeedType, SourceRepository> feedToSource = null,
CollectorHttpClient client = null,
HttpClientHandler clientHandler = null,
CancellationToken?token = null,
ILogger <ValidationContext> logger = null,
IPackageTimestampMetadataResource timestampMetadataResource = null,
IPackageRegistrationMetadataResource v2Resource = null,
IPackageRegistrationMetadataResource v3Resource = null)
{
if (feedToSource == null)
{
feedToSource = new Dictionary <FeedType, SourceRepository>();
var v2Repository = new Mock <SourceRepository>();
var v3Repository = new Mock <SourceRepository>();
feedToSource.Add(FeedType.HttpV2, v2Repository.Object);
feedToSource.Add(FeedType.HttpV3, v3Repository.Object);
v2Repository.Setup(x => x.GetResource <IPackageTimestampMetadataResource>())
.Returns(timestampMetadataResource ?? Mock.Of <IPackageTimestampMetadataResource>());
v2Repository.Setup(x => x.GetResource <IPackageRegistrationMetadataResource>())
.Returns(v2Resource ?? Mock.Of <IPackageRegistrationMetadataResource>());
v3Repository.Setup(x => x.GetResource <IPackageRegistrationMetadataResource>())
.Returns(v3Resource ?? Mock.Of <IPackageRegistrationMetadataResource>());
v3Repository.Setup(x => x.GetResource <HttpSourceResource>())
.Returns(new HttpSourceResource(new HttpSource(
new PackageSource("https://example/v3/index.json"),
() => Task.FromResult <HttpHandlerResource>(new HttpHandlerResourceV3(
clientHandler ?? new HttpClientHandler(),
clientHandler ?? Mock.Of <HttpMessageHandler>())),
NullThrottle.Instance)));
}
var sourceRepositories = new ValidationSourceRepositories(
feedToSource[FeedType.HttpV2],
feedToSource[FeedType.HttpV3]);
return(new ValidationContextStub(
package ?? _packageIdentity,
entries ?? Enumerable.Empty <CatalogIndexEntry>(),
deletionAuditEntries ?? Enumerable.Empty <DeletionAuditEntry>(),
sourceRepositories,
client ?? new CollectorHttpClient(clientHandler),
token ?? CancellationToken.None,
logger ?? Mock.Of <ILogger <ValidationContext> >()));
}
private ValidationContextStub(
PackageIdentity package,
IEnumerable <CatalogIndexEntry> entries,
IEnumerable <DeletionAuditEntry> deletionAuditEntries,
IReadOnlyDictionary <FeedType, SourceRepository> feedToSource,
CollectorHttpClient client,
CancellationToken token,
ILogger <ValidationContext> logger)
: base(package, entries, deletionAuditEntries, feedToSource, client, token, logger)
{
}
protected ValidationContext CreateValidationContext(IEnumerable <CatalogIndexEntry> catalogEntries = null)
{
catalogEntries = catalogEntries ?? new CatalogIndexEntry[0];
var httpClient = new CollectorHttpClient(_mockServer);
return(ValidationContextStub.Create(
PackageIdentity,
catalogEntries,
client: httpClient));
}
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
List<Task<string>> tasks = new List<Task<string>>();
foreach (JObject item in items)
{
Uri itemUri = item["url"].ToObject<Uri>();
tasks.Add(client.GetStringAsync(itemUri));
}
await Task.WhenAll(tasks.ToArray());
}
protected override async Task <bool> OnProcessBatchAsync(
CollectorHttpClient client,
IEnumerable <CatalogCommitItem> items,
JToken context,
DateTime commitTimeStamp,
bool isLastBatch,
CancellationToken cancellationToken)
{
await _logic.OnProcessBatchAsync(items);
return(true);
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
List<Task<string>> tasks = new List<Task<string>>();
foreach (JObject item in items)
{
Uri itemUri = item["@id"].ToObject<Uri>();
tasks.Add(client.GetStringAsync(itemUri, cancellationToken));
}
await Task.WhenAll(tasks.ToArray());
return true;
}
protected override Task<bool> OnProcessBatch(CollectorHttpClient client, IEnumerable<JToken> items, JToken context, DateTime commitTimeStamp, CancellationToken cancellationToken)
{
Console.WriteLine("------------------------------------------------");
Console.WriteLine("COMMIT: {0}", commitTimeStamp.ToString("O"));
foreach (JToken item in items)
{
Console.WriteLine("{0} {1}", item["@id"], item["@type"]);
Console.WriteLine("{0} {1} {2}", item["nuget:id"], item["nuget:version"], item["commitId"]);
}
Console.WriteLine("------------------------------------------------");
return Task.FromResult(true);
}
protected override async Task Fetch(CollectorHttpClient client, Uri index, DateTime last)
{
JObject root = await client.GetJObjectAsync(index);
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject rootItem in root["item"])
{
DateTime pageTimeStamp = rootItem["timeStamp"]["@value"].ToObject<DateTime>();
if (pageTimeStamp > last)
{
int count = int.Parse(rootItem["count"].ToString());
Total += count;
}
}
}
protected override Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
foreach (JObject item in items)
{
string id = item["nuget:id"].ToString();
string version = item["nuget:version"].ToString();
IList<string> versions;
if (!Result.TryGetValue(id, out versions))
{
versions = new List<string>();
Result.Add(id, versions);
}
versions.Add(version);
}
return Task.FromResult(true);
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject item in items)
{
Uri itemUri = item["@id"].ToObject<Uri>();
tasks.Add(client.GetJObjectAsync(itemUri, cancellationToken));
}
await Task.WhenAll(tasks.ToArray());
foreach (Task<JObject> task in tasks)
{
JObject obj = task.Result;
Result.Add(obj["id"].ToString().ToLowerInvariant());
}
return true;
}
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject item in items)
{
Uri itemUri = item["url"].ToObject<Uri>();
tasks.Add(client.GetJObjectAsync(itemUri));
}
await Task.WhenAll(tasks.ToArray());
//DEBUG
Console.WriteLine("{0}", client.RequestCount);
foreach (Task<JObject> task in tasks)
{
JObject obj = task.Result;
_result.Add(obj["id"].ToString().ToLowerInvariant());
}
}
protected async override Task<bool> OnProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context, CancellationToken cancellationToken)
{
List<Task<string>> tasks = new List<Task<string>>();
foreach (JObject item in items)
{
DateTime itemMinDownloadTimestamp = item["http://nuget.org/schema#minDownloadTimestamp"]["@value"].ToObject<DateTime>();
DateTime itemMaxDownloadTimestamp = item["http://nuget.org/schema#maxDownloadTimestamp"]["@value"].ToObject<DateTime>();
if (SelectItem(itemMinDownloadTimestamp, itemMaxDownloadTimestamp))
{
Uri itemUri = item["url"].ToObject<Uri>();
tasks.Add(client.GetStringAsync(itemUri, cancellationToken));
}
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks.ToArray());
foreach (Task<string> task in tasks)
{
JArray statisticsPage = JArray.Parse(task.Result);
foreach (JArray row in statisticsPage)
{
DateTime rowTimeStamp = row[1].ToObject<DateTime>();
if (SelectRow(rowTimeStamp))
{
Count++;
}
}
}
}
return true;
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IEnumerable<JToken> items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken)
{
foreach (JToken item in items)
{
string id = item["nuget:id"].ToString().ToLowerInvariant();
string version = item["nuget:version"].ToString().ToLowerInvariant();
string type = item["@type"].ToString().Replace("nuget:", Schema.Prefixes.NuGet);
if (type == Schema.DataTypes.PackageDetails.ToString())
{
// Add/update package
string nuspec = await LoadNuspec(client, id, version, cancellationToken);
if (nuspec != null)
{
var requestUri = Utilities.GetNugetCacheBustingUri(new Uri(ContentBaseAddress, string.Format("{0}.{1}.nupkg", id, version)));
using (Stream stream = await client.GetStreamAsync(requestUri))
{
await _dnxMaker.AddPackage(stream, nuspec, id, version, cancellationToken);
}
Trace.TraceInformation("commit: {0}/{1}", id, version);
}
else
{
Trace.TraceWarning("no nuspec available for {0}/{1} skipping", id, version);
}
}
else if (type == Schema.DataTypes.PackageDelete.ToString())
{
await _dnxMaker.DeletePackage(id, version, cancellationToken);
Trace.TraceInformation("commit delete: {0}/{1}", id, version);
}
}
return true;
}
