public async Task RunAsync_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Throws()
{
var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData);
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec");
var nupkgStream = File.OpenRead(@"Packages\ListedPackage.1.0.1.zip");
var expectedNupkg = GetStreamBytes(nupkgStream);
var catalogStorage = Catalogs.CreateTestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatch();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(nupkgStream)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
var exception = await Assert.ThrowsAsync <InvalidOperationException>(
() => _target.RunAsync(front, back, CancellationToken.None));
Assert.Equal("The catalog batch 10/13/2015 6:40:07 AM contains multiple entries for the same package identity. Package(s): listedpackage 1.0.0", exception.Message);
}
public async Task RunAsync_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(HttpStatusCode statusCode)
{
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.Return404OnUnknownAction = true;
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(statusCode)
{
Content = _noContent
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
var exception = await Assert.ThrowsAsync <InvalidOperationException>(
() => _target.RunAsync(front, back, CancellationToken.None));
Assert.Equal(
$"Expected status code OK for package download, actual: {statusCode}",
exception.Message);
Assert.Equal(0, _catalogToDnxStorage.Content.Count);
}
public async Task RunAsync_WithPackageCreatedThenDeleted_LeavesNoArtifacts()
{
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.nuspec");
var catalogStorage = Catalogs.CreateTestCatalogWithPackageCreatedThenDeleted();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/otherpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip"))
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(1, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
}
public async Task PutsSemVer2PackagesInLegacyStorageWhenSemVer2IsDisabled()
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: false);
var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package();
await _mockServer.AddStorageAsync(catalogStorage);
var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue);
var back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
var legacyCursor = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(legacyCursor.Key);
var legacyIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json"));
Assert.NotNull(legacyIndex.Key);
var legacyLeaf = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json"));
Assert.NotNull(legacyLeaf.Key);
Assert.Equal(3, _legacyStorage.Content.Count);
}
public async void CanReadWhatItSaves()
{
StorageContent savedContent = null;
var storageMock = CreateStorageMock();
storageMock
.Protected().Setup <Task>("OnSave", ItExpr.IsAny <Uri>(), ItExpr.IsAny <StorageContent>(), ItExpr.IsAny <CancellationToken>())
.Callback <Uri, StorageContent, CancellationToken>((uri, content, token) => { savedContent = content; })
.Returns(Task.FromResult(0));
storageMock
.Protected().Setup <Task <StorageContent> >("OnLoad", ItExpr.IsAny <Uri>(), ItExpr.IsAny <CancellationToken>())
.Returns <Uri, CancellationToken>((uri, token) => Task.FromResult(savedContent));
DateTimeOffset defaultValue = new DateTimeOffset(2017, 5, 5, 17, 8, 42, TimeSpan.Zero);
DateTimeOffset actualValue = new DateTimeOffset(2017, 5, 5, 17, 49, 42, TimeSpan.Zero);
var durableCursorSaver = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, defaultValue);
durableCursorSaver.Value = actualValue;
await durableCursorSaver.Save(CancellationToken.None);
var durableCursorLoader = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, defaultValue);
await durableCursorLoader.Load(CancellationToken.None);
Assert.Equal(actualValue, durableCursorLoader.Value);
}
public async Task RunAsync_WhenSourceNupkgIsNotFound_SkipsPackage()
{
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.nuspec");
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
{
Content = _noContent
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(1, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
}
static async Task Test1Async()
{
await MakeTestCatalog();
Func <HttpMessageHandler> handlerFunc = () =>
{
return(new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
});
};
TestCollector collector = new TestCollector("Test1", new Uri("http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
DurableCursor front = new DurableCursor(new Uri("http://localhost:8000/cursor/front.json"), storage, MemoryCursor.Min.Value);
//DurableCursor back = new DurableCursor(new Uri("http://localhost:8000/cursor/back.json"), storage);
MemoryCursor back = MemoryCursor.Max;
bool didWork = await collector.Run(front, back);
if (!didWork)
{
Console.WriteLine("executed but no work was done");
}
}
public async Task RunAsync_WhenPackageDoesNotHaveNuspec_SkipsPackage()
{
var zipWithNoNuspec = CreateZipStreamWithEntry("readme.txt", "content");
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(zipWithNoNuspec)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(1, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri));
}
public async Task RunAsync_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLeaf()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip"))
}));
_mockServer.SetAction(
"/packages/listedpackage.1.0.1.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip"))
}));
_mockServer.SetAction(
"/packages/unlistedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip"))
}));
_mockServer.SetAction(
"/packages/otherpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip"))
}));
ReadWriteCursor front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
Assert.Equal(9, _catalogToDnxStorage.Content.Count);
Assert.Equal(5, _mockServer.Requests.Count);
Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri);
Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri);
// The packages were processed in random order.
var remainingRequests = _mockServer.Requests
.Skip(2)
.Take(3)
.Select(request => request.RequestUri.AbsoluteUri)
.OrderBy(uri => uri)
.ToArray();
Assert.Contains("/listedpackage.1.0.0.nupkg", remainingRequests[0]);
Assert.Contains("/listedpackage.1.0.1.nupkg", remainingRequests[1]);
Assert.Contains("/unlistedpackage.1.0.0.nupkg", remainingRequests[2]);
}
public async Task RunAsync_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage()
{
_catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[]
{
new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg")
}, areRequiredPropertiesPresentAsync: false);
_catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name));
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec");
var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip");
var expectedNupkg = GetStreamBytes(nupkgStream);
await _catalogToDnxStorage.SaveAsync(
new Uri("http://tempuri.org/listedpackage/index.json"),
new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")),
CancellationToken.None);
_target = new DnxCatalogCollector(
new Uri("http://tempuri.org/index.json"),
_catalogToDnxStorageFactory,
_nullPreferredPackageSourceStorage,
_contentBaseAddress,
Mock.Of <ITelemetryService>(),
_logger,
_maxDegreeOfParallelism,
() => _mockServer);
var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(nupkgStream)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(4, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson));
}
public static void CreateNewCursor(string[] args)
{
IDictionary <string, string> arguments = GetArguments(args, 0);
StorageFactory storageFactory = CreateStorageFactory(arguments, verbose: true);
Storage storage = storageFactory.Create();
DurableCursor cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, GetDefaultValue(arguments));
cursor.Load().Wait();
cursor.Save().Wait();
}
public async Task ThrowsIfCommitTimesOut()
{
// Arrange
var storage = new MemoryStorage();
var storageFactory = new TestStorageFactory(name => storage.WithName(name));
MockServerHttpClientHandler mockServer;
mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage();
await mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(
storage.ResolveUri("cursor.json"),
storage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
var commitTimeout = TimeSpan.FromSeconds(1);
var stuckDuration = TimeSpan.FromMinutes(1);
var telemetryService = new Mock <ITelemetryService>();
var indexCommitDurationMetric = new Mock <IDisposable>();
telemetryService.Setup(t => t.TrackIndexCommitDuration()).Returns(indexCommitDurationMetric.Object);
using (var testDirectory = TestDirectory.Create())
{
var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory));
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory))
using (var stuckIndexWriter = StuckIndexWriter.FromIndexWriter(indexWriter, stuckDuration))
{
var target = new SearchIndexFromCatalogCollector(
new Uri("http://tempuri.org/index.json"),
stuckIndexWriter,
commitEachBatch: true,
commitTimeout: commitTimeout,
baseAddress: null,
telemetryService: telemetryService.Object,
logger: new TestLogger(),
handlerFunc: () => mockServer,
httpRetryStrategy: new NoRetryStrategy());
// Act & Assert
await Assert.ThrowsAsync <OperationCanceledException>(() => target.RunAsync(front, back, CancellationToken.None));
telemetryService.Verify(t => t.TrackIndexCommitDuration(), Times.Once);
telemetryService.Verify(t => t.TrackIndexCommitTimeout(), Times.Once);
indexCommitDurationMetric.Verify(m => m.Dispose(), Times.Never);
}
}
}
private async Task ExecuteAsync(CancellationToken token)
{
using (var cancelledCts = new CancellationTokenSource())
using (var produceWorkCts = new CancellationTokenSource())
{
// Initialize the indexes, container and excluded packages data.
await InitializeAsync();
// Here, we fetch the current catalog timestamp to use as the initial cursor value for
// catalog2azuresearch. The idea here is that database is always more up-to-date than the catalog.
// We're about to read the database so if we capture a catalog timestamp now, we are guaranteed that
// any data we get from a database query will be more recent than the data represented by this catalog
// timestamp. When catalog2azuresearch starts up for the first time to update the index produced by this
// job, it will probably encounter some duplicate packages, but this is okay.
//
// Note that we could capture any dependency cursors here instead of catalog cursor, but this is
// pointless because there is no reliable way to filter out data fetched from the database based on a
// catalog-based cursor value. Suppose the dependency cursor is catalog2registration. If
// catalog2registration is very behind, then the index produced by this job will include packages that
// are not yet restorable (since they are not in the registration hives). This could lead to a case
// where a user is able to search for a package that he cannot restore. We mitigate this risk by
// trusting that our end-to-end tests will fail when catalog2registration (or any other V3 component) is
// broken, this blocking the deployment of new Azure Search indexes.
var catalogIndex = await _catalogClient.GetIndexAsync(_options.Value.CatalogIndexUrl);
var initialCursorValue = catalogIndex.CommitTimestamp;
_logger.LogInformation("The initial cursor value will be {CursorValue:O}.", initialCursorValue);
var initialAuxiliaryData = await PushAllPackageRegistrationsAsync(cancelledCts, produceWorkCts);
// Write the owner data file.
await WriteOwnerDataAsync(initialAuxiliaryData.Owners);
// Write the download data file.
await WriteDownloadDataAsync(initialAuxiliaryData.Downloads);
// Write the verified packages data file.
await WriteVerifiedPackagesDataAsync(initialAuxiliaryData.VerifiedPackages);
// Write popularity transfers data file.
await WritePopularityTransfersDataAsync(initialAuxiliaryData.PopularityTransfers);
// Write the cursor.
_logger.LogInformation("Writing the initial cursor value to be {CursorValue:O}.", initialCursorValue);
var frontCursorStorage = _storageFactory.Create();
var frontCursor = new DurableCursor(
frontCursorStorage.ResolveUri(Catalog2AzureSearchCommand.CursorRelativeUri),
frontCursorStorage,
DateTime.MinValue);
frontCursor.Value = initialCursorValue.UtcDateTime;
await frontCursor.SaveAsync(token);
}
}
public async Task RunAsync_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage()
{
_catalogToDnxStorage = new SynchronizedMemoryStorage(new[]
{
new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"),
});
_catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name));
_mockServer = new MockServerHttpClientHandler();
_mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec");
_target = new DnxCatalogCollector(
new Uri("http://tempuri.org/index.json"),
_catalogToDnxStorageFactory,
_nullPreferredPackageSourceStorage,
_contentBaseAddress,
Mock.Of <ITelemetryService>(),
new Mock <ILogger>().Object,
_maxDegreeOfParallelism,
() => _mockServer);
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.1.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip"))
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(2, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
Assert.Equal(GetExpectedIndexJsonContent("1.0.1"), Encoding.UTF8.GetString(indexJson));
}
static async Task Test2Async()
{
await MakeTestCatalog();
Func <HttpMessageHandler> handlerFunc = () =>
{
return(new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
});
};
TestCollector collectorA = new TestCollector("A", new Uri("http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
DurableCursor cursorA = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorA.json"), storage, MemoryCursor.Min.Value);
DurableCursor cursorB = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorB.json"), storage, MemoryCursor.Min.Value);
Console.WriteLine("check catalog...");
bool run = false;
do
{
run = false;
run |= await collectorA.Run(cursorA, MemoryCursor.Max);
run |= await collectorB.Run(cursorB, cursorA);
}while (run);
Console.WriteLine("ADDING MORE CATALOG");
await MoreTestCatalog();
do
{
run = false;
run |= await collectorA.Run(cursorA, MemoryCursor.Max);
run |= await collectorB.Run(cursorB, cursorA);
}while (run);
Console.WriteLine("ALL DONE");
}
public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, string expectedCursorBeforeRetry)
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: false);
var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit();
await _mockServer.AddStorageAsync(catalogStorage);
// Make the first request for a catalog leaf node fail. This will cause the registration collector
// to fail the first time but pass the second time.
FailFirstRequest(catalogUri);
expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O");
ReadWriteCursor front = new DurableCursor(
_legacyStorage.ResolveUri("cursor.json"),
_legacyStorage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
// Act
await Assert.ThrowsAsync <BatchProcessingException>(
() => _target.RunAsync(front, back, CancellationToken.None));
var cursorBeforeRetry = front.Value;
await _target.RunAsync(front, back, CancellationToken.None);
var cursorAfterRetry = front.Value;
// Assert
var unlistedPackage100 = _legacyStorage
.Content
.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json"));
Assert.NotNull(unlistedPackage100.Key);
var listedPackage101 = _legacyStorage
.Content
.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json"));
Assert.NotNull(listedPackage101.Key);
var anotherPackage100 = _legacyStorage
.Content
.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/anotherpackage/1.0.0.json"));
Assert.NotNull(anotherPackage100.Key);
Assert.Equal(MemoryCursor.MinValue, cursorBeforeRetry);
Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry);
}
private async Task ExecuteAsync(CancellationToken token)
{
// Initialize the cursors.
ReadCursor backCursor;
if (_options.Value.DependencyCursorUrls != null &&
_options.Value.DependencyCursorUrls.Any())
{
_logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls);
backCursor = new AggregateCursor(_options
.Value
.DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc)));
}
else
{
_logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information.");
backCursor = MemoryCursor.CreateMax();
}
var frontCursorStorage = _storageFactory.Create();
var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri);
var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue);
_logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri);
LogContainerUrl(HiveType.Legacy, c => c.LegacyStorageContainer);
LogContainerUrl(HiveType.Gzipped, c => c.GzippedStorageContainer);
LogContainerUrl(HiveType.SemVer2, c => c.SemVer2StorageContainer);
// Optionally create the containers.
if (_options.Value.CreateContainers)
{
await CreateContainerIfNotExistsAsync(c => c.LegacyStorageContainer);
await CreateContainerIfNotExistsAsync(c => c.GzippedStorageContainer);
await CreateContainerIfNotExistsAsync(c => c.SemVer2StorageContainer);
}
await frontCursor.LoadAsync(token);
await backCursor.LoadAsync(token);
_logger.LogInformation(
"The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.",
frontCursor.Value,
backCursor.Value);
// Run the collector.
await _collector.RunAsync(
frontCursor,
backCursor,
token);
}
public async void UsesDefaultValue()
{
var storageMock = CreateStorageMock();
storageMock
.Protected().Setup <Task>("OnLoad", ItExpr.IsAny <Uri>(), ItExpr.IsAny <CancellationToken>())
.Returns(Task.FromResult <StorageContent>(null));
DateTimeOffset defaultValue = new DateTimeOffset(2017, 5, 5, 17, 8, 42, TimeSpan.Zero);
var durableCursor = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, defaultValue);
await durableCursor.Load(CancellationToken.None);
Assert.Equal(defaultValue, durableCursor.Value);
}
public async void SavesToStorage()
{
var storageMock = CreateStorageMock();
storageMock
.Protected().Setup <Task>("OnSave", ItExpr.IsAny <Uri>(), ItExpr.IsAny <StorageContent>(), ItExpr.IsAny <CancellationToken>())
.Returns(Task.FromResult(0))
.Verifiable();
var durableCursor = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, new DateTimeOffset(2017, 5, 5, 17, 8, 42, TimeSpan.Zero));
await durableCursor.Save(CancellationToken.None);
storageMock.Verify();
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
ServicePointManager.DefaultConnectionLimit = DegreeOfParallelism;
var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
var packageStorageBase = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
var failCacheTime = arguments.GetOrDefault(FailCacheTime, TimeSpan.FromHours(1));
var auxStorageFactory = CreateAuxStorageFactory(arguments, verbose);
var targetStorageFactory = CreateTargetStorageFactory(arguments, verbose);
var packageStorage = new AzureStorage(
storageBaseUri: new Uri(packageStorageBase),
maxExecutionTime: TimeSpan.FromMinutes(15),
serverTimeout: TimeSpan.FromMinutes(10),
useServerSideCopy: true,
compressContent: false,
verbose: true,
throttle: null);
var source = arguments.GetOrThrow <string>(Arguments.Source);
var iconProcessor = new IconProcessor(TelemetryService, LoggerFactory.CreateLogger <IconProcessor>());
var httpHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);
var httpMessageHandler = httpHandlerFactory();
var httpClient = new HttpClient(httpMessageHandler);
var simpleHttpClient = new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>());
var catalogClient = new CatalogClient(simpleHttpClient, LoggerFactory.CreateLogger <CatalogClient>());
var httpResponseProvider = new HttpClientWrapper(httpClient);
var externalIconProvider = new ExternalIconContentProvider(httpResponseProvider, LoggerFactory.CreateLogger <ExternalIconContentProvider>());
var iconCopyResultCache = new IconCopyResultCache(auxStorageFactory.Create(), failCacheTime, LoggerFactory.CreateLogger <IconCopyResultCache>());
var leafProcessor = new CatalogLeafDataProcessor(
packageStorage,
iconProcessor,
externalIconProvider,
iconCopyResultCache,
TelemetryService,
LoggerFactory.CreateLogger <CatalogLeafDataProcessor>());
_collector = new IconsCollector(
new Uri(source),
TelemetryService,
targetStorageFactory,
catalogClient,
leafProcessor,
iconCopyResultCache,
auxStorageFactory,
CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose),
LoggerFactory.CreateLogger <IconsCollector>());
var cursorStorage = auxStorageFactory.Create();
_front = new DurableCursor(cursorStorage.ResolveUri("c2icursor.json"), cursorStorage, DateTime.MinValue.ToUniversalTime());
}
public async Task CreatesRegistrationsWithSemVer2()
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: true);
var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package();
await _mockServer.AddStorageAsync(catalogStorage);
var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue);
var back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
// Verify the contents of the legacy (non-SemVer 2.0.0) storage
Assert.Equal(1, _legacyStorage.Content.Count);
var legacyCursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(legacyCursorJson.Key);
// Verify the contents of the SemVer 2.0.0 storage
Assert.Equal(2, _semVer2Storage.Content.Count);
var semVer2CursorJson = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.Null(semVer2CursorJson.Key);
var index = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json"));
Assert.NotNull(index.Key);
Assert.Contains("\"catalog:CatalogRoot\"", index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json\"", index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/testpackage.semver2.1.0.0-alpha.1.nupkg\"", index.Value.GetContentString());
Assert.Contains("\"version\":\"1.0.0-alpha.1+githash\"", index.Value.GetContentString());
Assert.Contains("1.0.0-alpha.1/1.0.0-alpha.1", index.Value.GetContentString());
Assert.Contains("\"lower\":\"1.0.0-alpha.1\",", index.Value.GetContentString());
Assert.Contains("\"upper\":\"1.0.0-alpha.1\"", index.Value.GetContentString());
var package = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json"));
Assert.NotNull(package.Key);
Assert.Contains("\"Package\"", package.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json\"", package.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/testpackage.semver2.1.0.0-alpha.1.nupkg\"", package.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/testpackage.semver2/index.json\"", package.Value.GetContentString());
}
public PackageHashProcessor(
IEntityRepository <Package> packageRepository,
IBatchProcessor batchProcessor,
IResultRecorder resultRecorder,
IOptionsSnapshot <PackageHashConfiguration> configuration,
DurableCursor cursor,
ILogger <PackageHashProcessor> logger)
{
_packageRepository = packageRepository ?? throw new ArgumentNullException(nameof(packageRepository));
_batchProcessor = batchProcessor ?? throw new ArgumentNullException(nameof(batchProcessor));
_resultRecorder = resultRecorder ?? throw new ArgumentNullException(nameof(resultRecorder));
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
_cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task HandlesDeleteCatalogItemWithNonNormalizedVersion()
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: false);
var catalogStorage = Catalogs.CreateTestCatalogWithNonNormalizedDelete();
await _mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue);
// Act
await _target.RunAsync(
front,
new MemoryCursor(DateTime.Parse("2015-10-12T10:08:54.1506742")),
CancellationToken.None);
var intermediateUris = _legacyStorage
.Content
.Select(pair => pair.Key.ToString())
.OrderBy(uri => uri)
.ToList();
await _target.RunAsync(
front,
MemoryCursor.CreateMax(),
CancellationToken.None);
// Assert
Assert.Equal(3, intermediateUris.Count);
Assert.Equal("http://tempuri.org/cursor.json", intermediateUris[0]);
Assert.Equal("http://tempuri.org/otherpackage/1.0.0.json", intermediateUris[1]);
Assert.Equal("http://tempuri.org/otherpackage/index.json", intermediateUris[2]);
// This should really be 1, but see:
// https://github.com/NuGet/Engineering/issues/404
var finalUris = _legacyStorage
.Content
.Select(pair => pair.Key.ToString())
.OrderBy(uri => uri)
.ToList();
Assert.Equal(2, finalUris.Count);
Assert.Equal("http://tempuri.org/cursor.json", finalUris[0]);
Assert.Equal("http://tempuri.org/otherpackage/1.0.0.json", finalUris[1]);
}
public async Task WhenPackageHasMultipleCommitsRespectsOrder(string pageContent)
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: false);
var catalogStorage = Catalogs.CreateTestCatalogWithThreeItemsForSamePackage(pageContent);
await _mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
Assert.Equal(3, _legacyStorage.Content.Count);
// Ensure storage has cursor.json
var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(cursorJson.Key);
// Check package entries - ListedPackage
var myPackageIndexFile = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/mypackage/index.json"));
Assert.NotNull(myPackageIndexFile.Key);
Assert.Contains("\"catalog:CatalogRoot\"", myPackageIndexFile.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", myPackageIndexFile.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json\"", myPackageIndexFile.Value.GetContentString());
Assert.Contains("\"packageContent\":\"http://tempuri.org/packages/mypackage.3.0.0.nupkg\"", myPackageIndexFile.Value.GetContentString());
Assert.Contains("\"lower\":\"3.0.0\",", myPackageIndexFile.Value.GetContentString());
Assert.Contains("\"upper\":\"3.0.0\"", myPackageIndexFile.Value.GetContentString());
var myPackageVersionFile = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/mypackage/3.0.0.json"));
Assert.NotNull(myPackageVersionFile.Key);
Assert.Contains("\"catalogEntry\":\"http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json\"", myPackageVersionFile.Value.GetContentString());
Assert.Contains("\"listed\":true", myPackageVersionFile.Value.GetContentString());
Assert.Contains("\"packageContent\":\"http://tempuri.org/packages/mypackage.3.0.0.nupkg\"", myPackageIndexFile.Value.GetContentString());
}
static async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool verbose, int interval)
{
CommitCollector collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress)
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back);
}while (run);
Thread.Sleep(interval * 1000);
}
}
public async Task RunAsync_WithValidPackage_CreatesFlatContainer()
{
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec");
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip");
var expectedNupkg = GetStreamBytes(nupkgStream);
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(nupkgStream)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(4, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec));
Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson));
Assert.Equal(expectedNupkg, nupkg);
Assert.Equal(
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n<package xmlns=\"http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd\">\r\n <metadata>\r\n <id>ListedPackage</id>\r\n <version>1.0.0</version>\r\n <authors>NuGet</authors>\r\n <requireLicenseAcceptance>false</requireLicenseAcceptance>\r\n <description>Package description.</description>\r\n </metadata>\r\n</package>",
Encoding.UTF8.GetString(nuspec));
}
public async Task ExecuteAsync(bool restart)
{
var fileSystemStorage = new FileStorageFactory(
new Uri("http://localhost/"),
Directory.GetCurrentDirectory(),
verbose: false);
var front = new DurableCursor(
new Uri("http://localhost/cursor.json"),
fileSystemStorage.Create(),
DateTime.MinValue);
if (restart)
{
await front.LoadAsync(CancellationToken.None);
front.Value = DateTime.MinValue;
await front.SaveAsync(CancellationToken.None);
}
var back = MemoryCursor.CreateMax();
await _collector.RunAsync(front, back, CancellationToken.None);
}
public async Task RunAsync_WhenPackageHasNuspecWithWrongName_ProcessesPackage()
{
var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData);
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.nuspec");
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/unlistedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(zipWithWrongNameNuspec)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(4, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec));
Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson));
Assert.Equal(zipWithWrongNameNuspec.ToArray(), nupkg);
Assert.Equal(_nuspecData, Encoding.UTF8.GetString(nuspec));
}
private async Task PrepareAsync()
{
_log.WriteLine("Making sure folder {0} exists.", _outputFolder);
if (!Directory.Exists(_outputFolder))
{
Directory.CreateDirectory(_outputFolder);
}
// Create reindex file
_log.WriteLine("Start preparing lightning reindex file...");
var latestCommit = DateTime.MinValue;
int numberOfEntries = 0;
string indexFile = Path.Combine(_outputFolder, "index.txt");
string optionalArgumentsTemplate = "optionalArguments";
using (var streamWriter = new StreamWriter(indexFile, false))
{
var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose);
var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory());
var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService);
var catalogIndexEntries = await catalogIndexReader.GetEntries();
foreach (var packageRegistrationGroup in catalogIndexEntries
.OrderBy(x => x.CommitTimeStamp)
.ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase)
.ThenBy(x => x.Version)
.GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase))
{
streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key);
var latestCatalogPages = new Dictionary <string, Uri>();
foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup)
{
string key = catalogIndexEntry.Version.ToNormalizedString();
if (latestCatalogPages.ContainsKey(key))
{
latestCatalogPages[key] = catalogIndexEntry.Uri;
}
else
{
latestCatalogPages.Add(key, catalogIndexEntry.Uri);
}
if (latestCommit < catalogIndexEntry.CommitTimeStamp)
{
latestCommit = catalogIndexEntry.CommitTimeStamp;
}
}
foreach (var latestCatalogPage in latestCatalogPages)
{
streamWriter.WriteLine("{0}", latestCatalogPage.Value);
}
}
}
_log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile);
// Write cursor to storage
_log.WriteLine("Start writing new cursor...");
var storage = _storageFactories.LegacyStorageFactory.Create();
var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit)
{
Value = latestCommit
};
await cursor.SaveAsync(CancellationToken.None);
_log.WriteLine("Finished writing new cursor.");
// Ensure the SemVer 2.0.0 storage containers is created, if applicable. The gzipped storage account is
// created above when we write the cursor.
_storageFactories.SemVer2StorageFactory?.Create();
// Write command files
_log.WriteLine("Start preparing lightning reindex command files...");
string templateFileContents;
using (var templateStreamReader = new StreamReader(_templateFile))
{
templateFileContents = await templateStreamReader.ReadToEndAsync();
}
int batchNumber = 0;
int batchSizeValue = int.Parse(_batchSize);
for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue)
{
var batchEnd = (batchStart + batchSizeValue - 1);
if (batchEnd >= numberOfEntries)
{
batchEnd = numberOfEntries - 1;
}
var cursorCommandFileName = "cursor" + batchNumber + ".cmd";
var cursorTextFileName = "cursor" + batchNumber + ".txt";
using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName)))
using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName)))
{
var commandStreamContents = templateFileContents;
var replacements = _arguments
.Concat(new[]
{
new KeyValuePair <string, string>("indexFile", indexFile),
new KeyValuePair <string, string>("cursorFile", cursorTextFileName)
});
foreach (var replacement in replacements)
{
commandStreamContents = commandStreamContents
.Replace($"[{replacement.Key}]", replacement.Value);
}
//the not required arguments need to be added only if they were passed in
//they cannot be hardcoded in the template
var optionalArguments = new StringBuilder();
AppendOptionalArgument(optionalArguments, Arguments.ContentIsFlatContainer);
AppendOptionalArgument(optionalArguments, Arguments.FlatContainerName);
AppendOptionalArgument(optionalArguments, Arguments.StorageSuffix);
AppendOptionalArgument(optionalArguments, Arguments.AllIconsInFlatContainer);
AppendOptionalArgument(optionalArguments, Arguments.Driver);
AppendOptionalArgument(optionalArguments, Arguments.Verbose);
commandStreamContents = commandStreamContents
.Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString());
await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents);
await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd);
}
batchNumber++;
}
_log.WriteLine("Finished preparing lightning reindex command files.");
_log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile);
_log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel.");
}
public async Task CreatesRegistrationsAndRespectsDeletes()
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: false);
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
Assert.Equal(6, _legacyStorage.Content.Count);
// Ensure storage has cursor.json
var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(cursorJson.Key);
// Check package entries - ListedPackage
var package1Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"listed\":true,", package1Index.Value.GetContentString());
Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"lower\":\"1.0.0\",", package1Index.Value.GetContentString());
Assert.Contains("\"upper\":\"1.0.1\"", package1Index.Value.GetContentString());
var package1 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json"));
Assert.Contains("\"listed\":true,", package1.Value.GetContentString());
Assert.NotNull(package1.Key);
var package2 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json"));
Assert.Contains("\"listed\":true,", package2.Value.GetContentString());
Assert.NotNull(package2.Key);
// Check package entries - UnlistedPackage
var package2Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json"));
Assert.NotNull(package2Index.Key);
Assert.Contains("\"listed\":false,", package2Index.Value.GetContentString());
Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString());
Assert.Contains("\"lower\":\"1.0.0\",", package2Index.Value.GetContentString());
Assert.Contains("\"upper\":\"1.0.0\"", package2Index.Value.GetContentString());
var package3 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json"));
Assert.Contains("\"listed\":false,", package3.Value.GetContentString());
Assert.NotNull(package3.Key);
// Ensure storage does not have the deleted "OtherPackage"
var otherPackageIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json"));
Assert.Null(otherPackageIndex.Key);
}
