public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken)
{
CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress),
UnlistShouldDelete = unlistShouldDelete
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back, cancellationToken);
}
while (run);
Thread.Sleep(interval * 1000);
}
}
public static async Task Test4Async()
{
//Uri catalogUri = new Uri("https://nugetdevstorage.blob.core.windows.net/catalog/index.json");
Uri catalogUri = new Uri("https://nugetjohtaylo.blob.core.windows.net/catalog/index.json");
//Uri catalogUri = new Uri("https://api.nuget.org/v3/catalog0/index.json");
string path = "c:\\data\\registration20150421";
DirectoryInfo directoryInfo = new DirectoryInfo(path);
if (directoryInfo.Exists)
{
directoryInfo.Delete(true);
}
directoryInfo.Create();
FileStorageFactory factory = new FileStorageFactory(new Uri("http://tempuri.org"), path);
CollectorBase collector = new RegistrationCollector(catalogUri, factory)
{
UnlistShouldDelete = true,
Concurrent = false
};
await collector.Run(CancellationToken.None);
}
public async Task CreatesRegistrationsAndRespectsDeletes()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var catalogToRegistrationStorage = new MemoryStorage();
var catalogToRegistrationStorageFactory = new TestStorageFactory(name => catalogToRegistrationStorage.WithName(name));
var mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", async request => new HttpResponseMessage(HttpStatusCode.OK));
await mockServer.AddStorage(catalogStorage);
// Setup collector
var target = new RegistrationCollector(new Uri("http://tempuri.org/index.json"), catalogToRegistrationStorageFactory, () => mockServer)
{
ContentBaseAddress = new Uri("http://tempuri.org/packages"),
UnlistShouldDelete = false
};
ReadWriteCursor front = new DurableCursor(catalogToRegistrationStorage.ResolveUri("cursor.json"), catalogToRegistrationStorage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
// Act
await target.Run(front, back, CancellationToken.None);
// Assert
Assert.Equal(6, catalogToRegistrationStorage.Content.Count);
// Ensure storage has cursor.json
var cursorJson = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(cursorJson.Key);
// Check package entries - ListedPackage
var package1Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"lower\": \"1.0.0\",", package1Index.Value.GetContentString());
Assert.Contains("\"upper\": \"1.0.1\"", package1Index.Value.GetContentString());
var package1 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json"));
Assert.NotNull(package1.Key);
var package2 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json"));
Assert.NotNull(package2.Key);
// Check package entries - UnlistedPackage
var package2Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString());
Assert.Contains("\"lower\": \"1.0.0\",", package2Index.Value.GetContentString());
Assert.Contains("\"upper\": \"1.0.0\"", package2Index.Value.GetContentString());
var package3 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json"));
Assert.NotNull(package3.Key);
// Ensure storage does not have the deleted "OtherPackage"
var otherPackageIndex = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json"));
Assert.Null(otherPackageIndex.Key);
}
async Task CreateRegistrationBlobs(Uri catalogIndex, Func<StorageHttpMessageHandler> handlerFunc, CancellationToken cancellationToken)
{
var factory = new MemoryStorageFactory(new Uri(_baseAddress, Registration), _store);
CommitCollector collector = new RegistrationCollector(catalogIndex, factory, handlerFunc)
{
ContentBaseAddress = new Uri("http://tempuri.org/content/"),
UnlistShouldDelete = false
};
await collector.Run(
new MemoryCursor(DateTime.MinValue.ToUniversalTime()),
new MemoryCursor(DateTime.MaxValue.ToUniversalTime()),
cancellationToken);
}