public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken)
{
CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress),
UnlistShouldDelete = unlistShouldDelete
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back, cancellationToken);
}
while (run);
Thread.Sleep(interval * 1000);
}
}
public async Task CreatesFlatContainerAndRespectsDeletes()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var catalogToDnxStorage = new MemoryStorage();
var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name));
var mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", async request => new HttpResponseMessage(HttpStatusCode.OK));
await mockServer.AddStorage(catalogStorage);
mockServer.SetAction("/listedpackage.1.0.0.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip")) });
mockServer.SetAction("/listedpackage.1.0.1.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) });
mockServer.SetAction("/unlistedpackage.1.0.0.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip")) });
mockServer.SetAction("/otherpackage.1.0.0.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) });
// Setup collector
var target = new DnxCatalogCollector(new Uri("http://tempuri.org/index.json"), catalogToDnxStorageFactory, () => mockServer)
{
ContentBaseAddress = new Uri("http://tempuri.org/packages")
};
ReadWriteCursor front = new DurableCursor(catalogToDnxStorage.ResolveUri("cursor.json"), catalogToDnxStorage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
// Act
await target.Run(front, back, CancellationToken.None);
// Assert
Assert.Equal(9, catalogToDnxStorage.Content.Count);
// Ensure storage has cursor.json
var cursorJson = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(cursorJson.Key);
// Check package entries - ListedPackage
var package1Index = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"1.0.0\"", package1Index.Value.GetContentString());
Assert.Contains("\"1.0.1\"", package1Index.Value.GetContentString());
var package1Nuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0/listedpackage.nuspec"));
var package1Nupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"));
Assert.NotNull(package1Nuspec.Key);
Assert.NotNull(package1Nupkg.Key);
var package2Nuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1/listedpackage.nuspec"));
var package2Nupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"));
Assert.NotNull(package2Nuspec.Key);
Assert.NotNull(package2Nupkg.Key);
// Check package entries - UnlistedPackage
var package3Index = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json"));
Assert.NotNull(package3Index.Key);
Assert.Contains("\"1.0.0\"", package3Index.Value.GetContentString());
var package3Nuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0/unlistedpackage.nuspec"));
var package3Nupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg"));
Assert.NotNull(package3Nuspec.Key);
Assert.NotNull(package3Nupkg.Key);
// Ensure storage does not have the deleted "OtherPackage"
var otherPackageIndex = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json"));
var otherPackageNuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/1.0.0/otherpackage.nuspec"));
var otherPackageNupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg"));
Assert.Null(otherPackageIndex.Key);
Assert.Null(otherPackageNuspec.Key);
Assert.Null(otherPackageNupkg.Key);
}
public static void CreateNewCursor(string[] args)
{
IDictionary<string, string> arguments = GetArguments(args, 0);
StorageFactory storageFactory = CreateStorageFactory(arguments, verbose: true);
Storage storage = storageFactory.Create();
DurableCursor cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, GetDefaultValue(arguments));
cursor.Load(CancellationToken.None).Wait();
cursor.Save(CancellationToken.None).Wait();
}
static async Task Test2Async()
{
await MakeTestCatalog();
Func<HttpMessageHandler> handlerFunc = () =>
{
return new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
};
};
TestCollector collectorA = new TestCollector("A", new Uri("http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
DurableCursor cursorA = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorA.json"), storage, MemoryCursor.MinValue);
DurableCursor cursorB = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorB.json"), storage, MemoryCursor.MinValue);
Console.WriteLine("check catalog...");
bool run = false;
do
{
run = false;
run |= await collectorA.Run(cursorA, MemoryCursor.CreateMax(), CancellationToken.None);
run |= await collectorB.Run(cursorB, cursorA, CancellationToken.None);
}
while (run);
Console.WriteLine("ADDING MORE CATALOG");
await MoreTestCatalog();
do
{
run = false;
run |= await collectorA.Run(cursorA, MemoryCursor.CreateMax(), CancellationToken.None);
run |= await collectorB.Run(cursorB, cursorA, CancellationToken.None);
}
while (run);
Console.WriteLine("ALL DONE");
}
static async Task Test1Async()
{
await MakeTestCatalog();
Func<HttpMessageHandler> handlerFunc = () =>
{
return new FileSystemEmulatorHandler
{
BaseAddress = new Uri("http://*****:*****@"c:\data\site",
InnerHandler = new HttpClientHandler()
};
};
TestCollector collector = new TestCollector("Test1", new Uri("http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
DurableCursor front = new DurableCursor(new Uri("http://localhost:8000/cursor/front.json"), storage, MemoryCursor.MinValue);
//DurableCursor back = new DurableCursor(new Uri("http://localhost:8000/cursor/back.json"), storage);
MemoryCursor back = MemoryCursor.CreateMax();
bool didWork = await collector.Run(front, back, CancellationToken.None);
if (!didWork)
{
Console.WriteLine("executed but no work was done");
}
}
private async Task PrepareAsync()
{
_log.WriteLine("Making sure folder {0} exists.", _outputFolder);
if (!Directory.Exists(_outputFolder))
{
Directory.CreateDirectory(_outputFolder);
}
// Create reindex file
_log.WriteLine("Start preparing lightning reindex file...");
var latestCommit = DateTime.MinValue;
int numberOfEntries = 0;
string indexFile = Path.Combine(_outputFolder, "index.txt");
using (var streamWriter = new StreamWriter(indexFile, false))
{
var collectorHttpClient = new CollectorHttpClient();
var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient);
var catalogIndexEntries = await catalogIndexReader.GetEntries();
foreach (var packageRegistrationGroup in catalogIndexEntries
.OrderBy(x => x.CommitTimeStamp)
.ThenBy(x => x.Id)
.ThenBy(x => x.Version)
.GroupBy(x => x.Id))
{
streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key);
var latestCatalogPages = new Dictionary<string, Uri>();
foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup)
{
string key = catalogIndexEntry.Version.ToNormalizedString();
if (latestCatalogPages.ContainsKey(key))
{
latestCatalogPages[key] = catalogIndexEntry.Uri;
}
else
{
latestCatalogPages.Add(key, catalogIndexEntry.Uri);
}
if (latestCommit < catalogIndexEntry.CommitTimeStamp)
{
latestCommit = catalogIndexEntry.CommitTimeStamp;
}
}
foreach (var latestCatalogPage in latestCatalogPages)
{
streamWriter.WriteLine("{0}", latestCatalogPage.Value);
}
}
}
_log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile);
// Write cursor to storage
_log.WriteLine("Start writing new cursor...");
var account = CloudStorageAccount.Parse(_storageAccount);
var storageFactory = (StorageFactory)new AzureStorageFactory(account, _storageContainer);
var storage = storageFactory.Create();
var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit);
cursor.Value = latestCommit;
await cursor.Save(CancellationToken.None);
_log.WriteLine("Finished writing new cursor.");
// Write command files
_log.WriteLine("Start preparing lightning reindex command files...");
string templateFileContents;
using (var templateStreamReader = new StreamReader(_templateFile))
{
templateFileContents = await templateStreamReader.ReadToEndAsync();
}
int batchNumber = 0;
int batchSizeValue = int.Parse(_batchSize);
for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue)
{
var batchEnd = (batchStart + batchSizeValue - 1);
if (batchEnd >= numberOfEntries)
{
batchEnd = numberOfEntries - 1;
}
var cursorCommandFileName = "cursor" + batchNumber + ".cmd";
var cursorTextFileName = "cursor" + batchNumber + ".txt";
using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName)))
{
using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName)))
{
var commandStreamContents = templateFileContents
.Replace("[index]", indexFile)
.Replace("[cursor]", cursorTextFileName)
.Replace("[contentbaseaddress]", _contentBaseAddress)
.Replace("[storageaccount]", _storageAccount)
.Replace("[storagecontainer]", _storageContainer)
.Replace("[storagebaseaddress]", _storageBaseAddress)
.Replace("[compress]", _compress.ToString().ToLowerInvariant());
await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents);
await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd);
}
}
batchNumber++;
}
_log.WriteLine("Finished preparing lightning reindex command files.");
_log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile);
_log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel.");
}
public async Task CreatesRegistrationsAndRespectsDeletes()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var catalogToRegistrationStorage = new MemoryStorage();
var catalogToRegistrationStorageFactory = new TestStorageFactory(name => catalogToRegistrationStorage.WithName(name));
var mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", async request => new HttpResponseMessage(HttpStatusCode.OK));
await mockServer.AddStorage(catalogStorage);
// Setup collector
var target = new RegistrationCollector(new Uri("http://tempuri.org/index.json"), catalogToRegistrationStorageFactory, () => mockServer)
{
ContentBaseAddress = new Uri("http://tempuri.org/packages"),
UnlistShouldDelete = false
};
ReadWriteCursor front = new DurableCursor(catalogToRegistrationStorage.ResolveUri("cursor.json"), catalogToRegistrationStorage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
// Act
await target.Run(front, back, CancellationToken.None);
// Assert
Assert.Equal(6, catalogToRegistrationStorage.Content.Count);
// Ensure storage has cursor.json
var cursorJson = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(cursorJson.Key);
// Check package entries - ListedPackage
var package1Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"lower\": \"1.0.0\",", package1Index.Value.GetContentString());
Assert.Contains("\"upper\": \"1.0.1\"", package1Index.Value.GetContentString());
var package1 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json"));
Assert.NotNull(package1.Key);
var package2 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json"));
Assert.NotNull(package2.Key);
// Check package entries - UnlistedPackage
var package2Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString());
Assert.Contains("\"lower\": \"1.0.0\",", package2Index.Value.GetContentString());
Assert.Contains("\"upper\": \"1.0.0\"", package2Index.Value.GetContentString());
var package3 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json"));
Assert.NotNull(package3.Key);
// Ensure storage does not have the deleted "OtherPackage"
var otherPackageIndex = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json"));
Assert.Null(otherPackageIndex.Key);
}
