public async static Task Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, ILogger log)
{
log.LogInformation($"IndexNuGetPackages started, next run: {myTimer.ScheduleStatus.Next}");
var reader = new CatalogIndexReader(httpClient);
var catalogIndex = await reader.ReadUrl("https://api.nuget.org/v3/catalog0/index.json");
var lastPage = catalogIndex.Items.OrderBy(i => i.CommitTimeStamp).Last();
log.LogInformation($"IndexNuGetPackages complete, last page {lastPage.Id}, commited at: {lastPage.CommitTimeStamp}");
}
public async Task ParseCatalogIndex()
{
var cursor = DateTime.UtcNow - TimeSpan.FromDays(1);
var reader = new CatalogIndexReader(new HttpClient());
var catalogIndex = await reader.ReadUrl("https://api.nuget.org/v3/catalog0/index.json");
Console.Out.WriteLine($"All pages since: {cursor}");
foreach (var page in catalogIndex.Items.Where(i => i.CommitTimeStamp > cursor).OrderByDescending(i => i.CommitTimeStamp))
{
Console.Out.WriteLine($"{page.CommitTimeStamp} - {page.Id} ({page.Count})");
}
}
public async Task GetEntries()
{
// Arrange
var indexUri = "http://tempuri.org/index.json";
var responses = new Dictionary <string, string>()
{
{ "http://tempuri.org/index.json", TestCatalogEntries.TestCatalogStorageWithThreePackagesIndex },
{ "http://tempuri.org/page0.json", TestCatalogEntries.TestCatalogStorageWithThreePackagesPage },
};
var reader = new CatalogIndexReader(
new Uri(indexUri),
new CollectorHttpClient(new InMemoryHttpHandler(responses)),
new Mock <ITelemetryService>().Object);
// Act
var entries = await reader.GetEntries();
// Assert
var entryList = entries.ToList();
Assert.Equal(3, entryList.Count);
Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", entryList[0].Uri.ToString());
Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", entryList[1].Uri.ToString());
Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", entryList[2].Uri.ToString());
Assert.Equal("2015-10-12T10:08:55.3335317", entryList[0].CommitTimeStamp.ToString("O"));
Assert.Equal("2015-10-12T10:08:54.1506742", entryList[1].CommitTimeStamp.ToString("O"));
Assert.Equal("2015-10-12T10:08:54.1506742", entryList[2].CommitTimeStamp.ToString("O"));
Assert.Equal("8a9e7694-73d4-4775-9b7a-20aa59b9773e", entryList[0].CommitId);
Assert.Equal("9a37734f-1960-4c07-8934-c8bc797e35c1", entryList[1].CommitId);
Assert.Equal("9a37734f-1960-4c07-8934-c8bc797e35c1", entryList[2].CommitId);
Assert.Equal("ListedPackage", entryList[0].Id);
Assert.Equal("ListedPackage", entryList[1].Id);
Assert.Equal("UnlistedPackage", entryList[2].Id);
Assert.Equal(new NuGetVersion("1.0.1"), entryList[0].Version);
Assert.Equal(new NuGetVersion("1.0.0"), entryList[1].Version);
Assert.Equal(new NuGetVersion("1.0.0"), entryList[2].Version);
Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[0].Types);
Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[1].Types);
Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[2].Types);
Assert.Same(entryList[0].Id, entryList[1].Id);
}
public static HashSet <PackageEntry> GetCatalogPackages(string catalogRootUrl, string storageConnectionString, DateTime startCommitTimeStamp, DateTime endCommitTimeStamp)
{
CollectorHttpClient client = new CollectorHttpClient();
CloudStorageAccount csa = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = csa.CreateCloudBlobClient();
Uri catalogIndex = new Uri(catalogRootUrl);
CatalogIndexReader reader = new CatalogIndexReader(catalogIndex, client);
//TBD Update CatalogIndexReader to return packages based on commit time stamp.Right now it returns all packages.
var task = reader.GetEntries();
task.Wait();
List <CatalogIndexEntry> entries = task.Result.ToList();
entries = entries.Where(e => e.CommitTimeStamp >= startCommitTimeStamp && e.CommitTimeStamp <= endCommitTimeStamp).ToList();
var catalogPackages = new HashSet <PackageEntry>(entries.Select(e => new PackageEntry(e.Id, e.Version.ToNormalizedString())), PackageEntry.Comparer);
return(catalogPackages);
}
public static async Task Test0Async(string[] args)
{
const string V2FeedCountQuery = "/Packages/$count";
Console.WriteLine("Simple count test for distinct package ids and version between v2 feed and catalog");
if (args.Length != 2)
{
Console.WriteLine("Please enter only 2 arguments. First v2gallery feed url, and second catalog index.json url");
return;
}
else
{
string v2FeedUrl = args[0].TrimEnd('/');
string v2FeedCountUrl = v2FeedUrl.TrimEnd('/') + V2FeedCountQuery;
int v2FeedCount = 0;
using (var client = new HttpClient())
{
var response = await client.GetAsync(v2FeedCountUrl);
string v2FeedCountString = await response.Content.ReadAsStringAsync();
v2FeedCount = Int32.Parse(v2FeedCountString);
}
string catalog = args[1];
Uri catalogIndex = new Uri(catalog);
CatalogIndexReader reader = new CatalogIndexReader(catalogIndex);
var task = reader.GetEntries();
task.Wait();
var entries = task.Result;
Console.WriteLine("Total packages count from catalog is " + entries.Count());
var distinctCatalogPackages = entries.Distinct(new CatalogIndexEntryIdVersionComparer());
int v3CatalogPackagesCount = distinctCatalogPackages.Count();
Console.WriteLine("Distinct packages count from catalog is " + v3CatalogPackagesCount);
Console.WriteLine("Distinct packages count from " + v2FeedUrl + " is " + v2FeedCount);
Console.WriteLine("Current difference between v2Feed and v3 catalog is " + (v2FeedCount - v3CatalogPackagesCount));
}
}
private async Task PrepareAsync()
{
_log.WriteLine("Making sure folder {0} exists.", _outputFolder);
if (!Directory.Exists(_outputFolder))
{
Directory.CreateDirectory(_outputFolder);
}
// Create reindex file
_log.WriteLine("Start preparing lightning reindex file...");
var latestCommit = DateTime.MinValue;
int numberOfEntries = 0;
string indexFile = Path.Combine(_outputFolder, "index.txt");
string optionalArgumentsTemplate = "optionalArguments";
using (var streamWriter = new StreamWriter(indexFile, false))
{
var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose);
var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory());
var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService);
var catalogIndexEntries = await catalogIndexReader.GetEntries();
foreach (var packageRegistrationGroup in catalogIndexEntries
.OrderBy(x => x.CommitTimeStamp)
.ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase)
.ThenBy(x => x.Version)
.GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase))
{
streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key);
var latestCatalogPages = new Dictionary <string, Uri>();
foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup)
{
string key = catalogIndexEntry.Version.ToNormalizedString();
if (latestCatalogPages.ContainsKey(key))
{
latestCatalogPages[key] = catalogIndexEntry.Uri;
}
else
{
latestCatalogPages.Add(key, catalogIndexEntry.Uri);
}
if (latestCommit < catalogIndexEntry.CommitTimeStamp)
{
latestCommit = catalogIndexEntry.CommitTimeStamp;
}
}
foreach (var latestCatalogPage in latestCatalogPages)
{
streamWriter.WriteLine("{0}", latestCatalogPage.Value);
}
}
}
_log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile);
// Write cursor to storage
_log.WriteLine("Start writing new cursor...");
var storage = _storageFactories.LegacyStorageFactory.Create();
var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit)
{
Value = latestCommit
};
await cursor.SaveAsync(CancellationToken.None);
_log.WriteLine("Finished writing new cursor.");
// Ensure the SemVer 2.0.0 storage containers is created, if applicable. The gzipped storage account is
// created above when we write the cursor.
_storageFactories.SemVer2StorageFactory?.Create();
// Write command files
_log.WriteLine("Start preparing lightning reindex command files...");
string templateFileContents;
using (var templateStreamReader = new StreamReader(_templateFile))
{
templateFileContents = await templateStreamReader.ReadToEndAsync();
}
int batchNumber = 0;
int batchSizeValue = int.Parse(_batchSize);
for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue)
{
var batchEnd = (batchStart + batchSizeValue - 1);
if (batchEnd >= numberOfEntries)
{
batchEnd = numberOfEntries - 1;
}
var cursorCommandFileName = "cursor" + batchNumber + ".cmd";
var cursorTextFileName = "cursor" + batchNumber + ".txt";
using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName)))
using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName)))
{
var commandStreamContents = templateFileContents;
var replacements = _arguments
.Concat(new[]
{
new KeyValuePair <string, string>("indexFile", indexFile),
new KeyValuePair <string, string>("cursorFile", cursorTextFileName)
});
foreach (var replacement in replacements)
{
commandStreamContents = commandStreamContents
.Replace($"[{replacement.Key}]", replacement.Value);
}
//the not required arguments need to be added only if they were passed in
//they cannot be hardcoded in the template
var optionalArguments = new StringBuilder();
AppendOptionalArgument(optionalArguments, Arguments.ContentIsFlatContainer);
AppendOptionalArgument(optionalArguments, Arguments.FlatContainerName);
AppendOptionalArgument(optionalArguments, Arguments.StorageSuffix);
AppendOptionalArgument(optionalArguments, Arguments.AllIconsInFlatContainer);
AppendOptionalArgument(optionalArguments, Arguments.Driver);
AppendOptionalArgument(optionalArguments, Arguments.Verbose);
commandStreamContents = commandStreamContents
.Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString());
await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents);
await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd);
}
batchNumber++;
}
_log.WriteLine("Finished preparing lightning reindex command files.");
_log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile);
_log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel.");
}
public override async Task RunCore()
{
int nextMasterRegId = 0;
DateTime position = Cursor.Position;
JToken nextMasterRegIdToken = null;
if (Cursor.Metadata.TryGetValue("nextMasterRegId", out nextMasterRegIdToken))
{
nextMasterRegId = nextMasterRegIdToken.ToObject <int>();
}
// Get the catalog index
Uri catalogIndexUri = new Uri(Config.GetProperty("CatalogIndex"));
Log("Reading index entries");
var indexReader = new CatalogIndexReader(catalogIndexUri);
var indexEntries = await indexReader.GetRolledUpEntries();
var context = indexReader.GetContext();
Log("Finding new or editted entries");
var changedEntries = new HashSet <string>(indexEntries.Where(e => e.CommitTimeStamp.CompareTo(position) > 0)
.Select(e => e.Id.ToLowerInvariant()), StringComparer.OrdinalIgnoreCase);
DateTime newPosition = indexEntries.Select(e => e.CommitTimeStamp).OrderByDescending(e => e).FirstOrDefault();
ConcurrentDictionary <string, ConcurrentBag <Uri> > batches = new ConcurrentDictionary <string, ConcurrentBag <Uri> >(StringComparer.OrdinalIgnoreCase);
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Parallel.ForEach(indexEntries, options, entry =>
{
if (changedEntries.Contains(entry.Id))
{
batches.AddOrUpdate(entry.Id, new ConcurrentBag <Uri>()
{
entry.Uri
}, (id, uris) =>
{
uris.Add(entry.Uri);
return(uris);
});
}
});
Uri contentBaseAddress = new Uri(Config.GetProperty("ContentBaseAddress"));
if (batches.Count > 0)
{
Log("Building registrations from: " + position.ToString("O"));
options.MaxDegreeOfParallelism = 4;
for (int i = 0; i < 3 && batches.Count > 0; i++)
{
if (i != 0)
{
options.MaxDegreeOfParallelism = 1;
Console.WriteLine("Single batch run.");
}
var ids = batches.Keys.OrderBy(s => s).ToArray();
Stopwatch buildTimer = new Stopwatch();
buildTimer.Start();
int startingCount = ids.Length;
Parallel.ForEach(ids, options, id =>
{
try
{
BatchRegistrationCollector regCollector = new BatchRegistrationCollector(null, _factory);
regCollector.ContentBaseAddress = contentBaseAddress;
Stopwatch timer = new Stopwatch();
timer.Start();
var uriGroup = batches[id].ToArray();
regCollector.ProcessGraphs(_client, id, uriGroup, context).Wait();
int rem = batches.Count;
timer.Stop();
string log = String.Format("Completed: {0} Duration: {1} Uris: {2} Remaining Ids: {3} Loop: {4}", id, timer.Elapsed, uriGroup.Length, rem, i);
Console.WriteLine(log);
// stats
double perPackage = buildTimer.Elapsed.TotalSeconds / (double)(startingCount - rem + 1);
DateTime finish = DateTime.Now.AddSeconds(Math.Ceiling(perPackage * rem));
Console.WriteLine("Estimated Finish: " + finish.ToString("O"));
ConcurrentBag <Uri> vals;
if (!batches.TryRemove(id, out vals))
{
Console.WriteLine("Unable to remove!");
}
}
catch (Exception ex)
{
LogError("Registration failed: " + id + " " + ex.ToString());
}
});
}
// mark this with the last commit we included
Cursor.Position = newPosition;
await Cursor.Save();
Log("Finished registrations: " + newPosition.ToString("O"));
}
}
private async Task PrepareAsync()
{
_log.WriteLine("Making sure folder {0} exists.", _outputFolder);
if (!Directory.Exists(_outputFolder))
{
Directory.CreateDirectory(_outputFolder);
}
// Create reindex file
_log.WriteLine("Start preparing lightning reindex file...");
var latestCommit = DateTime.MinValue;
int numberOfEntries = 0;
string indexFile = Path.Combine(_outputFolder, "index.txt");
string storageCredentialArgumentsTemplate = "storageCredentialArguments";
string optionalArgumentsTemplate = "optionalArguments";
using (var streamWriter = new StreamWriter(indexFile, false))
{
var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose);
var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory());
var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService);
var catalogIndexEntries = await catalogIndexReader.GetEntries();
foreach (var packageRegistrationGroup in catalogIndexEntries
.OrderBy(x => x.CommitTimeStamp)
.ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase)
.ThenBy(x => x.Version)
.GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase))
{
streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key);
var latestCatalogPages = new Dictionary <string, Uri>();
foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup)
{
string key = catalogIndexEntry.Version.ToNormalizedString();
if (latestCatalogPages.ContainsKey(key))
{
latestCatalogPages[key] = catalogIndexEntry.Uri;
}
else
{
latestCatalogPages.Add(key, catalogIndexEntry.Uri);
}
if (latestCommit < catalogIndexEntry.CommitTimeStamp)
{
latestCommit = catalogIndexEntry.CommitTimeStamp;
}
}
foreach (var latestCatalogPage in latestCatalogPages)
{
streamWriter.WriteLine("{0}", latestCatalogPage.Value);
}
}
}
_log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile);
// Create the containers
_log.WriteLine("Creating the containers...");
var container = GetAutofacContainer();
var blobClient = container.Resolve <ICloudBlobClient>();
var config = container.Resolve <IOptionsSnapshot <Catalog2RegistrationConfiguration> >().Value;
foreach (var name in new[] { config.LegacyStorageContainer, config.GzippedStorageContainer, config.SemVer2StorageContainer })
{
var reference = blobClient.GetContainerReference(name);
var permissions = new BlobContainerPermissions {
PublicAccess = BlobContainerPublicAccessType.Blob
};
await reference.CreateIfNotExistAsync(permissions);
}
// Write cursor to storage
_log.WriteLine("Start writing new cursor...");
var storageFactory = container.ResolveKeyed <IStorageFactory>(DependencyInjectionExtensions.CursorBindingKey);
var storage = storageFactory.Create();
var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit)
{
Value = latestCommit
};
await cursor.SaveAsync(CancellationToken.None);
_log.WriteLine("Finished writing new cursor.");
// Write command files
_log.WriteLine("Start preparing lightning reindex command files...");
string templateFileContents;
using (var templateStreamReader = new StreamReader(_templateFile))
{
templateFileContents = await templateStreamReader.ReadToEndAsync();
}
int batchNumber = 0;
int batchSizeValue = int.Parse(_batchSize);
for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue)
{
var batchEnd = (batchStart + batchSizeValue - 1);
if (batchEnd >= numberOfEntries)
{
batchEnd = numberOfEntries - 1;
}
var cursorCommandFileName = "cursor" + batchNumber + ".cmd";
var cursorTextFileName = "cursor" + batchNumber + ".txt";
using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName)))
using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName)))
{
var commandStreamContents = templateFileContents;
var replacements = _arguments
.Concat(new[]
{
new KeyValuePair <string, string>("indexFile", indexFile),
new KeyValuePair <string, string>("cursorFile", cursorTextFileName)
});
foreach (var replacement in replacements)
{
commandStreamContents = commandStreamContents
.Replace($"[{replacement.Key}]", replacement.Value);
}
// Since we only need to set the storage key or the storage sas token, only one will be added to the template.
var storageCredentialArguments = new StringBuilder();
AddStorageCredentialArgument(storageCredentialArguments, Arguments.StorageSasValue, Arguments.StorageKeyValue);
AddStorageCredentialArgument(storageCredentialArguments, Arguments.CompressedStorageSasValue, Arguments.CompressedStorageKeyValue);
AddStorageCredentialArgument(storageCredentialArguments, Arguments.SemVer2StorageSasValue, Arguments.SemVer2StorageKeyValue);
commandStreamContents = commandStreamContents
.Replace($"[{storageCredentialArgumentsTemplate}]", storageCredentialArguments.ToString());
//the not required arguments need to be added only if they were passed in
//they cannot be hardcoded in the template
var optionalArguments = new StringBuilder();
AppendArgument(optionalArguments, Arguments.FlatContainerName);
AppendArgument(optionalArguments, Arguments.StorageSuffix);
AppendArgument(optionalArguments, Arguments.Verbose);
commandStreamContents = commandStreamContents
.Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString());
await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents);
await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd);
}
batchNumber++;
}
_log.WriteLine("Finished preparing lightning reindex command files.");
_log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile);
_log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel.");
}
