public static async Task BuildCatalogAsync(string path, Storage storage, IEnumerable<string> ids)
{
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 600);
const int BatchSize = 200;
int i = 0;
int commitCount = 0;
IEnumerable<string> files = GetFileList(path, ids);
Console.WriteLine("initial build files = {0}", files.Count());
foreach (string fullName in files)
{
writer.Add(new NuspecPackageCatalogItem(fullName));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount++);
}
public static async Task Test0Async()
{
//string nuspecs = @"c:\data\nuget\nuspecs";
string nuspecs = @"c:\data\nuget\nuspecs";
//Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
//Storage storage = new FileStorage("http://*****:*****@"c:\data\site\dotnetrdf");
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\ordered");
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 15);
const int BatchSize = 10;
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
//foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*.xml"))
//foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("dotnetrdf.*.xml"))
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("entityframework.*.xml"))
{
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
if (++i % BatchSize == 0)
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount++);
}
}
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount++);
}
public static async Task<Uri> AddToCatalog(CatalogItem catalogItem, string connectionString, string container, string catalogBaseAddress, CancellationToken cancellationToken)
{
StorageWriteLock writeLock = new StorageWriteLock(connectionString, container);
await writeLock.AquireAsync();
Uri rootUri = null;
Exception exception = null;
try
{
Storage storage = CreateStorage(connectionString, container, catalogBaseAddress);
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage);
writer.Add(catalogItem);
await writer.Commit(null, cancellationToken);
rootUri = writer.RootUri;
}
catch (Exception e)
{
exception = e;
}
await writeLock.ReleaseAsync();
if (exception != null)
{
throw exception;
}
return rootUri;
}
public static async Task Test1Async()
{
string nuspecs = @"c:\data\nuget\nuspecs";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\full");
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 20);
int total = 0;
//int[] commitSize = { 50, 40, 25, 50, 10, 30, 40, 5, 400, 30, 10, 20, 40, 50, 90, 70, 50, 50, 50, 50, 60, 70 };
int[] commitSize = {
20, 20, 20, 20, 20,
20, 20, 20, 20, 20,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200,
//200, 200, 200, 200, 200
};
int i = 0;
int commitCount = 0;
DirectoryInfo directoryInfo = new DirectoryInfo(nuspecs);
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("dotnetrdf.*.xml"))
{
if (commitCount == commitSize.Length)
{
break;
}
writer.Add(new NuspecPackageCatalogItem(fileInfo.FullName));
total++;
if (++i == commitSize[commitCount])
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
Console.WriteLine("commit number {0}", commitCount);
commitCount++;
i = 0;
}
}
if (i > 0)
{
await writer.Commit(DateTime.UtcNow, null, CancellationToken.None);
}
Console.WriteLine("total: {0}", total);
}
public async Task Load(string path, CancellationToken cancellationToken)
{
var directoryInfo = new DirectoryInfo(path);
foreach (var fileInfo in directoryInfo.EnumerateFiles("*.nuspec"))
{
AddNuspec(fileInfo);
}
// Catalog
var factory = new MemoryStorageFactory(new Uri(_baseAddress, Catalog), _store);
var storage = factory.Create();
var catalog = new AppendOnlyCatalogWriter(storage);
foreach (var registration in Data.Values)
{
foreach (var package in registration.Values)
{
var metadata = new NupkgMetadata
{
Nuspec = package
};
catalog.Add(new PackageCatalogItem(metadata));
}
}
await catalog.Commit(null, cancellationToken);
Uri catalogIndex = new Uri(storage.BaseAddress, "index.json");
Func<StorageHttpMessageHandler> handlerFunc = () => { return new StorageHttpMessageHandler(storage); };
await CreateRegistrationBlobs(catalogIndex, handlerFunc, cancellationToken);
await CreateFlatContainer(catalogIndex, handlerFunc, cancellationToken);
await CreateLuceneIndex(catalogIndex, handlerFunc, cancellationToken);
await CreateIndex(cancellationToken);
}
static async Task<DateTime> DownloadMetadata2Catalog(HttpClient client, SortedList<DateTime, IList<Tuple<Uri, FeedDetails>>> packages, Storage storage, DateTime lastCreated, DateTime lastEdited, bool? createdPackages, CancellationToken cancellationToken)
{
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
DateTime lastDate = createdPackages.HasValue ? (createdPackages.Value ? lastCreated : lastEdited) : DateTime.MinValue;
if (packages == null || packages.Count == 0)
{
return lastDate;
}
foreach (KeyValuePair<DateTime, IList<Tuple<Uri, FeedDetails>>> entry in packages)
{
foreach (Tuple<Uri, FeedDetails> packageItem in entry.Value)
{
Uri uri = packageItem.Item1;
FeedDetails details = packageItem.Item2;
HttpResponseMessage response = await client.GetAsync(uri,cancellationToken);
if (response.IsSuccessStatusCode)
{
using (Stream stream = await response.Content.ReadAsStreamAsync())
{
CatalogItem item = Utils.CreateCatalogItem(stream, entry.Key, null, uri.ToString(), details.CreatedDate, details.LastEditedDate, details.PublishedDate, details.LicenseNames, details.LicenseReportUrl);
if (item != null)
{
writer.Add(item);
Trace.TraceInformation("Add: {0}", uri);
}
else
{
Trace.TraceWarning("Unable to extract metadata from: {0}", uri);
}
}
}
else
{
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
// the feed is out of sync with the actual package storage - if we don't have the package there is nothing to be done we might as well move onto the next package
Trace.TraceWarning(string.Format("Unable to download: {0} http status: {1}", uri, response.StatusCode));
}
else
{
// this should trigger a restart - of this program - and not more the cursor forward
Trace.TraceError(string.Format("Unable to download: {0} http status: {1}", uri, response.StatusCode));
throw new Exception(string.Format("Unable to download: {0} http status: {1}", uri, response.StatusCode));
}
}
}
lastDate = entry.Key;
}
if (createdPackages.HasValue)
{
lastCreated = createdPackages.Value ? lastDate : lastCreated;
lastEdited = !createdPackages.Value ? lastDate : lastEdited;
}
IGraph commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, lastCreated, lastEdited);
await writer.Commit(commitMetadata, cancellationToken);
Trace.TraceInformation("COMMIT");
return lastDate;
}
public static async Task Test3Async()
{
System.Net.ServicePointManager.DefaultConnectionLimit = 1024;
IDictionary<string, string> packageHashLookup = LoadPackageHashLookup();
HashSet<string> packageExceptionLookup = LoadPackageExceptionLookup();
string nupkgs = @"c:\data\nuget\gallery\";
Storage storage = new FileStorage("http://*****:*****@"c:\data\site\ordered");
//StorageCredentials credentials = new StorageCredentials("", "");
//CloudStorageAccount account = new CloudStorageAccount(credentials, true);
//string storageContainer = "test1";
//string storagePath = "";
//string storageBaseAddress = "http://nugetjohtaylo.blob.core.windows.net/test1";
//StorageFactory storageFactory = new AzureStorageFactory(account, storageContainer, storagePath, new Uri(storageBaseAddress)) { Verbose = true };
//Storage storage = storageFactory.Create();
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
const int BatchSize = 64;
int commitCount = 0;
IDictionary<string, DateTime> packageCreated = LoadPackageCreatedLookup();
DateTime lastCreated = (await PackageCatalog.ReadCommitMetadata(writer, CancellationToken.None)).Item1 ?? DateTime.MinValue;
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
// filter by lastCreated here
Queue<KeyValuePair<string, DateTime>> packageCreatedQueue = new Queue<KeyValuePair<string, DateTime>>(packageCreated.Where(p => p.Value > lastCreated && !packageExceptionLookup.Contains(p.Key)).OrderBy(p => p.Value));
int completed = 0;
Stopwatch runtime = new Stopwatch();
runtime.Start();
Task commitTask = null;
var context = writer.Context;
Uri rootUri = writer.RootUri;
while (packageCreatedQueue.Count > 0)
{
List<KeyValuePair<string, DateTime>> batch = new List<KeyValuePair<string, DateTime>>();
ConcurrentBag<CatalogItem> batchItems = new ConcurrentBag<CatalogItem>();
while (batch.Count < BatchSize && packageCreatedQueue.Count > 0)
{
completed++;
var packagePair = packageCreatedQueue.Dequeue();
lastCreated = packagePair.Value;
batch.Add(packagePair);
}
var commitTime = DateTime.UtcNow;
Parallel.ForEach(batch, options, entry =>
{
FileInfo fileInfo = new FileInfo(nupkgs + entry.Key);
if (fileInfo.Exists)
{
using (Stream stream = new FileStream(fileInfo.FullName, FileMode.Open))
{
string packageHash = null;
packageHashLookup.TryGetValue(fileInfo.Name, out packageHash);
CatalogItem item = Utils.CreateCatalogItem(stream, entry.Value, packageHash, fileInfo.FullName);
batchItems.Add(item);
}
}
});
if (commitTask != null)
{
commitTask.Wait();
}
foreach (var item in batchItems)
{
writer.Add(item);
}
commitTask = Task.Run(async () => await writer.Commit(commitTime, PackageCatalog.CreateCommitMetadata(writer.RootUri, lastCreated, null), CancellationToken.None));
// stats
double perPackage = runtime.Elapsed.TotalSeconds / (double)completed;
DateTime finish = DateTime.Now.AddSeconds(perPackage * packageCreatedQueue.Count);
Console.WriteLine("commit number {0} Completed: {1} Remaining: {2} Estimated Finish: {3}",
commitCount++,
completed,
packageCreatedQueue.Count,
finish.ToString("O"));
}
// wait for the final commit
if (commitTask != null)
{
commitTask.Wait();
}
Console.WriteLine("Finished in: " + runtime.Elapsed);
}
protected override void RunCore(CancellationToken cancellationToken)
{
Config.Catalog.LocalFolder.Create();
int total = _nupkgs.Count;
Log("Processing " + total + " nupkgs");
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Task commitTask = null;
using (var writer = new AppendOnlyCatalogWriter(Config.Catalog.Storage))
{
while (_nupkgs.Count > 0)
{
Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(_batchSize);
// create the batch
while (currentBatch.Count < _batchSize && _nupkgs.Count > 0)
{
string file = _nupkgs.Dequeue();
if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase))
{
currentBatch.Enqueue(new NupkgCatalogItem(file));
}
else
{
currentBatch.Enqueue(new NuspecPackageCatalogItem(file));
}
}
// process the nupkgs and nuspec files in parallel
Parallel.ForEach(currentBatch, options, nupkg =>
{
nupkg.Load();
});
// wait for the previous commit to finish before adding more
if (commitTask != null)
{
commitTask.Wait();
}
// add everything from the queue
foreach (PackageCatalogItem item in currentBatch)
{
writer.Add(item);
}
// commit
commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow, null, cancellationToken));
ProgressUpdate(total - _nupkgs.Count, total);
}
// wait for the final commit
if (commitTask != null)
{
commitTask.Wait();
}
}
}
static async Task<Uri> AddToCatalog(Stream nupkgStream, CancellationToken cancellationToken)
{
string storagePrimary = _configurationService.Get("Storage.Primary");
CloudStorageAccount account = CloudStorageAccount.Parse(storagePrimary);
Storage storage = new AzureStorage(account, "catalog");
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage);
writer.Add(Utils.CreateCatalogItem(nupkgStream, null, null, ""));
await writer.Commit(null, cancellationToken);
return writer.RootUri;
}
static async Task MoreTestCatalog()
{
string baseAddress = "http://*****:*****@"c:\data\site\cursor";
Storage storage = new FileStorage(baseAddress, path);
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
writer.Add(new TestCatalogItem(8));
await writer.Commit(new DateTime(2014, 1, 11, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(9));
await writer.Commit(new DateTime(2014, 1, 13, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(10));
await writer.Commit(new DateTime(2014, 1, 14, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(11));
await writer.Commit(new DateTime(2014, 1, 15, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(12));
await writer.Commit(new DateTime(2014, 1, 17, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(13));
await writer.Commit(new DateTime(2014, 1, 18, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(14));
await writer.Commit(new DateTime(2014, 1, 20, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
Console.WriteLine("test catalog created");
}
static async Task MakeTestCatalog()
{
string baseAddress = "http://*****:*****@"c:\data\site\cursor";
DirectoryInfo folder = new DirectoryInfo(path);
if (folder.Exists)
{
Console.WriteLine("test catalog already created");
return;
}
Storage storage = new FileStorage(baseAddress, path);
AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(storage, 550);
writer.Add(new TestCatalogItem(1));
await writer.Commit(new DateTime(2014, 1, 1, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(2));
await writer.Commit(new DateTime(2014, 1, 3, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);
writer.Add(new TestCatalogItem(3));
await writer.Commit(new DateTime(2014, 1, 4, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);
writer.Add(new TestCatalogItem(4));
await writer.Commit(new DateTime(2014, 1, 5, 0, 0, 0, DateTimeKind.Utc),null, CancellationToken.None);
writer.Add(new TestCatalogItem(5));
await writer.Commit(new DateTime(2014, 1, 7, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
writer.Add(new TestCatalogItem(6));
await writer.Commit(new DateTime(2014, 1, 8, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None); ;
writer.Add(new TestCatalogItem(7));
await writer.Commit(new DateTime(2014, 1, 10, 0, 0, 0, DateTimeKind.Utc), null, CancellationToken.None);
Console.WriteLine("test catalog created");
}
private static async Task<DateTime> Deletes2Catalog(SortedList<DateTime, IList<PackageIdentity>> packages, Storage storage, DateTime lastCreated, DateTime lastEdited, DateTime lastDeleted, CancellationToken cancellationToken)
{
var writer = new AppendOnlyCatalogWriter(storage, maxPageSize: 550);
if (packages == null || packages.Count == 0)
{
return lastDeleted;
}
foreach (var entry in packages)
{
foreach (var packageIdentity in entry.Value)
{
var catalogItem = new DeleteCatalogItem(packageIdentity.Id, packageIdentity.Version, entry.Key);
writer.Add(catalogItem);
Trace.TraceInformation("Delete: {0} {1}", packageIdentity.Id, packageIdentity.Version);
}
lastDeleted = entry.Key;
}
var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted));
await writer.Commit(commitMetadata, cancellationToken);
Trace.TraceInformation("COMMIT");
return lastDeleted;
}
public static async Task<DateTime> DownloadMetadata2Catalog(HttpClient client, SortedList<DateTime, IList<PackageDetails>> packages, Storage storage, DateTime lastCreated, DateTime lastEdited, DateTime lastDeleted, bool? createdPackages, CancellationToken cancellationToken, ILogger logger)
{
var writer = new AppendOnlyCatalogWriter(storage, maxPageSize: 550);
var lastDate = DetermineLastDate(lastCreated, lastEdited, createdPackages);
if (packages == null || packages.Count == 0)
{
return lastDate;
}
foreach (var entry in packages)
{
foreach (var packageItem in entry.Value)
{
// When downloading the package binary, add a query string parameter
// that corresponds to the operation's timestamp.
// This query string will ensure the package is not cached
// (e.g. on the CDN) and returns the "latest and greatest" package metadata.
var packageUri = Utilities.GetNugetCacheBustingUri(packageItem.ContentUri, entry.Key.ToString("O"));
var response = await client.GetAsync(packageUri, cancellationToken);
if (response.IsSuccessStatusCode)
{
using (var stream = await response.Content.ReadAsStreamAsync())
{
CatalogItem item = Utils.CreateCatalogItem(
packageItem.ContentUri.ToString(),
stream,
packageItem.CreatedDate,
packageItem.LastEditedDate,
packageItem.PublishedDate);
if (item != null)
{
writer.Add(item);
logger?.LogInformation("Add metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
}
else
{
logger?.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
}
}
}
else
{
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
// the feed is out of sync with the actual package storage - if we don't have the package there is nothing to be done we might as well move onto the next package
logger?.LogWarning("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode);
}
else
{
// this should trigger a restart - of this program - and not move the cursor forward
logger?.LogError("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode);
throw new Exception(
$"Unable to download: {packageItem.ContentUri} http status: {response.StatusCode}");
}
}
}
lastDate = entry.Key;
}
if (createdPackages.HasValue)
{
lastCreated = createdPackages.Value ? lastDate : lastCreated;
lastEdited = !createdPackages.Value ? lastDate : lastEdited;
}
var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted));
await writer.Commit(commitMetadata, cancellationToken);
logger?.LogInformation("COMMIT metadata to catalog.");
return lastDate;
}
public override async Task RunCore(CancellationToken cancellationToken)
{
TimeSpan hold = TimeSpan.FromMinutes(90);
int cantonCommitId = 0;
JToken cantonCommitIdToken = null;
if (Cursor.Metadata.TryGetValue("cantonCommitId", out cantonCommitIdToken))
{
cantonCommitId = cantonCommitIdToken.ToObject<int>();
}
Queue<JObject> orderedMessages = new Queue<JObject>();
var blobClient = Account.CreateCloudBlobClient();
Stopwatch giveup = new Stopwatch();
giveup.Start();
Dictionary<int, string> unQueuedMessages = new Dictionary<int, string>();
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
ConcurrentBag<CantonCatalogItem> batchItems = new ConcurrentBag<CantonCatalogItem>();
Task commitTask = null;
try
{
using (AppendOnlyCatalogWriter writer = new AppendOnlyCatalogWriter(Storage, 600))
{
try
{
// get everything in the queue
Log("Getting work");
var newWork = GetWork();
Log("Done getting work");
// everything must run in canton commit order!
while (_run && (newWork.Count > 0 || unQueuedMessages.Count > 0 || orderedMessages.Count > 0))
{
Log(String.Format("New: {0} Waiting: {1} Ordered: {2}", newWork.Count, unQueuedMessages.Count, orderedMessages.Count));
int[] newIds = newWork.Keys.ToArray();
foreach (int curId in newIds)
{
string s = newWork[curId];
JObject json = JObject.Parse(s);
int id = json["cantonCommitId"].ToObject<int>();
if (id >= cantonCommitId && !unQueuedMessages.ContainsKey(id))
{
unQueuedMessages.Add(id, s);
}
else
{
LogError("Ignoring old cantonCommitId: " + id + " We are on: " + cantonCommitId);
}
}
// load up the next 4096 work items we need
while (unQueuedMessages.ContainsKey(cantonCommitId) && orderedMessages.Count < 4096)
{
JObject json = JObject.Parse(unQueuedMessages[cantonCommitId]);
orderedMessages.Enqueue(json);
unQueuedMessages.Remove(cantonCommitId);
cantonCommitId++;
giveup.Restart();
}
// just take up to the batch size
Queue<JObject> currentBatch = new Queue<JObject>();
// get up to the batchsize
while (currentBatch.Count < BatchSize && orderedMessages.Count > 0 && (currentBatch.Count + batchItems.Count) < BatchSize)
{
currentBatch.Enqueue(orderedMessages.Dequeue());
}
if (currentBatch.Count > 0)
{
Stopwatch timer = new Stopwatch();
timer.Start();
int before = batchItems.Count;
Parallel.ForEach(currentBatch, options, workJson =>
{
try
{
int curId = workJson["cantonCommitId"].ToObject<int>();
string resourceUriString = workJson["uri"].ToString();
if (!StringComparer.OrdinalIgnoreCase.Equals(resourceUriString, "https://failed/"))
{
Uri resourceUri = new Uri(resourceUriString);
// the page is loaded from storage in the background
CantonCatalogItem item = new CantonCatalogItem(Account, resourceUri, curId);
// download the graph, this is a blocking call
item.LoadGraph();
// add the item to the batch to be committed in order later
batchItems.Add(item);
}
else
{
Log("Skipping failed page: " + curId);
}
}
catch (Exception ex)
{
LogError("Unable to create page: " + ex.ToString());
}
});
timer.Stop();
Console.WriteLine(String.Format(CultureInfo.InvariantCulture, "Loaded {0} pre-built pages in {1}", (batchItems.Count - before), timer.Elapsed));
}
// commit the items
if (batchItems.Count >= BatchSize)
{
CantonCatalogItem[] curItems = batchItems.ToArray();
batchItems = new ConcurrentBag<CantonCatalogItem>();
if (commitTask != null)
{
await commitTask;
}
// make certain this ALL runs on another thread
commitTask = Task.Run(async () => await Commit(writer, curItems));
}
// get the next work item
if (_run)
{
newWork = GetWork();
}
else
{
newWork = new Dictionary<int, string>();
}
if (newWork.Count < 1 && _run)
{
// just give up after 5 minutes
// TODO: handle this better
if (giveup.Elapsed > TimeSpan.FromMinutes(30) || unQueuedMessages.Count > 20000)
{
while (!unQueuedMessages.ContainsKey(cantonCommitId))
{
LogError("Giving up on: " + cantonCommitId);
cantonCommitId++;
}
}
else
{
// avoid getting out of control when the pages aren't ready yet
Log("PageCommitJob Waiting for: " + cantonCommitId);
Thread.Sleep(TimeSpan.FromSeconds(15));
}
}
}
}
finally
{
// commit anything that was waiting
if (commitTask != null)
{
commitTask.Wait();
}
Commit(writer, batchItems.ToArray()).Wait();
}
}
}
finally
{
Log("returning work to the queue");
// put everything back into the queue
ParallelOptions qOpts = new ParallelOptions();
qOpts.MaxDegreeOfParallelism = 128;
Parallel.ForEach(orderedMessages, qOpts, json =>
{
Queue.AddMessage(new CloudQueueMessage(json.ToString()));
});
Parallel.ForEach(unQueuedMessages.Values, qOpts, s =>
{
Queue.AddMessage(new CloudQueueMessage(s));
});
Log("returning work to the queue done");
}
}
private async Task Commit(AppendOnlyCatalogWriter writer, CantonCatalogItem[] batchItems)
{
var orderedBatch = batchItems.ToList();
orderedBatch.Sort(CantonCatalogItem.Compare);
int lastHighestCommit = 0;
DateTime? latestPublished = null;
// add the items to the writer
foreach (var orderedItem in orderedBatch)
{
lastHighestCommit = orderedItem.CantonCommitId + 1;
writer.Add(orderedItem);
}
Task cursorTask = null;
// only save the cursor if we did something
if (lastHighestCommit > 0)
{
// find the most recent package
latestPublished = batchItems.Select(c => c.Published).OrderByDescending(d => d).FirstOrDefault();
// update the cursor
JObject obj = new JObject();
// add one here since we are already added the current number
obj.Add("cantonCommitId", lastHighestCommit);
Log("Cursor cantonCommitId: " + lastHighestCommit);
Cursor.Position = DateTime.UtcNow;
Cursor.Metadata = obj;
cursorTask = Cursor.Save();
}
if (writer.Count > 0)
{
// perform the commit
Stopwatch timer = new Stopwatch();
timer.Start();
IGraph commitData = PackageCatalog.CreateCommitMetadata(writer.RootUri, latestPublished, latestPublished);
// commit
await writer.Commit(DateTime.UtcNow, commitData, CancellationToken.None);
timer.Stop();
Console.WriteLine("Commit duration: " + timer.Elapsed);
}
if (cursorTask != null)
{
await cursorTask;
}
}
