public override async Task RunCore()
{
int nextMasterRegId = 0;
DateTime position = Cursor.Position;
JToken nextMasterRegIdToken = null;
if (Cursor.Metadata.TryGetValue("nextMasterRegId", out nextMasterRegIdToken))
{
nextMasterRegId = nextMasterRegIdToken.ToObject <int>();
}
// Get the catalog index
Uri catalogIndexUri = new Uri(Config.GetProperty("CatalogIndex"));
Log("Reading index entries");
var indexReader = new CatalogIndexReader(catalogIndexUri);
var indexEntries = await indexReader.GetRolledUpEntries();
var context = indexReader.GetContext();
Log("Finding new or editted entries");
var changedEntries = new HashSet <string>(indexEntries.Where(e => e.CommitTimeStamp.CompareTo(position) > 0)
.Select(e => e.Id.ToLowerInvariant()), StringComparer.OrdinalIgnoreCase);
DateTime newPosition = indexEntries.Select(e => e.CommitTimeStamp).OrderByDescending(e => e).FirstOrDefault();
ConcurrentDictionary <string, ConcurrentBag <Uri> > batches = new ConcurrentDictionary <string, ConcurrentBag <Uri> >(StringComparer.OrdinalIgnoreCase);
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Parallel.ForEach(indexEntries, options, entry =>
{
if (changedEntries.Contains(entry.Id))
{
batches.AddOrUpdate(entry.Id, new ConcurrentBag <Uri>()
{
entry.Uri
}, (id, uris) =>
{
uris.Add(entry.Uri);
return(uris);
});
}
});
Uri contentBaseAddress = new Uri(Config.GetProperty("ContentBaseAddress"));
if (batches.Count > 0)
{
Log("Building registrations from: " + position.ToString("O"));
options.MaxDegreeOfParallelism = 4;
for (int i = 0; i < 3 && batches.Count > 0; i++)
{
if (i != 0)
{
options.MaxDegreeOfParallelism = 1;
Console.WriteLine("Single batch run.");
}
var ids = batches.Keys.OrderBy(s => s).ToArray();
Stopwatch buildTimer = new Stopwatch();
buildTimer.Start();
int startingCount = ids.Length;
Parallel.ForEach(ids, options, id =>
{
try
{
BatchRegistrationCollector regCollector = new BatchRegistrationCollector(null, _factory);
regCollector.ContentBaseAddress = contentBaseAddress;
Stopwatch timer = new Stopwatch();
timer.Start();
var uriGroup = batches[id].ToArray();
regCollector.ProcessGraphs(_client, id, uriGroup, context).Wait();
int rem = batches.Count;
timer.Stop();
string log = String.Format("Completed: {0} Duration: {1} Uris: {2} Remaining Ids: {3} Loop: {4}", id, timer.Elapsed, uriGroup.Length, rem, i);
Console.WriteLine(log);
// stats
double perPackage = buildTimer.Elapsed.TotalSeconds / (double)(startingCount - rem + 1);
DateTime finish = DateTime.Now.AddSeconds(Math.Ceiling(perPackage * rem));
Console.WriteLine("Estimated Finish: " + finish.ToString("O"));
ConcurrentBag <Uri> vals;
if (!batches.TryRemove(id, out vals))
{
Console.WriteLine("Unable to remove!");
}
}
catch (Exception ex)
{
LogError("Registration failed: " + id + " " + ex.ToString());
}
});
}
// mark this with the last commit we included
Cursor.Position = newPosition;
await Cursor.Save();
Log("Finished registrations: " + newPosition.ToString("O"));
}
}
public override async Task RunCore(CancellationToken cancellationToken)
{
int nextMasterRegId = 0;
DateTime position = Cursor.Position;
JToken nextMasterRegIdToken = null;
if (Cursor.Metadata.TryGetValue("nextMasterRegId", out nextMasterRegIdToken))
{
nextMasterRegId = nextMasterRegIdToken.ToObject<int>();
}
// Get the catalog index
Uri catalogIndexUri = new Uri(Config.GetProperty("CatalogIndex"));
Log("Reading index entries");
var indexReader = new CatalogIndexReader(catalogIndexUri);
var indexEntries = await indexReader.GetRolledUpEntries();
var context = indexReader.GetContext();
Log("Finding new or editted entries");
var changedEntries = new HashSet<string>(indexEntries.Where(e => e.CommitTimeStamp.CompareTo(position) > 0)
.Select(e => e.Id.ToLowerInvariant()), StringComparer.OrdinalIgnoreCase);
DateTime newPosition = indexEntries.Select(e => e.CommitTimeStamp).OrderByDescending(e => e).FirstOrDefault();
ConcurrentDictionary<string, ConcurrentBag<Uri>> batches = new ConcurrentDictionary<string, ConcurrentBag<Uri>>(StringComparer.OrdinalIgnoreCase);
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Parallel.ForEach(indexEntries, options, entry =>
{
if (changedEntries.Contains(entry.Id))
{
batches.AddOrUpdate(entry.Id, new ConcurrentBag<Uri>() { entry.Uri }, (id, uris) =>
{
uris.Add(entry.Uri);
return uris;
});
}
});
Uri contentBaseAddress = new Uri(Config.GetProperty("ContentBaseAddress"));
if (batches.Count > 0)
{
Log("Building registrations from: " + position.ToString("O"));
options.MaxDegreeOfParallelism = 4;
for (int i = 0; i < 3 && batches.Count > 0; i++)
{
if (i != 0)
{
options.MaxDegreeOfParallelism = 1;
Console.WriteLine("Single batch run.");
}
var ids = batches.Keys.OrderBy(s => s).ToArray();
Stopwatch buildTimer = new Stopwatch();
buildTimer.Start();
int startingCount = ids.Length;
Parallel.ForEach(ids, options, id =>
{
try
{
BatchRegistrationCollector regCollector = new BatchRegistrationCollector(null, _factory);
regCollector.ContentBaseAddress = contentBaseAddress;
Stopwatch timer = new Stopwatch();
timer.Start();
var uriGroup = batches[id].ToArray();
regCollector.ProcessGraphs(_client, id, uriGroup, context, cancellationToken).Wait();
int rem = batches.Count;
timer.Stop();
string log = String.Format("Completed: {0} Duration: {1} Uris: {2} Remaining Ids: {3} Loop: {4}", id, timer.Elapsed, uriGroup.Length, rem, i);
Console.WriteLine(log);
// stats
double perPackage = buildTimer.Elapsed.TotalSeconds / (double)(startingCount - rem + 1);
DateTime finish = DateTime.Now.AddSeconds(Math.Ceiling(perPackage * rem));
Console.WriteLine("Estimated Finish: " + finish.ToString("O"));
ConcurrentBag<Uri> vals;
if (!batches.TryRemove(id, out vals))
{
Console.WriteLine("Unable to remove!");
}
}
catch (Exception ex)
{
LogError("Registration failed: " + id + " " + ex.ToString());
}
});
}
// mark this with the last commit we included
Cursor.Position = newPosition;
await Cursor.Save();
Log("Finished registrations: " + newPosition.ToString("O"));
}
}
