protected override async Task<bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken)
{
JObject root = await client.GetJObjectAsync(Index, cancellationToken);
IEnumerable<CatalogItem> rootItems = root["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value)
.OrderBy(item => item.CommitTimeStamp);
bool acceptNextBatch = false;
foreach (CatalogItem rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
JToken context = null;
page.TryGetValue("@context", out context);
var batches = await CreateBatches(page["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value));
var orderedBatches = batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToList();
var lastBatch = orderedBatches.LastOrDefault();
foreach (var batch in orderedBatches)
{
acceptNextBatch = await OnProcessBatch(
client,
batch.Items.Select(item => item.Value),
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
front.Value = batch.CommitTimeStamp;
await front.Save(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Save has value: {0}", front);
if (!acceptNextBatch)
{
break;
}
}
if (!acceptNextBatch)
{
break;
}
}
return acceptNextBatch;
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IEnumerable<JToken> items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken)
{
JObject catalogIndex = (_baseAddress != null) ? await client.GetJObjectAsync(Index, cancellationToken) : null;
IEnumerable<JObject> catalogItems = await FetchCatalogItems(client, items, cancellationToken);
var numDocs = _indexWriter.NumDocs();
_logger.LogInformation(string.Format("Index contains {0} documents.", _indexWriter.NumDocs()));
ProcessCatalogIndex(_indexWriter, catalogIndex, _baseAddress);
ProcessCatalogItems(_indexWriter, catalogItems, _baseAddress);
var docsDifference = _indexWriter.NumDocs() - numDocs;
UpdateCommitMetadata(commitTimeStamp, docsDifference);
_logger.LogInformation(string.Format("Processed catalog items. Index now contains {0} documents. (total uncommitted {1}, batch {2})",
_indexWriter.NumDocs(), _metadataForNextCommit.Count, docsDifference));
if (_commitEachBatch || isLastBatch)
{
EnsureCommitted();
}
return true;
}
public static async Task ProcessGraphs(
string id,
IDictionary<string, IGraph> sortedGraphs,
StorageFactory storageFactory,
Uri contentBaseAddress,
int partitionSize,
int packageCountThreshold,
CancellationToken cancellationToken)
{
int versionAlreadyExistsCount = 0;
existingVersionsWithID = new List<string>();
try
{
Storage storage = storageFactory.Create(id.ToLowerInvariant());
Uri resourceUri = storage.ResolveUri("index.json");
string json = await storage.LoadString(resourceUri, cancellationToken);
int count = Utils.CountItems(json);
//Determine if there are any versions that are existing already
CollectorHttpClient httpClient = new CollectorHttpClient();
foreach (var graph in sortedGraphs)
{
JObject jsonContent = await httpClient.GetJObjectAsync(new Uri(graph.Key), cancellationToken);
string existingId = jsonContent["@id"].ToString();
string existingVersionWithId = existingId.Substring(existingId.LastIndexOf("/") + 1);
string existingVersion = jsonContent["version"].ToString() + ".json";
//Determine if the version is actually available
//In Registration blobs, the format is /packageID/packageVersion.json
//So to check the existence of version we need to know only the version.json
if (storage.Exists(existingVersion))
{
//When we compare later in AddExistingItems, we need the "packageId.packageversion.json" for comparison so store it with Id
existingVersionsWithID.Add(existingVersionWithId);
versionAlreadyExistsCount++;
}
}
int total = count + sortedGraphs.Count - versionAlreadyExistsCount;
if (total < packageCountThreshold)
{
await SaveSmallRegistration(storage, storageFactory.BaseAddress, sortedGraphs, contentBaseAddress, partitionSize, cancellationToken);
}
else
{
await SaveLargeRegistration(storage, storageFactory.BaseAddress, sortedGraphs, json, contentBaseAddress, partitionSize, cancellationToken);
}
}
catch (Exception e)
{
throw new Exception(string.Format("Process id = {0}", id), e);
}
}
static async Task<IEnumerable<JObject>> FetchCatalogItems(CollectorHttpClient client, IEnumerable<JToken> items, CancellationToken cancellationToken)
{
IList<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JToken item in items)
{
Uri catalogItemUri = item["@id"].ToObject<Uri>();
tasks.Add(client.GetJObjectAsync(catalogItemUri, cancellationToken));
}
await Task.WhenAll(tasks);
return tasks.Select(t => t.Result);
}
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IEnumerable<JToken> items, JToken context, DateTime commitTimeStamp, CancellationToken cancellationToken)
{
JObject catalogIndex = (_baseAddress != null) ? await client.GetJObjectAsync(Index, cancellationToken) : null;
IEnumerable<JObject> catalogItems = await FetchCatalogItems(client, items, cancellationToken);
using (IndexWriter indexWriter = CreateIndexWriter(_directory))
{
Trace.TraceInformation("Index contains {0} documents", indexWriter.NumDocs());
ProcessCatalogIndex(indexWriter, catalogIndex, _baseAddress);
ProcessCatalogItems(indexWriter, catalogItems, _baseAddress);
indexWriter.ExpungeDeletes();
indexWriter.Commit(CreateCommitMetadata(commitTimeStamp));
Trace.TraceInformation("COMMIT index contains {0} documents commitTimeStamp {1}", indexWriter.NumDocs(), commitTimeStamp.ToString("O"));
}
return true;
}
protected async Task <IEnumerable <CatalogCommit> > FetchCatalogCommitsAsync(
CollectorHttpClient client,
ReadCursor front,
ReadCursor back,
CancellationToken cancellationToken)
{
JObject root;
using (_telemetryService.TrackDuration(
TelemetryConstants.CatalogIndexReadDurationSeconds,
new Dictionary <string, string>()
{
{ TelemetryConstants.Uri, Index.AbsoluteUri }
}))
{
root = await client.GetJObjectAsync(Index, cancellationToken);
}
var commits = root["items"].Select(item => CatalogCommit.Create((JObject)item));
return(GetCommitsInRange(commits, front.Value, back.Value));
}
protected async Task <IEnumerable <CatalogCommit> > FetchCatalogCommitsAsync(
CollectorHttpClient client,
ReadWriteCursor front,
CancellationToken cancellationToken)
{
JObject root;
using (_telemetryService.TrackDuration(
TelemetryConstants.CatalogIndexReadDurationSeconds,
new Dictionary <string, string>()
{
{ TelemetryConstants.Uri, Index.AbsoluteUri }
}))
{
root = await client.GetJObjectAsync(Index, cancellationToken);
}
IEnumerable <CatalogCommit> commits = root["items"]
.Select(item => CatalogCommit.Create((JObject)item))
.Where(item => item.CommitTimeStamp > front.Value)
.OrderBy(item => item.CommitTimeStamp);
return(commits);
}
protected override async Task<bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken)
{
int beforeBatchCount = BatchCount;
IList<JObject> items = new List<JObject>();
JObject root = await client.GetJObjectAsync(Index, cancellationToken);
JToken context = null;
root.TryGetValue("@context", out context);
IEnumerable<JToken> rootItems = root["items"].OrderBy(item => item["commitTimeStamp"].ToObject<DateTime>());
DateTime resumeDateTime = front.Value;
bool acceptNextBatch = true;
foreach (JObject rootItem in rootItems)
{
if (!acceptNextBatch)
{
break;
}
DateTime rootItemCommitTimeStamp = rootItem["commitTimeStamp"].ToObject<DateTime>();
if (rootItemCommitTimeStamp <= front.Value)
{
continue;
}
Uri pageUri = rootItem["@id"].ToObject<Uri>();
JObject page = await client.GetJObjectAsync(pageUri);
IEnumerable<JToken> pageItems = page["items"].OrderBy(item => item["commitTimeStamp"].ToObject<DateTime>());
foreach (JObject pageItem in pageItems)
{
DateTime pageItemCommitTimeStamp = pageItem["commitTimeStamp"].ToObject<DateTime>();
if (pageItemCommitTimeStamp <= front.Value)
{
continue;
}
if (pageItemCommitTimeStamp > back.Value)
{
break;
}
items.Add(pageItem);
resumeDateTime = pageItemCommitTimeStamp;
if (items.Count == _batchSize)
{
acceptNextBatch = await ProcessBatch(client, items, context, front, resumeDateTime, cancellationToken);
if (!acceptNextBatch)
{
break;
}
}
}
}
if (acceptNextBatch && items.Count > 0)
{
await ProcessBatch(client, items, context, front, resumeDateTime, cancellationToken);
}
int afterBatchCount = BatchCount;
PreviousRunBatchCount = (afterBatchCount - beforeBatchCount);
return (PreviousRunBatchCount > 0);
}
internal static async Task <bool> ProcessCatalogCommitsAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
FetchCatalogCommitsAsync fetchCatalogCommitsAsync,
CreateCommitItemBatchesAsync createCommitItemBatchesAsync,
ProcessCommitItemBatchAsync processCommitItemBatchAsync,
int maxConcurrentBatches,
ILogger logger,
CancellationToken cancellationToken)
{
var rootItems = await fetchCatalogCommitsAsync(client, front, back, cancellationToken);
var hasAnyBatchFailed = false;
var hasAnyBatchBeenProcessed = false;
foreach (CatalogCommit rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
var context = (JObject)page["@context"];
CatalogCommitItemBatch[] batches = await CreateBatchesForAllAvailableItemsInPageAsync(
front,
back,
page,
context,
createCommitItemBatchesAsync);
if (!batches.Any())
{
continue;
}
hasAnyBatchBeenProcessed = true;
DateTime maxCommitTimeStamp = GetMaxCommitTimeStamp(batches);
var unprocessedBatches = batches.ToList();
var processingBatches = new List <CatalogCommitItemBatchTask>();
var exceptions = new List <Exception>();
StartProcessingBatchesIfNoFailures(
client,
context,
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
processCommitItemBatchAsync,
cancellationToken);
while (processingBatches.Any())
{
var activeTasks = processingBatches.Where(batch => !batch.Task.IsCompleted)
.Select(batch => batch.Task)
.DefaultIfEmpty(Task.CompletedTask);
await Task.WhenAny(activeTasks);
for (var i = 0; i < processingBatches.Count; ++i)
{
var batch = processingBatches[i];
if (batch.Task.IsFaulted || batch.Task.IsCanceled)
{
hasAnyBatchFailed = true;
if (batch.Task.Exception != null)
{
var exception = ExceptionUtilities.Unwrap(batch.Task.Exception);
exceptions.Add(exception);
}
}
if (batch.Task.IsCompleted)
{
processingBatches.RemoveAt(i);
--i;
}
}
if (!hasAnyBatchFailed)
{
StartProcessingBatchesIfNoFailures(
client,
context,
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
processCommitItemBatchAsync,
cancellationToken);
}
}
if (hasAnyBatchFailed)
{
foreach (var exception in exceptions)
{
logger.LogError(_eventId, exception, Strings.BatchProcessingFailure);
}
var innerException = exceptions.Count == 1 ? exceptions.Single() : new AggregateException(exceptions);
throw new BatchProcessingException(innerException);
}
front.Value = maxCommitTimeStamp;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation($"{nameof(CatalogCommitUtilities)}.{nameof(ProcessCatalogCommitsAsync)} " +
$"{nameof(front)}.{nameof(front.Value)} saved since timestamp changed from previous: {{0}}", front);
}
return(hasAnyBatchBeenProcessed);
}
protected override async Task <bool> FetchAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
CancellationToken cancellationToken)
{
IEnumerable <CatalogCommit> commits = await FetchCatalogCommitsAsync(client, front, cancellationToken);
bool acceptNextBatch = false;
foreach (CatalogCommit commit in commits)
{
JObject page = await client.GetJObjectAsync(commit.Uri, cancellationToken);
JToken context = null;
page.TryGetValue("@context", out context);
var batches = await CreateBatchesAsync(page["items"]
.Select(item => CatalogCommitItem.Create((JObject)context, (JObject)item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value));
var orderedBatches = batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToList();
var lastBatch = orderedBatches.LastOrDefault();
DateTime?previousCommitTimeStamp = null;
foreach (var batch in orderedBatches)
{
// If the commit timestamp has changed from the previous batch, commit. This is important because if
// two batches have the same commit timestamp but processing the second fails, we should not
// progress the cursor forward.
if (previousCommitTimeStamp.HasValue && previousCommitTimeStamp != batch.CommitTimeStamp)
{
front.Value = previousCommitTimeStamp.Value;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Value saved since timestamp changed from previous: {0}", front);
}
using (_telemetryService.TrackDuration(TelemetryConstants.ProcessBatchSeconds, new Dictionary <string, string>()
{
{ TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() }
}))
{
acceptNextBatch = await OnProcessBatchAsync(
client,
batch.Items,
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
}
// If this is the last batch, commit the cursor.
if (ReferenceEquals(batch, lastBatch))
{
front.Value = batch.CommitTimeStamp;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Value saved due to last batch: {0}", front);
}
previousCommitTimeStamp = batch.CommitTimeStamp;
Trace.TraceInformation("CommitCatalog.Fetch front.Value is: {0}", front);
if (!acceptNextBatch)
{
break;
}
}
if (!acceptNextBatch)
{
break;
}
}
return(acceptNextBatch);
}
protected override async Task <bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back)
{
int beforeBatchCount = BatchCount;
IList <JObject> items = new List <JObject>();
JObject root = await client.GetJObjectAsync(Index);
JToken context = null;
root.TryGetValue("@context", out context);
IEnumerable <JToken> rootItems = root["items"].OrderBy(item => item["commitTimeStamp"].ToObject <DateTime>());
DateTime resumeDateTime = front.Value;
bool acceptNextBatch = true;
foreach (JObject rootItem in rootItems)
{
if (!acceptNextBatch)
{
break;
}
DateTime rootItemCommitTimeStamp = rootItem["commitTimeStamp"].ToObject <DateTime>();
if (rootItemCommitTimeStamp <= front.Value)
{
continue;
}
Uri pageUri = rootItem["@id"].ToObject <Uri>();
JObject page = await client.GetJObjectAsync(pageUri);
IEnumerable <JToken> pageItems = page["items"].OrderBy(item => item["commitTimeStamp"].ToObject <DateTime>());
foreach (JObject pageItem in pageItems)
{
DateTime pageItemCommitTimeStamp = pageItem["commitTimeStamp"].ToObject <DateTime>();
if (pageItemCommitTimeStamp <= front.Value)
{
continue;
}
if (pageItemCommitTimeStamp > back.Value)
{
break;
}
items.Add(pageItem);
resumeDateTime = pageItemCommitTimeStamp;
if (items.Count == _batchSize)
{
acceptNextBatch = await ProcessBatch(client, items, context, front, resumeDateTime);
if (!acceptNextBatch)
{
break;
}
}
}
}
if (acceptNextBatch && items.Count > 0)
{
await ProcessBatch(client, items, context, front, resumeDateTime);
}
int afterBatchCount = BatchCount;
PreviousRunBatchCount = (afterBatchCount - beforeBatchCount);
return(PreviousRunBatchCount > 0);
}
