|
public GetGraphAsync ( |
||
| address | ||
| return | Task |
|
protected override async Task<bool> OnProcessBatch(CollectorHttpClient client, IEnumerable<JToken> items, JToken context, DateTime commitTimeStamp, bool isLastBatch, CancellationToken cancellationToken)
{
List<Task<IGraph>> tasks = new List<Task<IGraph>>();
foreach (JObject item in items)
{
if (Utils.IsType((JObject)context, item, _types))
{
Uri itemUri = item["@id"].ToObject<Uri>();
tasks.Add(client.GetGraphAsync(itemUri));
}
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks.ToArray());
TripleStore store = new TripleStore();
foreach (Task<IGraph> task in tasks)
{
store.Add(task.Result, true);
}
await ProcessStore(store, cancellationToken);
}
return true;
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp)
{
List <Task <IGraph> > tasks = new List <Task <IGraph> >();
foreach (JObject item in items)
{
if (Utils.IsType((JObject)context, item, _types))
{
Uri itemUri = item["@id"].ToObject <Uri>();
tasks.Add(client.GetGraphAsync(itemUri));
}
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks.ToArray());
TripleStore store = new TripleStore();
foreach (Task <IGraph> task in tasks)
{
store.Add(task.Result, true);
}
await ProcessStore(store);
}
return(true);
}
protected override async Task ProcessSortedBatchAsync(
CollectorHttpClient client,
KeyValuePair <string, IList <CatalogCommitItem> > sortedBatch,
JToken context,
CancellationToken cancellationToken)
{
var graphs = new Dictionary <string, IGraph>();
var graphTasks = new Dictionary <string, Task <IGraph> >();
foreach (var item in sortedBatch.Value)
{
var isMatch = false;
foreach (Uri type in _types)
{
if (item.TypeUris.Any(typeUri => typeUri.AbsoluteUri == type.AbsoluteUri))
{
isMatch = true;
break;
}
}
if (isMatch)
{
// Load package details from catalog.
// Download the graph to a read-only container. This allows operations on each graph to be safely
// parallelized.
var task = client.GetGraphAsync(item.Uri, readOnly: true, token: cancellationToken);
graphTasks.Add(item.Uri.AbsoluteUri, task);
}
}
await Task.WhenAll(graphTasks.Values.ToArray());
foreach (var task in graphTasks)
{
graphs.Add(task.Key, task.Value.Result);
}
if (graphs.Count > 0)
{
var sortedGraphs = new KeyValuePair <string, IReadOnlyDictionary <string, IGraph> >(sortedBatch.Key, graphs);
await ProcessGraphsAsync(sortedGraphs, cancellationToken);
}
}
public void CatalogTest_VerifyPackage()
{
string pathA = CreateNupkg("MyPackage", "1.2.1");
CatalogStep step = new CatalogStep(Config, new string[] { pathA });
step.Run();
var file = Config.Catalog.LocalFolder.GetFiles("mypackage.1.2.1.json", SearchOption.AllDirectories).FirstOrDefault();
JsonTextReader reader = new JsonTextReader(file.OpenText());
CollectorHttpClient client = new CollectorHttpClient(Config.Catalog.FileSystemEmulator);
var task = client.GetGraphAsync(new Uri(Config.Catalog.BaseAddress.AbsoluteUri + "/index.json"));
task.Wait();
IGraph graph = task.Result;
}
protected override async Task ProcessSortedBatch(CollectorHttpClient client, KeyValuePair<string, IList<JObject>> sortedBatch, JToken context, CancellationToken cancellationToken)
{
IDictionary<string, IGraph> graphs = new Dictionary<string, IGraph>();
foreach (JObject item in sortedBatch.Value)
{
if (Utils.IsType((JObject)context, item, _types))
{
string itemUri = item["@id"].ToString();
IGraph graph = await client.GetGraphAsync(new Uri(itemUri), cancellationToken);
graphs.Add(itemUri, graph);
}
}
if (graphs.Count > 0)
{
await ProcessGraphs(new KeyValuePair<string, IDictionary<string, IGraph>>(sortedBatch.Key, graphs), cancellationToken);
}
}
protected override async Task ProcessSortedBatch(CollectorHttpClient client, KeyValuePair <string, IList <JObject> > sortedBatch, JToken context)
{
IDictionary <string, IGraph> graphs = new Dictionary <string, IGraph>();
foreach (JObject item in sortedBatch.Value)
{
if (Utils.IsType((JObject)context, item, _types))
{
string itemUri = item["@id"].ToString();
IGraph graph = await client.GetGraphAsync(new Uri(itemUri));
graphs.Add(itemUri, graph);
}
}
if (graphs.Count > 0)
{
await ProcessGraphs(new KeyValuePair <string, IDictionary <string, IGraph> >(sortedBatch.Key, graphs));
}
}
public async Task ProcessGraphs(CollectorHttpClient client, string packageId, IEnumerable<Uri> catalogPageUris, JObject context, CancellationToken cancellationToken)
{
ConcurrentDictionary<string, IGraph> graphs = new ConcurrentDictionary<string, IGraph>();
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
var uris = catalogPageUris.ToArray();
Parallel.ForEach(uris, options, uri =>
{
var task = client.GetGraphAsync(uri);
task.Wait();
if (!graphs.TryAdd(uri.AbsoluteUri, task.Result))
{
throw new Exception("Duplicate graph: " + uri);
}
});
await ProcessGraphs(new KeyValuePair<string, IDictionary<string, IGraph>>(packageId, graphs), cancellationToken);
}
protected override async Task ProcessSortedBatchAsync(
CollectorHttpClient client,
KeyValuePair <string, IList <JObject> > sortedBatch,
JToken context,
CancellationToken cancellationToken)
{
var graphs = new Dictionary <string, IGraph>();
var graphTasks = new Dictionary <string, Task <IGraph> >();
foreach (var item in sortedBatch.Value)
{
if (Utils.IsType((JObject)context, item, _types))
{
var itemUri = item["@id"].ToString();
// Load package details from catalog.
// Download the graph to a read-only container. This allows operations on each graph to be safely
// parallelized.
var task = client.GetGraphAsync(new Uri(itemUri), readOnly: true, token: cancellationToken);
graphTasks.Add(itemUri, task);
}
}
await Task.WhenAll(graphTasks.Values.ToArray());
foreach (var task in graphTasks)
{
graphs.Add(task.Key, task.Value.Result);
}
if (graphs.Count > 0)
{
var sortedGraphs = new KeyValuePair <string, IReadOnlyDictionary <string, IGraph> >(sortedBatch.Key, graphs);
await ProcessGraphsAsync(sortedGraphs, cancellationToken);
}
}
protected override async Task ProcessSortedBatch(
CollectorHttpClient client,
KeyValuePair<string, IList<JObject>> sortedBatch,
JToken context,
CancellationToken cancellationToken)
{
var graphs = new Dictionary<string, IGraph>();
var graphTasks = new Dictionary<string, Task<IGraph>>();
foreach (var item in sortedBatch.Value)
{
if (Utils.IsType((JObject)context, item, _types))
{
var itemUri = item["@id"].ToString();
var task = client.GetGraphAsync(new Uri(itemUri), cancellationToken);
graphTasks.Add(itemUri, task);
if (!Concurrent)
{
task.Wait(cancellationToken);
}
}
}
await Task.WhenAll(graphTasks.Values.ToArray());
foreach (var task in graphTasks)
{
graphs.Add(task.Key, task.Value.Result);
}
if (graphs.Count > 0)
{
await ProcessGraphs(new KeyValuePair<string, IDictionary<string, IGraph>>(sortedBatch.Key, graphs), cancellationToken);
}
}
private async Task StrikeAsync()
{
_log.WriteLine("Start lightning strike for {0}...", _cursorFile);
// Get batch range
int batchStart;
int batchEnd;
using (var cursorStreamReader = new StreamReader(_cursorFile))
{
var batchRange = (await cursorStreamReader.ReadLineAsync()).Split(',');
batchStart = int.Parse(batchRange[0]);
batchEnd = int.Parse(batchRange[1]);
_log.WriteLine("Batch range: {0} - {1}", batchStart, batchEnd);
}
if (batchStart > batchEnd)
{
_log.WriteLine("Batch already finished.");
return;
}
// Time to strike
var collectorHttpClient = new CollectorHttpClient();
var account = CloudStorageAccount.Parse(_storageAccount);
var storageFactory = (StorageFactory)new AzureStorageFactory(account, _storageContainer, null, new Uri(_storageBaseAddress))
{
CompressContent = _compress
};
var startElement = string.Format("Element@{0}.", batchStart);
var endElement = string.Format("Element@{0}.", batchEnd + 1);
using (var indexStreamReader = new StreamReader(_indexFile))
{
string line;
// Skip entries that are not in the current batch bounds
do
{
line = await indexStreamReader.ReadLineAsync();
}
while (!line.Contains(startElement));
// Run until we're outside the current batch bounds
while (!string.IsNullOrEmpty(line) && !line.Contains(endElement) && !indexStreamReader.EndOfStream)
{
_log.WriteLine(line);
try
{
var packageId = line.Split(new[] { ". " }, StringSplitOptions.None).Last().Trim();
var sortedGraphs = new Dictionary<string, IGraph>();
line = await indexStreamReader.ReadLineAsync();
while (!string.IsNullOrEmpty(line) && !line.Contains("Element@") && !indexStreamReader.EndOfStream)
{
// Fetch graph for package version
var url = line.TrimEnd();
var graph = await collectorHttpClient.GetGraphAsync(new Uri(url));
if (sortedGraphs.ContainsKey(url))
{
sortedGraphs[url] = graph;
}
else
{
sortedGraphs.Add(url, graph);
}
// To reduce memory footprint, we're flushing out large registrations
// in very small batches.
if (graph.Nodes.Count() > 3000 && sortedGraphs.Count >= 2)
{
// Process graphs
await ProcessGraphsAsync(packageId, sortedGraphs, storageFactory, _contentBaseAddress);
// Destroy!
sortedGraphs = new Dictionary<string, IGraph>();
}
// Read next line
line = await indexStreamReader.ReadLineAsync();
}
// Process graphs
if (sortedGraphs.Any())
{
await ProcessGraphsAsync(packageId, sortedGraphs, storageFactory, _contentBaseAddress);
}
// Update cursor file so next time we have less work to do
batchStart++;
await UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd);
}
catch (Exception)
{
UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd).Wait();
throw;
}
}
}
await UpdateCursorFileAsync("DONE" + _cursorFile, batchStart, batchEnd);
_log.WriteLine("Finished lightning strike for {0}.", _cursorFile);
}
