internal static void Run()
{
if (string.IsNullOrEmpty(Endpoints))
{
throw new ArgumentException("FhirEndpoints value is empty");
}
endpoints = Endpoints.Split(";", StringSplitOptions.RemoveEmptyEntries).ToList();
var globalPrefix = $"RequestedBlobRange=[{NumberOfBlobsToSkip + 1}-{MaxBlobIndexForImport}]";
Console.WriteLine($"{globalPrefix}: Starting...");
var blobContainerClient = GetContainer(ConnectionString, ContainerName);
var blobs = blobContainerClient.GetBlobs().OrderBy(_ => _.Name).Where(_ => _.Name.EndsWith(".ndjson", StringComparison.OrdinalIgnoreCase)).ToList();
Console.WriteLine($"Found ndjson blobs={blobs.Count} in {ContainerName}.");
var take = MaxBlobIndexForImport == 0 ? blobs.Count : MaxBlobIndexForImport - NumberOfBlobsToSkip;
blobs = blobs.Skip(NumberOfBlobsToSkip).Take(take).ToList();
var swWrites = Stopwatch.StartNew();
var swReport = Stopwatch.StartNew();
var currentBlobRanges = new Tuple <int, int> [ReadThreads];
BatchExtensions.ExecuteInParallelBatches(blobs, ReadThreads, BlobRangeSize, (reader, blobRangeInt) =>
{
var localSw = Stopwatch.StartNew();
var writes = 0L;
var blobRangeIndex = blobRangeInt.Item1;
var blobsInt = blobRangeInt.Item2;
var firstBlob = NumberOfBlobsToSkip + (blobRangeIndex * BlobRangeSize) + 1;
var lastBlob = NumberOfBlobsToSkip + (blobRangeIndex * BlobRangeSize) + blobsInt.Count;
currentBlobRanges[reader] = Tuple.Create(firstBlob, lastBlob);
var prefix = $"Reader={reader}.BlobRange=[{firstBlob}-{lastBlob}]";
var incrementor = new IndexIncrementor(endpoints.Count);
Console.WriteLine($"{prefix}: Starting...");
// 100 below is a compromise between processing with maximum available threads (value 1) and inefficiency in wrapping single resource in a list.
BatchExtensions.ExecuteInParallelBatches(GetLinesInBlobRange(blobsInt, prefix), WriteThreads / ReadThreads, 100, (thread, lineBatch) =>
{
Interlocked.Increment(ref writers);
foreach (var line in lineBatch.Item2)
{
Interlocked.Increment(ref totalWrites);
Interlocked.Increment(ref writes);
PutResource(line, incrementor);
if (swReport.Elapsed.TotalSeconds > ReportingPeriodSec)
{
lock (swReport)
{
if (swReport.Elapsed.TotalSeconds > ReportingPeriodSec)
{
var currWrites = Interlocked.Read(ref totalWrites);
var currReads = Interlocked.Read(ref totalReads);
var minBlob = currentBlobRanges.Min(_ => _.Item1);
var maxBlob = currentBlobRanges.Max(_ => _.Item2);
Console.WriteLine($"{globalPrefix}.WorkingBlobRange=[{minBlob}-{maxBlob}].Readers=[{Interlocked.Read(ref readers)}/{ReadThreads}].Writers=[{Interlocked.Read(ref writers)}].EndPointCalls=[{Interlocked.Read(ref epCalls)}].Waits=[{Interlocked.Read(ref waits)}]: reads={currReads} writes={currWrites} secs={(int)swWrites.Elapsed.TotalSeconds} read-speed={(int)(currReads / swReads.Elapsed.TotalSeconds)} lines/sec write-speed={(int)(currWrites / swWrites.Elapsed.TotalSeconds)} res/sec");
swReport.Restart();
}
}
}
}
Interlocked.Decrement(ref writers);
});
Console.WriteLine($"{prefix}: Completed writes. Total={writes} secs={(int)localSw.Elapsed.TotalSeconds} speed={(int)(writes / localSw.Elapsed.TotalSeconds)} res/sec");
});
Console.WriteLine($"{globalPrefix}.Readers=[{readers}/{ReadThreads}].Writers=[{writers}].EndPointCalls=[{epCalls}].Waits=[{waits}]: total reads={totalReads} total writes={totalWrites} secs={(int)swWrites.Elapsed.TotalSeconds} read-speed={(int)(totalReads / swReads.Elapsed.TotalSeconds)} lines/sec write-speed={(int)(totalWrites / swWrites.Elapsed.TotalSeconds)} res/sec");
}
private static void PutResource(string jsonString, IndexIncrementor incrementor)
{
var(resourceType, resourceId) = ParseJson(jsonString);
using var content = new StringContent(jsonString, Encoding.UTF8, "application/json");
var maxRetries = MaxRetries;
var retries = 0;
var networkError = false;
var bad = false;
string endpoint;
do
{
endpoint = endpoints[incrementor.Next()];
var uri = new Uri(endpoint + "/" + resourceType + "/" + resourceId);
bad = false;
Interlocked.Increment(ref epCalls);
try
{
Thread.Sleep(40);
var response = HttpClient.PutAsync(uri, content).Result;
switch (response.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.Created:
break;
default:
bad = true;
var statusString = response.StatusCode.ToString();
if (response.StatusCode != HttpStatusCode.BadGateway || retries > 0) // too many bad gateway messages in the log
{
Console.WriteLine($"Retries={retries} Endpoint={endpoint} HttpStatusCode={statusString} ResourceType={resourceType} ResourceId={resourceId}");
}
if (response.StatusCode == HttpStatusCode.TooManyRequests) // retry overload errors forever
{
maxRetries++;
}
break;
}
}
catch (Exception e)
{
networkError = IsNetworkError(e);
if (!networkError)
{
Console.WriteLine($"Retries={retries} Endpoint={endpoint} ResourceType={resourceType} ResourceId={resourceId} Error={(networkError ? "network" : e.Message)}");
}
bad = true;
if (networkError) // retry network errors forever
{
maxRetries++;
}
}
finally
{
Interlocked.Decrement(ref epCalls);
}
if (bad && retries < maxRetries)
{
retries++;
Interlocked.Increment(ref waits);
Thread.Sleep(networkError ? 1000 : 200 * retries);
Interlocked.Decrement(ref waits);
}
}while (bad && retries < maxRetries);
if (bad)
{
Console.WriteLine($"Failed writing ResourceType={resourceType} ResourceId={resourceId}. Retries={retries} Endpoint={endpoint}");
}
}
