private async Task WriteJson(IJsonOutput output, ProgressOutput progress,
IList <IDictionary <string, object> > taken)
{
await output.WriteBatchAsync(taken).ConfigureAwait(false);
var newVal = Interlocked.Add(ref _totalWritten, taken.Count);
var progressOutput = progress.Render(newVal, Count);
PositionalConsole.WriteLineAt(0, 1,
$"Written {newVal.ToString($"D{_countDigits}")} of {Count}...{progressOutput}");
taken.Clear();
}
private async Task ConsumeJson(object obj)
{
var output = (IJsonOutput)obj;
var taken = new List <IDictionary <string, object> >();
var progress = new ProgressOutput(1);
while (!_jsonQueue.IsCompleted)
{
if (_jsonQueue.TryTake(out var next))
{
taken.Add(next);
if (taken.Count == BatchSize)
{
try {
await WriteJson(output, progress, taken).ConfigureAwait(false);
} catch (Exception e) {
Console.ForegroundColor = ConsoleColor.DarkRed;
PositionalConsole.WriteLine($"Error during output write: {e.InnerException ?? e}");
Console.ResetColor();
return;
}
}
}
}
if (taken.Any())
{
try {
await WriteJson(output, progress, taken).ConfigureAwait(false);
} catch (Exception e) {
Console.ForegroundColor = ConsoleColor.DarkRed;
PositionalConsole.WriteLine($"Error during output write: {e.InnerException ?? e}");
Console.ResetColor();
}
}
}
/// <summary>
/// Generates the data and writes it to the specified output. This method is
/// synchronous but the actual work is asynchronous. This method will return when
/// all work is done.
/// </summary>
/// <param name="input">The filename of the file containing the template input</param>
/// <param name="outputType">The type of output to create (For a list refer to <seealso cref="ListOutput"/>)</param>
/// <param name="extraArgs">The arguments needed for the specified output type (For a list refer to <seealso cref="ListOutput"/>)(</param>
public void Generate(string input, string outputType, string[] extraArgs = null)
{
var progress = new ProgressOutput(500);
Setup();
var outputObject = AvailableOutputs.FirstOrDefault(x => x.Name == outputType);
if (outputObject == null)
{
throw new InvalidOperationException(
$"Output type '{outputType}' not found in program or loaded plugins");
}
if (!outputObject.PreloadArgs(extraArgs ?? new string[0]))
{
Console.ForegroundColor = ConsoleColor.DarkRed;
PositionalConsole.WriteLine($"Failed to load arguments into '{outputType}' output");
Console.ResetColor();
throw new ApplicationException($"Failed to load arguments into '{outputType}' output");
}
var tasks = StartConsumers(outputObject);
var template = File.ReadAllText(input).Replace("\r\n", "\n");
var templateObject = JsonConvert.DeserializeObject <IReadOnlyDictionary <string, object> >(template);
var parallelOpts = new ParallelOptions
{
MaxDegreeOfParallelism = _configuration.CreateJobCount
};
var exceptions = new List <Exception>();
Parallel.For(0, Count, parallelOpts, async(i, state) =>
{
if (state.IsExceptional)
{
return;
}
var result = new Dictionary <string, object>();
try {
await ApplyTemplate(templateObject, result).ConfigureAwait(false);
} catch (Exception e) {
state.Stop();
exceptions.Add(e);
}
var newVal = Interlocked.Increment(ref _totalCreated);
var progressOutput = progress.Render(newVal, Count);
PositionalConsole.WriteLineAt(0, 0,
$"Created {newVal.ToString($"D{_countDigits}")} of {Count}...{progressOutput}");
_jsonQueue.Add(result);
});
if (exceptions.Any())
{
var agg = new AggregateException("Error during JSON generation", exceptions);
PositionalConsole.WriteLine($"Error during processing: {agg}", ConsoleColor.DarkRed);
}
else
{
PositionalConsole.WriteLine("Finished adding, waiting for write jobs...");
}
_jsonQueue.CompleteAdding();
Task.WaitAll(tasks);
PositionalConsole.WriteLine("Write jobs finished!");
(outputObject as IDisposable)?.Dispose();
}
