static LogEvent[] ReadBatch(LogEventReader reader, Func <LogEvent, bool> filter,
int count, InvalidDataHandling invalidDataHandling)
{
var batch = new List <LogEvent>();
do
{
try
{
while (batch.Count < count && reader.TryRead(out var evt))
{
if (filter == null || filter(evt))
{
batch.Add(evt);
}
}
}
catch (Exception ex)
{
if (ex is JsonReaderException || ex is InvalidDataException)
{
if (invalidDataHandling == InvalidDataHandling.Ignore)
{
continue;
}
}
throw;
}
return(batch.ToArray());
} while (true);
}
public static void PipeEvents(LogEventReader source, Logger destination, InvalidDataHandling invalidDataHandling)
{
do
{
try
{
while (source.TryRead(out var evt))
{
destination.Write(evt);
}
return;
}
catch (Exception ex)
{
if (ex is JsonReaderException || ex is InvalidDataException)
{
if (invalidDataHandling == InvalidDataHandling.Ignore)
{
continue;
}
if (invalidDataHandling == InvalidDataHandling.Report)
{
destination.Error(ex, "An event was not in CLEF format.");
continue;
}
}
throw;
}
} while (true);
}
public static async Task <int> ShipEvents(
SeqConnection connection,
ILogEventReader reader,
List <ILogEventEnricher> enrichers,
InvalidDataHandling invalidDataHandling,
SendFailureHandling sendFailureHandling,
Func <LogEvent, bool> filter = null)
{
if (connection == null)
{
throw new ArgumentNullException(nameof(connection));
}
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
if (enrichers == null)
{
throw new ArgumentNullException(nameof(enrichers));
}
var batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling);
while (true)
{
var sendSucceeded = false;
try
{
sendSucceeded = await SendBatchAsync(
connection,
batch.LogEvents,
enrichers,
sendFailureHandling != SendFailureHandling.Ignore);
}
catch (Exception ex)
{
if (sendFailureHandling != SendFailureHandling.Ignore)
{
Log.Error(ex, "Failed to send an event batch");
}
}
if (!sendSucceeded && sendFailureHandling == SendFailureHandling.Fail)
{
return(1);
}
if (batch.IsLast)
{
break;
}
batch = await ReadBatchAsync(reader, filter, BatchSize, invalidDataHandling);
}
return(0);
}
public void SourceRead_InvalidFloatInSequence_ToTable(InvalidDataHandling dataHandlingStrategy)
{
var source = new DicomDatasetCollectionSource();
source.InvalidDataHandlingStrategy = dataHandlingStrategy;
//when we have a dicom file with an invalid Float number
var ds = new DicomDataset();
ds.Add(DicomTag.PatientAge, "123Y");
var sequence = new DicomSequence(DicomTag.AcquisitionContextSequence,
new DicomDataset()
{
{ DicomTag.WedgeAngleFloat, "3.40282347e+038" }
});
ds.Add(sequence);
var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm");
source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener());
source.FilenameField = "RelFileName";
DataTable dt = null;
switch (dataHandlingStrategy)
{
case InvalidDataHandling.MarkCorrupt:
dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
//row was not processed (which leaves data table with 0 rows and hence component returns null)
Assert.IsNull(dt);
//corrupt message should appear in the worklist
Assert.AreEqual(1, worklist.CorruptMessages.Count);
return;
case InvalidDataHandling.ConvertToNullAndWarn:
dt = source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken());
Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]);
Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]);
Assert.AreEqual(DBNull.Value, dt.Rows[0]["AcquisitionContextSequence"]);
Assert.AreEqual(0, worklist.CorruptMessages.Count);
break;
case InvalidDataHandling.ThrowException:
Assert.Throws <OverflowException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()));
return;
default:
throw new ArgumentOutOfRangeException("dataHandlingStrategy");
}
}
static LogEvent[] PipeEvents(string input, InvalidDataHandling invalidDataHandling)
{
var output = new CollectingSink();
using (var source = new LogEventReader(new StringReader(input)))
using (var destination = new LoggerConfiguration()
.MinimumLevel.Is(LevelAlias.Minimum)
.WriteTo.Sink(output)
.CreateLogger())
{
EventPipe.PipeEvents(source, destination, invalidDataHandling);
}
return(output.Events);
}
public void SourceRead_InvalidFloat_ToTable(InvalidDataHandling dataHandlingStrategy)
{
var source = new DicomDatasetCollectionSource();
source.InvalidDataHandlingStrategy = dataHandlingStrategy;
var ds = new DicomDataset();
ds.Add(DicomTag.PatientAge, "123Y");
ds.Add(DicomTag.WedgeAngleFloat, "3.40282347e+038");
var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm", new Dictionary <string, string> {
{ "MessageGuid", "123x321" }
});
source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener());
source.FilenameField = "RelFileName";
DataTable dt = null;
switch (dataHandlingStrategy)
{
case InvalidDataHandling.ThrowException:
Assert.Throws <OverflowException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()));
return;
case InvalidDataHandling.ConvertToNullAndWarn:
var tomem = new ToMemoryDataLoadEventListener(true);
dt = source.GetChunk(tomem, new GracefulCancellationToken());
Assert.AreEqual(DBNull.Value, dt.Rows[0]["WedgeAngleFloat"]);
//should be a warning about WedgeAngleFloat logged
var warning = tomem.EventsReceivedBySender.SelectMany(e => e.Value).Single(v => v.ProgressEventType == ProgressEventType.Warning);
Assert.IsTrue(warning.Message.Contains("WedgeAngleFloat"));
Assert.IsTrue(warning.Message.Contains("MessageGuid"));
Assert.IsTrue(warning.Message.Contains("123x321"));
Assert.IsTrue(warning.Message.Contains("fish.dcm"));
break;
default:
throw new ArgumentOutOfRangeException("dataHandlingStrategy");
}
Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]);
Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]);
}
static async Task <BatchResult> ReadBatchAsync(
ILogEventReader reader,
Func <LogEvent, bool> filter,
int count,
InvalidDataHandling invalidDataHandling)
{
var batch = new List <LogEvent>();
var isLast = false;
do
{
try
{
while (batch.Count < count)
{
var rr = await reader.TryReadAsync();
isLast = rr.IsAtEnd;
var evt = rr.LogEvent;
if (evt == null)
{
break;
}
if (filter == null || filter(evt))
{
batch.Add(evt);
}
}
}
catch (Exception ex)
{
if (ex is JsonReaderException || ex is InvalidDataException)
{
if (invalidDataHandling == InvalidDataHandling.Ignore)
{
continue;
}
}
throw;
}
return(new BatchResult(batch.ToArray(), isLast));
} while (true);
}
public static async Task <int> ShipEvents(
SeqConnection connection,
string apiKey,
ILogEventReader reader,
InvalidDataHandling invalidDataHandling,
SendFailureHandling sendFailureHandling,
int batchSize,
Func <LogEvent, bool> filter = null)
{
if (connection == null)
{
throw new ArgumentNullException(nameof(connection));
}
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
var batch = await ReadBatchAsync(reader, filter, batchSize, invalidDataHandling);
var retries = 0;
while (true)
{
var sendSucceeded = false;
try
{
sendSucceeded = await SendBatchAsync(
connection,
apiKey,
batch.LogEvents,
sendFailureHandling != SendFailureHandling.Ignore);
}
catch (Exception ex)
{
if (sendFailureHandling != SendFailureHandling.Ignore)
{
Log.Error(ex, "Failed to send an event batch");
}
}
if (!sendSucceeded)
{
if (sendFailureHandling == SendFailureHandling.Fail)
{
return(1);
}
if (sendFailureHandling == SendFailureHandling.Retry)
{
var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000);
await Task.Delay(millisecondsDelay);
retries += 1;
continue;
}
}
retries = 0;
if (batch.IsLast)
{
break;
}
batch = await ReadBatchAsync(reader, filter, batchSize, invalidDataHandling);
}
return(0);
}
public override void Enable(OptionSet options)
{
options.Add("invalid-data=",
"Specify how invalid data is handled: `fail` (default) or `ignore`",
v => InvalidDataHandling = (InvalidDataHandling)Enum.Parse(typeof(InvalidDataHandling), v, ignoreCase: true));
}
public static async Task <int> ShipEvents(
SeqConnection connection,
LogEventReader reader,
List <ILogEventEnricher> enrichers,
InvalidDataHandling invalidDataHandling,
Func <LogEvent, bool> filter = null)
{
if (connection == null)
{
throw new ArgumentNullException(nameof(connection));
}
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
if (enrichers == null)
{
throw new ArgumentNullException(nameof(enrichers));
}
var batch = ReadBatch(reader, filter, BatchSize, invalidDataHandling);
while (batch.Length > 0)
{
StringContent content;
using (var builder = new StringWriter())
{
foreach (var evt in batch)
{
foreach (var enricher in enrichers)
{
enricher.Enrich(evt, null);
}
Formatter.Format(evt, builder);
}
content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediatType);
}
var result = await connection.Client.HttpClient.PostAsync(ApiConstants.IngestionEndpoint, content);
if (result.IsSuccessStatusCode)
{
batch = ReadBatch(reader, filter, BatchSize, invalidDataHandling);
continue;
}
var resultJson = await result.Content.ReadAsStringAsync();
if (!string.IsNullOrWhiteSpace(resultJson))
{
try
{
var error = JsonConvert.DeserializeObject <dynamic>(resultJson);
Log.Error("Failed with status code {StatusCode}: {ErrorMessage}",
result.StatusCode,
(string)error.ErrorMessage);
}
catch
{
// ignored
}
}
Log.Error("Failed with status code {StatusCode} ({ReasonPhrase})", result.StatusCode, result.ReasonPhrase);
return(1);
}
return(0);
}
public void SourceRead_InvalidFloatInSequence_WithElevation_ToTable(InvalidDataHandling dataHandlingStrategy)
{
//create the elevation configuration
var elevationRules = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "ElevationConfig.xml"));
File.WriteAllText(elevationRules.FullName,
@"<!DOCTYPE TagElevationRequestCollection
[
<!ELEMENT TagElevationRequestCollection (TagElevationRequest*)>
<!ELEMENT TagElevationRequest (ColumnName,ElevationPathway,Conditional?)>
<!ELEMENT ColumnName (#PCDATA)>
<!ELEMENT ElevationPathway (#PCDATA)>
<!ELEMENT Conditional (ConditionalPathway,ConditionalRegex)>
<!ELEMENT ConditionalPathway (#PCDATA)>
<!ELEMENT ConditionalRegex (#PCDATA)>
]>
<TagElevationRequestCollection>
<TagElevationRequest>
<ColumnName>WedgeAngleFloat</ColumnName>
<ElevationPathway>AcquisitionContextSequence->WedgeAngleFloat</ElevationPathway>
</TagElevationRequest>
</TagElevationRequestCollection>");
//setup the source reader
var source = new DicomDatasetCollectionSource();
source.InvalidDataHandlingStrategy = dataHandlingStrategy;
source.TagElevationConfigurationFile = elevationRules;
//don't load the sequence, just the elevation
source.TagBlacklist = new Regex("AcquisitionContextSequence");
//The dataset we are trying to load
var ds = new DicomDataset();
ds.Add(DicomTag.PatientAge, "123Y");
var sequence = new DicomSequence(DicomTag.AcquisitionContextSequence,
new DicomDataset()
{
{ DicomTag.WedgeAngleFloat, "3.40282347e+038" } //dodgy float in sequence (the sequence we are trying to elevate)
});
ds.Add(sequence);
var worklist = new ExplicitListDicomDatasetWorklist(new[] { ds }, "fish.dcm", new Dictionary <string, string> {
{ "MessageGuid", "123x321" }
});
source.PreInitialize(worklist, new ThrowImmediatelyDataLoadEventListener());
source.FilenameField = "RelFileName";
DataTable dt = null;
switch (dataHandlingStrategy)
{
case InvalidDataHandling.ThrowException:
Assert.Throws <OverflowException>(() => source.GetChunk(new ThrowImmediatelyDataLoadEventListener(), new GracefulCancellationToken()));
return;
case InvalidDataHandling.ConvertToNullAndWarn:
var tomem = new ToMemoryDataLoadEventListener(true);
dt = source.GetChunk(tomem, new GracefulCancellationToken());
Assert.AreEqual(DBNull.Value, dt.Rows[0]["WedgeAngleFloat"]);
//should be a warning about WedgeAngleFloat logged
var warning = tomem.EventsReceivedBySender.SelectMany(e => e.Value).Single(v => v.ProgressEventType == ProgressEventType.Warning);
Assert.IsTrue(warning.Message.Contains("WedgeAngleFloat"));
Assert.IsTrue(warning.Message.Contains("MessageGuid"));
Assert.IsTrue(warning.Message.Contains("123x321"));
Assert.IsTrue(warning.Message.Contains("fish.dcm"));
break;
default:
throw new ArgumentOutOfRangeException("dataHandlingStrategy");
}
Assert.AreEqual("123Y", dt.Rows[0]["PatientAge"]);
Assert.AreEqual("fish.dcm", dt.Rows[0]["RelFileName"]);
}
