private void AppendVisit(FieldVisitInfo importedVisit)
{
var locationInfo = LocationInfo ?? ResultsAppender.GetLocationByIdentifier(importedVisit.LocationInfo.LocationIdentifier);
var visit = ResultsAppender.AddFieldVisit(locationInfo, importedVisit.FieldVisitDetails);
foreach (var controlCondition in importedVisit.ControlConditions)
{
ResultsAppender.AddControlCondition(visit, controlCondition);
}
foreach (var crossSectionSurvey in importedVisit.CrossSectionSurveys)
{
UpgradeCrossSectionSurvey(crossSectionSurvey);
ResultsAppender.AddCrossSectionSurvey(visit, crossSectionSurvey);
}
foreach (var dischargeActivity in importedVisit.DischargeActivities)
{
ResultsAppender.AddDischargeActivity(visit, dischargeActivity);
}
foreach (var levelSurvey in importedVisit.LevelSurveys)
{
ResultsAppender.AddLevelSurvey(visit, levelSurvey);
}
foreach (var reading in importedVisit.Readings)
{
ResultsAppender.AddReading(visit, reading);
}
}
private List <Configuration> LoadConfigurations()
{
var pluginConfigurations = ResultsAppender.GetPluginConfigurations();
if (!pluginConfigurations.Any())
{
Log.Error($"No configurations loaded.");
return(new List <Configuration>());
}
var configurationLoader = new ConfigurationLoader
{
Log = Log
};
var allConfigurations = pluginConfigurations
.Select(kvp => configurationLoader.Load(kvp.Key, kvp.Value))
.Where(configuration => configuration != null)
.ToList();
var configurations = allConfigurations
.Where(configuration => !configuration.IsDisabled)
.OrderBy(configuration => configuration.Priority)
.ToList();
if (!configurations.Any())
{
Log.Error($"No enabled configurations found. ({pluginConfigurations.Count} were disabled or invalid).");
return(new List <Configuration>());
}
configurations = configurations
.Where(configuration => LocationInfo != null || configuration.Location != null)
.ToList();
foreach (var configuration in configurations)
{
new ConfigurationValidator
{
LocationInfo = LocationInfo,
Configuration = configuration
}
.Validate();
}
return(configurations);
}
private void ParseRow()
{
var locationIdentifier = GetColumnValue(Survey.LocationColumn);
if (Survey.LocationAliases.TryGetValue(locationIdentifier, out var aliasedIdentifier))
{
locationIdentifier = aliasedIdentifier;
}
var locationInfo = LocationInfo ?? ResultsAppender.GetLocationByIdentifier(locationIdentifier);
var comments = MergeTextColumns(Survey.CommentColumns);
var party = MergeTextColumns(Survey.PartyColumns);
var timestamp = ParseTimestamp(locationInfo);
var readings = Survey
.ReadingColumns
.Select(ParseReading)
.Where(r => r != null)
.Select(r =>
{
r.DateTimeOffset = timestamp;
return(r);
})
.ToList();
var fieldVisitInfo = ResultsAppender.AddFieldVisit(locationInfo,
new FieldVisitDetails(new DateTimeInterval(timestamp, TimeSpan.Zero))
{
Comments = comments,
Party = party
});
foreach (var reading in readings)
{
ResultsAppender.AddReading(fieldVisitInfo, reading);
}
}
private ParseFileResult ParseDataFile(Configuration configuration, byte[] csvBytes)
{
var csvText = ReadTextFromBytes(configuration, csvBytes);
var rowParser = new RowParser(configuration, ResultsAppender, LocationInfo);
var(prefaceLines, dataRowReader) = ExtractPrefaceLines(configuration, csvText);
rowParser.AddPrefaceLines(prefaceLines);
if (!rowParser.IsPrefaceValid())
{
return(ParseFileResult.CannotParse());
}
var dataRowCount = 0;
using (var reader = dataRowReader)
{
var separator = configuration.Separator ?? DefaultFieldSeparator;
var fieldParser = GetCsvParser(reader, separator);
while (!fieldParser.EndOfData)
{
rowParser.LineNumber = rowParser.PrefaceLineCount + fieldParser.LineNumber;
string[] fields = null;
try
{
fields = fieldParser.ReadFields();
}
catch (Exception)
{
if (dataRowCount == 0)
{
// We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document.
return(ParseFileResult.CannotParse());
}
}
if (fields == null)
{
continue;
}
if (fields.Length > 0 && fields.All(string.IsNullOrEmpty))
{
continue;
}
if (fields.Length > 0 && !string.IsNullOrEmpty(configuration.CommentLinePrefix) && fields[0].StartsWith(configuration.CommentLinePrefix))
{
continue;
}
if (dataRowCount == 0 && configuration.IsHeaderRowRequired)
{
try
{
if (rowParser.IsHeaderFullyParsed(fields))
{
++dataRowCount;
}
continue;
}
catch (Exception exception)
{
// Most CSV files have a header.
// So a problem mapping the header is a strong indicator that this CSV file is not intended for us.
if (exception is AllHeadersMissingException)
{
// When all headers are missing, then we should exit without logging anything special.
// We'll just let the other plugins have a turn
}
else
{
// Some of the headers matched, so log a warning before returning CannotParse.
// This might be the only hint in the log that the configuration is incorrect.
Log.Info($"Partial headers detected: {exception.Message}");
}
return(ParseFileResult.CannotParse());
}
}
if (IsFooterDetected(configuration, () => string.Join(separator, fields)))
{
break;
}
try
{
rowParser.Parse(fields);
}
catch (Exception exception)
{
if (!ResultsAppender.AnyResultsAppended)
{
throw;
}
Log.Error($"Ignoring invalid CSV row: {exception.Message}");
}
++dataRowCount;
}
if (dataRowCount == 0)
{
if (configuration.NoDataRowsExpected)
{
// When all the data comes from the preface, then this will parse the preface context
rowParser.Parse(new string[0]);
}
else
{
// No rows were parsed.
if (rowParser.RemainingHeaderLines > 0)
{
return(ParseFileResult.CannotParse("No matching header row was detected."));
}
return(ParseFileResult.CannotParse());
}
}
Log.Info($"Configuration='{configuration.Id}' Priority={configuration.Priority} parsed {ResultsAppender.SummaryInfo()}.");
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
