public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender,
ILog logger)
{
try
{
var gaugingSummary = ParseGaugingSummaryFromFile(fileStream);
var gaugingSummaryProcessor = new GaugingSummaryProcessor(fieldDataResultsAppender, logger);
gaugingSummaryProcessor.ProcessGaugingSummary(gaugingSummary);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception ex)
when(ex is PocketGaugerZipFileMissingRequiredContentException || ex is PocketGaugerZipFileException)
{
return(ParseFileResult.CannotParse());
}
catch (PocketGaugerDataFormatException e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e));
}
catch (Exception e)
{
return(ParseFileResult.CannotParse(e));
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation,
IFieldDataResultsAppender appender, ILog logger)
{
var parser = new AquaCalcCsvParser(fileStream)
{
Settings = appender.GetPluginConfigurations()
};
if (!parser.CanParse())
{
return(ParseFileResult.CannotParse());
}
var parsedData = parser.Parse();
if (targetLocation == null)
{
targetLocation = appender.GetLocationByIdentifier(parsedData.GageId);
}
var visitDetails = new VisitMapper(parsedData).GetVisitDetails(targetLocation.UtcOffset);
logger.Info($"Got visit details: '{visitDetails.StartDate:s}@{targetLocation.LocationIdentifier}'");
var visitInfo = appender.AddFieldVisit(targetLocation, visitDetails);
AppendActivity(appender, parsedData, visitInfo, logger);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private ParseFileResult ParseArchive(ZipArchive zipArchive, LocationInfo locationInfo, List <IFieldDataPlugin> plugins)
{
foreach (var entry in zipArchive.Entries)
{
var isParsed = false;
foreach (var plugin in plugins)
{
var result = ParseEntry(entry, plugin, locationInfo);
if (result.Status == ParseFileStatus.CannotParse)
{
continue;
}
if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
Log.Error($"Plugin {PluginLoader.GetPluginNameAndVersion(plugin)} failed to parse '{entry.FullName}' with {result.Status}('{result.ErrorMessage}')");
return(result);
}
isParsed = true;
break;
}
if (!isParsed)
{
return(ParseFileResult.CannotParse());
}
}
// All entries were parsed by one of the plugins
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
try
{
var parser = new Parser(fieldDataResultsAppender, logger);
var eHsn = parser.LoadFromStream(fileStream);
if (eHsn == null)
{
return(ParseFileResult.CannotParse());
}
try
{
parser.Parse(eHsn);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
logger.Error($"File can be parsed but an error occurred: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
catch (Exception exception)
{
logger.Error($"Something went wrong: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.CannotParse(exception));
}
}
private ParseFileResult ProcessParsedFileContents(CrossSectionSurvey parsedFileContents)
{
var crossSectionSurvey = MapToCrossSectionSurvey(parsedFileContents);
var locationIdentifier = GetLocationIdentifier(parsedFileContents);
var fieldVisitInfo = _fieldVisitHandler.GetFieldVisit(locationIdentifier, crossSectionSurvey);
_fieldDataResultsAppender.AddCrossSectionSurvey(fieldVisitInfo, crossSectionSurvey);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private static ParseFileResult GetFieldDataResults(FieldDataResultsGenerator resultsGenerator, FieldVisitRecord record)
{
try
{
resultsGenerator.GenerateFieldDataResults(record);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception ex)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(ex));
}
}
private ParseFileResult DummyResult(IFieldDataResultsAppender appender, LocationInfo targetLocation)
{
if (targetLocation == null)
{
targetLocation = appender.GetLocationByIdentifier("Foo");
}
appender.AddFieldVisit(targetLocation, new FieldVisitDetails(new DateTimeInterval(
new DateTimeOffset(2000, 1, 1, 0, 0, 0, TimeSpan.Zero),
TimeSpan.FromHours(1))));
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
try
{
var zipArchive = GetZipArchiveOrNull(stream);
if (zipArchive == null)
{
return(ParseFileResult.CannotParse());
}
using (zipArchive)
{
if (zipArchive.Entries.Any(e => e.Name != e.FullName))
{
return(ParseFileResult.CannotParse());
}
if (!zipArchive.Entries.Any())
{
return(ParseFileResult.CannotParse());
}
var plugins = LoadPlugins();
if (!plugins.Any())
{
return(ParseFileResult.CannotParse());
}
var result = ParseArchive(zipArchive, locationInfo, plugins);
if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
return(result);
}
// Now append all the visits
ResultsAppender.AppendAllResults();
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
catch (Exception exception)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
private ParseFileResult MapToFramework()
{
if (!AppendedResults.AppendedVisits.Any())
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid("No visits contained in the JSON document."));
}
Log.Info($"Importing {AppendedResults.AppendedVisits.Count} visits parsed by '{AppendedResults.PluginAssemblyQualifiedTypeName}' using '{AppendedResults.FrameworkAssemblyQualifiedName}'");
foreach (var visit in AppendedResults.AppendedVisits)
{
AppendVisit(visit);
}
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private ParseFileResult ParseXmlRootNoThrow(LocationInfo location, Channel channel, IFieldDataResultsAppender appender, ILog logger)
{
try
{
var parser = new Parser(location, appender, logger);
parser.Parse(channel);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
logger.Error($"Something went wrong: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var eHsn = new Parser()
.LoadFromStream(fileStream);
if (eHsn == null)
{
return(ParseFileResult.CannotParse());
}
var targetLocation = fieldDataResultsAppender.GetLocationByIdentifier("MyDummyLocation");
var now = DateTimeOffset.UtcNow;
var visit = fieldDataResultsAppender.AddFieldVisit(targetLocation,
new FieldVisitDetails(new DateTimeInterval(now.AddHours(-1), now)));
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private ParseFileResult AppendResults(LocationInfo locationInfo)
{
try
{
UnitSystem = CreateUnitSystem();
var visit = CreateVisit(locationInfo);
var dischargeActivity = CreateDischargeActivity(visit);
AddGageHeightMeasurement(dischargeActivity);
var manualGauging = CreateManualGauging(dischargeActivity);
var startStation = DataFile.Stations.First();
var endStation = DataFile.Stations.Last();
foreach (var station in DataFile.Stations)
{
manualGauging.Verticals.Add(CreateVertical(manualGauging.Verticals.Count, station, startStation, endStation));
}
AdjustUnknownTotalDischargePortion(manualGauging);
_resultsAppender.AddDischargeActivity(visit, dischargeActivity);
manualGauging.MeterCalibration = manualGauging
.Verticals
.Select(v => v.VelocityObservation.MeterCalibration)
.FirstOrDefault(mc => mc != null);
AddTemperatureReadings(visit);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
// Something has gone sideways rather hard. The framework won't log the exception's stack trace
// so we explicitly do that here, to help track down any bugs.
LogException("Parsing error", exception);
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
_log = logger;
try
{
using (var delayedAppender = new DelayedAppender(fieldDataResultsAppender))
{
_fieldDataResultsAppender = delayedAppender;
var parsedRecords = _parser.ParseInputData(fileStream);
if (parsedRecords == null)
{
return(ParseFileResult.CannotParse(NoRecordsInInputFile));
}
if (_parser.Errors.Any())
{
if (_parser.ValidRecords > 0)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(
$"{InputFileContainsInvalidRecords}: {_parser.Errors.Length} errors:\n{string.Join("\n", _parser.Errors.Take(3))}"));
}
return(ParseFileResult.CannotParse());
}
_log.Info($"Parsed {_parser.ValidRecords} rows from input file.");
SaveRecords(parsedRecords);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
catch (Exception e)
{
_log.Error($"Failed to parse file; {e.Message}\n{e.StackTrace}");
return(ParseFileResult.CannotParse(e));
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger)
{
try
{
Config = new ConfigLoader(appender)
.Load();
var summary = GetSummary(fileStream, logger);
if (summary == null)
{
return(ParseFileResult.CannotParse());
}
if (targetLocation == null)
{
var locationIdentifier = ExtractLocationIdentifier(summary.StationName);
if (string.IsNullOrEmpty(locationIdentifier))
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing station name"));
}
targetLocation = appender.GetLocationByIdentifier(locationIdentifier);
}
var parser = new Parser(Config, targetLocation, appender, logger);
parser.Parse(summary);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e.Message));
}
}
private ParseFileResult ParseSurvey(string csvText)
{
var validator = new SurveyValidator {
Survey = Survey
};
var lineCount = 0;
using (var reader = new StringReader(csvText))
{
var rowParser = GetCsvParser(reader);
for (; !rowParser.EndOfData; ++lineCount)
{
LineNumber = rowParser.LineNumber;
try
{
Fields = rowParser.ReadFields();
}
catch (Exception)
{
if (lineCount == 0)
{
// We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document.
return(ParseFileResult.CannotParse());
}
}
if (Fields == null)
{
continue;
}
if (lineCount == 0 && Survey.FirstLineIsHeader)
{
try
{
HeaderMap = validator.BuildHeaderMap(Fields);
continue;
}
catch (Exception exception)
{
// Most Survey123 files have a header.
// So a problem mapping the header is a strong indicator that this CSV file is not intended for us.
if (exception is AllHeadersMissingException)
{
// When all headers are missing, then we should exit without logging anything special.
// We'll just let the other plugins have a turn
}
else
{
// Some of the headers matched, so log a warning before returning CannotParse.
// This might be the only hint in the log that the survey configuration JSON is incorrect.
Log.Info($"Partial headers detected: {exception.Message}");
}
return(ParseFileResult.CannotParse());
}
}
try
{
ParseRow();
}
catch (Exception exception)
{
if (!ResultsAppender.AnyResultsAppended)
{
throw;
}
Log.Error($"Ignoring invalid CSV row: {exception.Message}");
}
}
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
private ParseFileResult ParseDataFile(Configuration configuration, byte[] csvBytes)
{
var csvText = ReadTextFromBytes(configuration, csvBytes);
var rowParser = new RowParser(configuration, ResultsAppender, LocationInfo);
var(prefaceLines, dataRowReader) = ExtractPrefaceLines(configuration, csvText);
rowParser.AddPrefaceLines(prefaceLines);
if (!rowParser.IsPrefaceValid())
{
return(ParseFileResult.CannotParse());
}
var dataRowCount = 0;
using (var reader = dataRowReader)
{
var separator = configuration.Separator ?? DefaultFieldSeparator;
var fieldParser = GetCsvParser(reader, separator);
while (!fieldParser.EndOfData)
{
rowParser.LineNumber = rowParser.PrefaceLineCount + fieldParser.LineNumber;
string[] fields = null;
try
{
fields = fieldParser.ReadFields();
}
catch (Exception)
{
if (dataRowCount == 0)
{
// We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document.
return(ParseFileResult.CannotParse());
}
}
if (fields == null)
{
continue;
}
if (fields.Length > 0 && fields.All(string.IsNullOrEmpty))
{
continue;
}
if (fields.Length > 0 && !string.IsNullOrEmpty(configuration.CommentLinePrefix) && fields[0].StartsWith(configuration.CommentLinePrefix))
{
continue;
}
if (dataRowCount == 0 && configuration.IsHeaderRowRequired)
{
try
{
if (rowParser.IsHeaderFullyParsed(fields))
{
++dataRowCount;
}
continue;
}
catch (Exception exception)
{
// Most CSV files have a header.
// So a problem mapping the header is a strong indicator that this CSV file is not intended for us.
if (exception is AllHeadersMissingException)
{
// When all headers are missing, then we should exit without logging anything special.
// We'll just let the other plugins have a turn
}
else
{
// Some of the headers matched, so log a warning before returning CannotParse.
// This might be the only hint in the log that the configuration is incorrect.
Log.Info($"Partial headers detected: {exception.Message}");
}
return(ParseFileResult.CannotParse());
}
}
if (IsFooterDetected(configuration, () => string.Join(separator, fields)))
{
break;
}
try
{
rowParser.Parse(fields);
}
catch (Exception exception)
{
if (!ResultsAppender.AnyResultsAppended)
{
throw;
}
Log.Error($"Ignoring invalid CSV row: {exception.Message}");
}
++dataRowCount;
}
if (dataRowCount == 0)
{
if (configuration.NoDataRowsExpected)
{
// When all the data comes from the preface, then this will parse the preface context
rowParser.Parse(new string[0]);
}
else
{
// No rows were parsed.
if (rowParser.RemainingHeaderLines > 0)
{
return(ParseFileResult.CannotParse("No matching header row was detected."));
}
return(ParseFileResult.CannotParse());
}
}
Log.Info($"Configuration='{configuration.Id}' Priority={configuration.Priority} parsed {ResultsAppender.SummaryInfo()}.");
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}