public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender,
ILog logger)
{
try
{
var gaugingSummary = ParseGaugingSummaryFromFile(fileStream);
var gaugingSummaryProcessor = new GaugingSummaryProcessor(fieldDataResultsAppender, logger);
gaugingSummaryProcessor.ProcessGaugingSummary(gaugingSummary);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception ex)
when(ex is PocketGaugerZipFileMissingRequiredContentException || ex is PocketGaugerZipFileException)
{
return(ParseFileResult.CannotParse());
}
catch (PocketGaugerDataFormatException e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e));
}
catch (Exception e)
{
return(ParseFileResult.CannotParse(e));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender appender, ILog logger)
{
var channel = QRevSerializer.DeserializeNoThrow(fileStream, logger);
if (channel == null)
{
return(ParseFileResult.CannotParse());
}
var locationIdentifier = channel.SiteInformation?.SiteID?.Value;
if (string.IsNullOrWhiteSpace(locationIdentifier))
{
logger.Error("File can be parsed but there is no SiteID specified.");
return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing <Channel/SiteInformation/SiteID>"));
}
try
{
var trimmedLocationIdentifier = locationIdentifier.Trim();
var location = appender.GetLocationByIdentifier(trimmedLocationIdentifier);
return(ParseXmlRootNoThrow(location, channel, appender, logger));
}
catch (Exception exception)
{
logger.Error($"Cannot find location with identifier '{locationIdentifier}'.");
return(ParseFileResult.CannotParse(exception));
}
}
private ParseFileResult ParseArchive(ZipArchive zipArchive, LocationInfo locationInfo, List <IFieldDataPlugin> plugins)
{
foreach (var entry in zipArchive.Entries)
{
var isParsed = false;
foreach (var plugin in plugins)
{
var result = ParseEntry(entry, plugin, locationInfo);
if (result.Status == ParseFileStatus.CannotParse)
{
continue;
}
if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
Log.Error($"Plugin {PluginLoader.GetPluginNameAndVersion(plugin)} failed to parse '{entry.FullName}' with {result.Status}('{result.ErrorMessage}')");
return(result);
}
isParsed = true;
break;
}
if (!isParsed)
{
return(ParseFileResult.CannotParse());
}
}
// All entries were parsed by one of the plugins
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
try
{
var parser = new Parser(fieldDataResultsAppender, logger);
var eHsn = parser.LoadFromStream(fileStream);
if (eHsn == null)
{
return(ParseFileResult.CannotParse());
}
try
{
parser.Parse(eHsn);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
logger.Error($"File can be parsed but an error occurred: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
catch (Exception exception)
{
logger.Error($"Something went wrong: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.CannotParse(exception));
}
}
private void SummarizeResults(ParseFileResult result, AppendedResults appendedResults)
{
try
{
if (result.Status == Context.ExpectedStatus)
{
SummarizeExpectedResults(result, appendedResults);
}
else
{
SummarizeFailedResults(result, appendedResults);
}
}
catch (ExpectedException e)
{
if (Context.DataPaths.Count > 1)
{
Log.Error(e.Message);
++FailedCount;
}
else
{
throw;
}
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender appender, ILog logger)
{
var xmlRoot = SectionBySectionSerializer.DeserializeNoThrow(fileStream, logger);
if (xmlRoot == null)
{
return(ParseFileResult.CannotParse());
}
var stationNo = xmlRoot.Summary?.WinRiver_II_Section_by_Section_Summary?.Station_No;
if (string.IsNullOrWhiteSpace(stationNo))
{
logger.Error("File can be parsed but there is no Station_No specified.");
return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing Station_No."));
}
try
{
var trimmedLocationIdentifier = stationNo.Trim();
var location = appender.GetLocationByIdentifier(trimmedLocationIdentifier);
return(ParseXmlRootNoThrow(location, xmlRoot, appender, logger));
}
catch (Exception exception)
{
logger.Error($"Cannot find location with identifier {stationNo}.");
return(ParseFileResult.CannotParse(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation,
IFieldDataResultsAppender appender, ILog logger)
{
var parser = new AquaCalcCsvParser(fileStream)
{
Settings = appender.GetPluginConfigurations()
};
if (!parser.CanParse())
{
return(ParseFileResult.CannotParse());
}
var parsedData = parser.Parse();
if (targetLocation == null)
{
targetLocation = appender.GetLocationByIdentifier(parsedData.GageId);
}
var visitDetails = new VisitMapper(parsedData).GetVisitDetails(targetLocation.UtcOffset);
logger.Info($"Got visit details: '{visitDetails.StartDate:s}@{targetLocation.LocationIdentifier}'");
var visitInfo = appender.AddFieldVisit(targetLocation, visitDetails);
AppendActivity(appender, parsedData, visitInfo, logger);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private void SummarizeResults(ParseFileResult result, AppendedResults appendedResults)
{
if (!result.Parsed)
{
if (result.Status == ParseFileStatus.CannotParse)
{
if (!string.IsNullOrEmpty(result.ErrorMessage))
{
Log.Error($"Can't parse '{DataPath}'. {result.ErrorMessage}");
}
else
{
Log.Warn($"File '{DataPath}' is not parsed by the plugin.");
}
}
else
{
Log.Error($"Result: Parsed={result.Parsed} Status={result.Status} ErrorMessage={result.ErrorMessage}");
}
}
else
{
if (!appendedResults.AppendedVisits.Any())
{
Log.Warn("File was parsed but no visits were appended.");
}
else
{
Log.Info($"Successfully parsed {appendedResults.AppendedVisits.Count} visits.");
}
}
}
private void SummarizeExpectedResults(ParseFileResult result, AppendedResults appendedResults)
{
if (result.Parsed)
{
if (!appendedResults.AppendedVisits.Any())
{
throw new ExpectedException("File was parsed but no visits were appended.");
}
_log.Info($"Successfully parsed {appendedResults.AppendedVisits.Count} visits.");
}
else
{
var actualError = result.ErrorMessage ?? string.Empty;
var expectedError = Context.ExpectedError ?? string.Empty;
if (!actualError.Equals(expectedError))
{
throw new ExpectedException(
$"Expected an error message of '{expectedError}' but received '{actualError}' instead.");
}
_log.Info($"ParsedResult.Status == {Context.ExpectedStatus} with Error='{Context.ExpectedError}' as expected.");
}
}
public ParseFileResult ParseFile(Stream fileStream)
{
CrossSectionSurvey parsedFileContents;
try
{
parsedFileContents = ProcessFileStream(fileStream);
}
catch (CrossSectionCsvFormatException)
{
return(ParseFileResult.CannotParse());
}
catch (Exception e)
{
return(ParseFileResult.CannotParse(e));
}
try
{
return(ProcessParsedFileContents(parsedFileContents));
}
catch (Exception e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e));
}
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
var jsonText = ReadTextFromStream(stream);
if (jsonText == null)
{
return(ParseFileResult.CannotParse());
}
try
{
LocationInfo = locationInfo;
AppendedResults = ParseJson(jsonText);
if (AppendedResults == null)
{
return(ParseFileResult.CannotParse());
}
return(MapToFramework());
}
catch (Exception exception)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
private ParseFileResult ParseEntry(ZipArchiveEntry entry, IFieldDataPlugin plugin, LocationInfo locationInfo)
{
using (var entryStream = entry.Open())
using (var reader = new BinaryReader(entryStream))
using (var memoryStream = new MemoryStream(reader.ReadBytes((int)entry.Length)))
{
var proxyLog = ProxyLog.Create(Log, plugin, entry);
var result = locationInfo == null
? plugin.ParseFile(memoryStream, ResultsAppender, proxyLog)
: plugin.ParseFile(memoryStream, locationInfo, ResultsAppender, proxyLog);
switch (result.Status)
{
case ParseFileStatus.CannotParse:
result = ParseFileResult.CannotParse($"{proxyLog.Prefix}: {result.ErrorMessage}");
break;
case ParseFileStatus.SuccessfullyParsedButDataInvalid:
result = ParseFileResult.SuccessfullyParsedButDataInvalid($"{proxyLog.Prefix}: {result.ErrorMessage}");
break;
}
return(result);
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger)
{
var xmlRoot = QRevSerializer.DeserializeNoThrow(fileStream, logger);
return(xmlRoot == null
? ParseFileResult.CannotParse()
: ParseXmlRootNoThrow(targetLocation, xmlRoot, appender, logger));
}
private ParseFileResult ProcessParsedFileContents(CrossSectionSurvey parsedFileContents)
{
var crossSectionSurvey = MapToCrossSectionSurvey(parsedFileContents);
var locationIdentifier = GetLocationIdentifier(parsedFileContents);
var fieldVisitInfo = _fieldVisitHandler.GetFieldVisit(locationIdentifier, crossSectionSurvey);
_fieldDataResultsAppender.AddCrossSectionSurvey(fieldVisitInfo, crossSectionSurvey);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private void SummarizeResults(ParseFileResult result, AppendedResults appendedResults)
{
if (result.Status == Context.ExpectedStatus)
{
SummarizeExpectedResults(result, appendedResults);
}
else
{
SummarizeFailedResults(result, appendedResults);
}
}
private static ParseFileResult GetFieldDataResults(FieldDataResultsGenerator resultsGenerator, FieldVisitRecord record)
{
try
{
resultsGenerator.GenerateFieldDataResults(record);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception ex)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(ex));
}
}
private ParseFileResult DummyResult(IFieldDataResultsAppender appender, LocationInfo targetLocation)
{
if (targetLocation == null)
{
targetLocation = appender.GetLocationByIdentifier("Foo");
}
appender.AddFieldVisit(targetLocation, new FieldVisitDetails(new DateTimeInterval(
new DateTimeOffset(2000, 1, 1, 0, 0, 0, TimeSpan.Zero),
TimeSpan.FromHours(1))));
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation,
IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var fieldVisit = ReadFile(fileStream);
//Only save data if it matches the targetLocation.
if (!targetLocation.LocationIdentifier.Equals(fieldVisit.LocationIdentifier))
{
return(ParseFileResult.CannotParse());
}
var resultsGenerator = new FieldDataResultsGenerator(fieldDataResultsAppender, targetLocation, logger);
return(GetFieldDataResults(resultsGenerator, fieldVisit));
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
try
{
var zipArchive = GetZipArchiveOrNull(stream);
if (zipArchive == null)
{
return(ParseFileResult.CannotParse());
}
using (zipArchive)
{
if (zipArchive.Entries.Any(e => e.Name != e.FullName))
{
return(ParseFileResult.CannotParse());
}
if (!zipArchive.Entries.Any())
{
return(ParseFileResult.CannotParse());
}
var plugins = LoadPlugins();
if (!plugins.Any())
{
return(ParseFileResult.CannotParse());
}
var result = ParseArchive(zipArchive, locationInfo, plugins);
if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
return(result);
}
// Now append all the visits
ResultsAppender.AppendAllResults();
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
catch (Exception exception)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
private ParseFileResult ParseXmlRootNoThrow(LocationInfo location, Channel channel, IFieldDataResultsAppender appender, ILog logger)
{
try
{
var parser = new Parser(location, appender, logger);
parser.Parse(channel);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
logger.Error($"Something went wrong: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
private ParseFileResult MapToFramework()
{
if (!AppendedResults.AppendedVisits.Any())
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid("No visits contained in the JSON document."));
}
Log.Info($"Importing {AppendedResults.AppendedVisits.Count} visits parsed by '{AppendedResults.PluginAssemblyQualifiedTypeName}' using '{AppendedResults.FrameworkAssemblyQualifiedName}'");
foreach (var visit in AppendedResults.AppendedVisits)
{
AppendVisit(visit);
}
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private ParseFileResult ParseDataFileWithLocale(Configuration configuration, byte[] csvBytes)
{
using (LocaleScope.WithLocale(configuration.LocaleName))
{
try
{
return(ParseDataFile(configuration, csvBytes));
}
catch (Exception exception)
{
Log.Error($"Configuration '{configuration.Id}' failed to parse file: {exception.Message}");
return(ParseFileResult.CannotParse(exception));
}
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var fieldVisit = ReadFile(fileStream);
try
{
var location = fieldDataResultsAppender.GetLocationByIdentifier(fieldVisit.LocationIdentifier);
logger.Info($"Parsing field data for location {location.LocationIdentifier}");
var resultsGenerator = new FieldDataResultsGenerator(fieldDataResultsAppender, location, logger);
return(GetFieldDataResults(resultsGenerator, fieldVisit));
}
catch (Exception)
{
logger.Error($"Cannot parse file, location {fieldVisit.LocationIdentifier} not found");
return(ParseFileResult.CannotParse());
}
}
private ParseFileResult AppendResults(LocationInfo locationInfo)
{
try
{
UnitSystem = CreateUnitSystem();
var visit = CreateVisit(locationInfo);
var dischargeActivity = CreateDischargeActivity(visit);
AddGageHeightMeasurement(dischargeActivity);
var manualGauging = CreateManualGauging(dischargeActivity);
var startStation = DataFile.Stations.First();
var endStation = DataFile.Stations.Last();
foreach (var station in DataFile.Stations)
{
manualGauging.Verticals.Add(CreateVertical(manualGauging.Verticals.Count, station, startStation, endStation));
}
AdjustUnknownTotalDischargePortion(manualGauging);
_resultsAppender.AddDischargeActivity(visit, dischargeActivity);
manualGauging.MeterCalibration = manualGauging
.Verticals
.Select(v => v.VelocityObservation.MeterCalibration)
.FirstOrDefault(mc => mc != null);
AddTemperatureReadings(visit);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
// Something has gone sideways rather hard. The framework won't log the exception's stack trace
// so we explicitly do that here, to help track down any bugs.
LogException("Parsing error", exception);
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var eHsn = new Parser()
.LoadFromStream(fileStream);
if (eHsn == null)
{
return(ParseFileResult.CannotParse());
}
var targetLocation = fieldDataResultsAppender.GetLocationByIdentifier("MyDummyLocation");
var now = DateTimeOffset.UtcNow;
var visit = fieldDataResultsAppender.AddFieldVisit(targetLocation,
new FieldVisitDetails(new DateTimeInterval(now.AddHours(-1), now)));
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
private void SummarizeFailedResults(ParseFileResult result, AppendedResults appendedResults)
{
if (result.Parsed)
{
throw new ExpectedException(
$"File was parsed successfully with {appendedResults.AppendedVisits.Count} visits appended.");
}
if (result.Status != ParseFileStatus.CannotParse)
{
throw new ExpectedException(
$"Result: Parsed={result.Parsed} Status={result.Status} ErrorMessage={result.ErrorMessage}");
}
if (!string.IsNullOrEmpty(result.ErrorMessage))
{
throw new ExpectedException($"Can't parse '{Context.DataPath}'. {result.ErrorMessage}");
}
throw new ExpectedException($"File '{Context.DataPath}' is not parsed by the plugin.");
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo)
{
DataFile = GetDataFile(stream);
if (DataFile == null)
{
return(ParseFileResult.CannotParse());
}
if (locationInfo == null)
{
if (string.IsNullOrEmpty(DataFile.Properties.SiteNumber))
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid($"No {nameof(DataFile.Properties.SiteNumber)} property is set, so no AQUARIUS location can be inferred. Try uploading the file directly to a location."));
}
locationInfo = _resultsAppender.GetLocationByIdentifier(DataFile.Properties.SiteNumber);
}
return(AppendResults(locationInfo));
}
private ParseFileResult ParseDataFileWithLocale(Configuration configuration, byte[] csvBytes)
{
using (LocaleScope.WithLocale(configuration.LocaleName))
{
try
{
if (new ExcelParser().TryLoadSingleSheet(configuration, csvBytes, out var sheetBytes))
{
csvBytes = sheetBytes;
}
return(ParseDataFile(configuration, csvBytes));
}
catch (Exception exception)
{
Log.Error($"Configuration '{configuration.Id}' failed to parse file: {exception.Message}");
return(ParseFileResult.CannotParse(exception));
}
}
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
var csvBytes = ReadBytesFromStream(stream);
if (csvBytes == null)
{
return(ParseFileResult.CannotParse());
}
try
{
LocationInfo = locationInfo;
var configurations = LoadConfigurations();
using (ResultsAppender)
{
var result = ParseFileResult.CannotParse();
foreach (var configuration in configurations)
{
result = ParseDataFileWithLocale(configuration, csvBytes);
if (result.Status == ParseFileStatus.CannotParse)
{
continue;
}
return(result);
}
return(result);
}
}
catch (Exception exception)
{
return(ParseFileResult.CannotParse(exception));
}
}
private bool TryParseFile(byte[] dataBytes, out ParseFileResultWithAttachments result)
{
result = new ParseFileResultWithAttachments();
try
{
using (var stream = new MemoryStream(dataBytes))
{
result.Result = LocationInfo != null
? Plugin.ParseFile(stream, LocationInfo, Appender, Logger)
: Plugin.ParseFile(stream, Appender, Logger);
}
return(true);
}
catch (Exception exception)
{
result.Result = ParseFileResult.CannotParse(exception);
return(false);
}
}
