private ParseFileResult ParseArchive(ZipArchive zipArchive, LocationInfo locationInfo, List <IFieldDataPlugin> plugins)
{
foreach (var entry in zipArchive.Entries)
{
var isParsed = false;
foreach (var plugin in plugins)
{
var result = ParseEntry(entry, plugin, locationInfo);
if (result.Status == ParseFileStatus.CannotParse)
{
continue;
}
if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
Log.Error($"Plugin {PluginLoader.GetPluginNameAndVersion(plugin)} failed to parse '{entry.FullName}' with {result.Status}('{result.ErrorMessage}')");
return(result);
}
isParsed = true;
break;
}
if (!isParsed)
{
return(ParseFileResult.CannotParse());
}
}
// All entries were parsed by one of the plugins
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
try
{
var parser = new Parser(fieldDataResultsAppender, logger);
var eHsn = parser.LoadFromStream(fileStream);
if (eHsn == null)
{
return(ParseFileResult.CannotParse());
}
try
{
parser.Parse(eHsn);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception exception)
{
logger.Error($"File can be parsed but an error occurred: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
catch (Exception exception)
{
logger.Error($"Something went wrong: {exception.Message}\n{exception.StackTrace}");
return(ParseFileResult.CannotParse(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender,
ILog logger)
{
try
{
var gaugingSummary = ParseGaugingSummaryFromFile(fileStream);
var gaugingSummaryProcessor = new GaugingSummaryProcessor(fieldDataResultsAppender, logger);
gaugingSummaryProcessor.ProcessGaugingSummary(gaugingSummary);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception ex)
when(ex is PocketGaugerZipFileMissingRequiredContentException || ex is PocketGaugerZipFileException)
{
return(ParseFileResult.CannotParse());
}
catch (PocketGaugerDataFormatException e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e));
}
catch (Exception e)
{
return(ParseFileResult.CannotParse(e));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender appender, ILog logger)
{
var channel = QRevSerializer.DeserializeNoThrow(fileStream, logger);
if (channel == null)
{
return(ParseFileResult.CannotParse());
}
var locationIdentifier = channel.SiteInformation?.SiteID?.Value;
if (string.IsNullOrWhiteSpace(locationIdentifier))
{
logger.Error("File can be parsed but there is no SiteID specified.");
return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing <Channel/SiteInformation/SiteID>"));
}
try
{
var trimmedLocationIdentifier = locationIdentifier.Trim();
var location = appender.GetLocationByIdentifier(trimmedLocationIdentifier);
return(ParseXmlRootNoThrow(location, channel, appender, logger));
}
catch (Exception exception)
{
logger.Error($"Cannot find location with identifier '{locationIdentifier}'.");
return(ParseFileResult.CannotParse(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender appender, ILog logger)
{
var xmlRoot = SectionBySectionSerializer.DeserializeNoThrow(fileStream, logger);
if (xmlRoot == null)
{
return(ParseFileResult.CannotParse());
}
var stationNo = xmlRoot.Summary?.WinRiver_II_Section_by_Section_Summary?.Station_No;
if (string.IsNullOrWhiteSpace(stationNo))
{
logger.Error("File can be parsed but there is no Station_No specified.");
return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing Station_No."));
}
try
{
var trimmedLocationIdentifier = stationNo.Trim();
var location = appender.GetLocationByIdentifier(trimmedLocationIdentifier);
return(ParseXmlRootNoThrow(location, xmlRoot, appender, logger));
}
catch (Exception exception)
{
logger.Error($"Cannot find location with identifier {stationNo}.");
return(ParseFileResult.CannotParse(exception));
}
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
var jsonText = ReadTextFromStream(stream);
if (jsonText == null)
{
return(ParseFileResult.CannotParse());
}
try
{
LocationInfo = locationInfo;
AppendedResults = ParseJson(jsonText);
if (AppendedResults == null)
{
return(ParseFileResult.CannotParse());
}
return(MapToFramework());
}
catch (Exception exception)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
private ParseFileResult ParseEntry(ZipArchiveEntry entry, IFieldDataPlugin plugin, LocationInfo locationInfo)
{
using (var entryStream = entry.Open())
using (var reader = new BinaryReader(entryStream))
using (var memoryStream = new MemoryStream(reader.ReadBytes((int)entry.Length)))
{
var proxyLog = ProxyLog.Create(Log, plugin, entry);
var result = locationInfo == null
? plugin.ParseFile(memoryStream, ResultsAppender, proxyLog)
: plugin.ParseFile(memoryStream, locationInfo, ResultsAppender, proxyLog);
switch (result.Status)
{
case ParseFileStatus.CannotParse:
result = ParseFileResult.CannotParse($"{proxyLog.Prefix}: {result.ErrorMessage}");
break;
case ParseFileStatus.SuccessfullyParsedButDataInvalid:
result = ParseFileResult.SuccessfullyParsedButDataInvalid($"{proxyLog.Prefix}: {result.ErrorMessage}");
break;
}
return(result);
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation,
IFieldDataResultsAppender appender, ILog logger)
{
var parser = new AquaCalcCsvParser(fileStream)
{
Settings = appender.GetPluginConfigurations()
};
if (!parser.CanParse())
{
return(ParseFileResult.CannotParse());
}
var parsedData = parser.Parse();
if (targetLocation == null)
{
targetLocation = appender.GetLocationByIdentifier(parsedData.GageId);
}
var visitDetails = new VisitMapper(parsedData).GetVisitDetails(targetLocation.UtcOffset);
logger.Info($"Got visit details: '{visitDetails.StartDate:s}@{targetLocation.LocationIdentifier}'");
var visitInfo = appender.AddFieldVisit(targetLocation, visitDetails);
AppendActivity(appender, parsedData, visitInfo, logger);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult ParseFile(Stream fileStream)
{
CrossSectionSurvey parsedFileContents;
try
{
parsedFileContents = ProcessFileStream(fileStream);
}
catch (CrossSectionCsvFormatException)
{
return(ParseFileResult.CannotParse());
}
catch (Exception e)
{
return(ParseFileResult.CannotParse(e));
}
try
{
return(ProcessParsedFileContents(parsedFileContents));
}
catch (Exception e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e));
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger)
{
var xmlRoot = QRevSerializer.DeserializeNoThrow(fileStream, logger);
return(xmlRoot == null
? ParseFileResult.CannotParse()
: ParseXmlRootNoThrow(targetLocation, xmlRoot, appender, logger));
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation,
IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var fieldVisit = ReadFile(fileStream);
//Only save data if it matches the targetLocation.
if (!targetLocation.LocationIdentifier.Equals(fieldVisit.LocationIdentifier))
{
return(ParseFileResult.CannotParse());
}
var resultsGenerator = new FieldDataResultsGenerator(fieldDataResultsAppender, targetLocation, logger);
return(GetFieldDataResults(resultsGenerator, fieldVisit));
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
try
{
var zipArchive = GetZipArchiveOrNull(stream);
if (zipArchive == null)
{
return(ParseFileResult.CannotParse());
}
using (zipArchive)
{
if (zipArchive.Entries.Any(e => e.Name != e.FullName))
{
return(ParseFileResult.CannotParse());
}
if (!zipArchive.Entries.Any())
{
return(ParseFileResult.CannotParse());
}
var plugins = LoadPlugins();
if (!plugins.Any())
{
return(ParseFileResult.CannotParse());
}
var result = ParseArchive(zipArchive, locationInfo, plugins);
if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid)
{
return(result);
}
// Now append all the visits
ResultsAppender.AppendAllResults();
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
catch (Exception exception)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
private ParseFileResult ParseDataFileWithLocale(Configuration configuration, byte[] csvBytes)
{
using (LocaleScope.WithLocale(configuration.LocaleName))
{
try
{
return(ParseDataFile(configuration, csvBytes));
}
catch (Exception exception)
{
Log.Error($"Configuration '{configuration.Id}' failed to parse file: {exception.Message}");
return(ParseFileResult.CannotParse(exception));
}
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var fieldVisit = ReadFile(fileStream);
try
{
var location = fieldDataResultsAppender.GetLocationByIdentifier(fieldVisit.LocationIdentifier);
logger.Info($"Parsing field data for location {location.LocationIdentifier}");
var resultsGenerator = new FieldDataResultsGenerator(fieldDataResultsAppender, location, logger);
return(GetFieldDataResults(resultsGenerator, fieldVisit));
}
catch (Exception)
{
logger.Error($"Cannot parse file, location {fieldVisit.LocationIdentifier} not found");
return(ParseFileResult.CannotParse());
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
var eHsn = new Parser()
.LoadFromStream(fileStream);
if (eHsn == null)
{
return(ParseFileResult.CannotParse());
}
var targetLocation = fieldDataResultsAppender.GetLocationByIdentifier("MyDummyLocation");
var now = DateTimeOffset.UtcNow;
var visit = fieldDataResultsAppender.AddFieldVisit(targetLocation,
new FieldVisitDetails(new DateTimeInterval(now.AddHours(-1), now)));
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo)
{
DataFile = GetDataFile(stream);
if (DataFile == null)
{
return(ParseFileResult.CannotParse());
}
if (locationInfo == null)
{
if (string.IsNullOrEmpty(DataFile.Properties.SiteNumber))
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid($"No {nameof(DataFile.Properties.SiteNumber)} property is set, so no AQUARIUS location can be inferred. Try uploading the file directly to a location."));
}
locationInfo = _resultsAppender.GetLocationByIdentifier(DataFile.Properties.SiteNumber);
}
return(AppendResults(locationInfo));
}
private ParseFileResult ParseDataFileWithLocale(Configuration configuration, byte[] csvBytes)
{
using (LocaleScope.WithLocale(configuration.LocaleName))
{
try
{
if (new ExcelParser().TryLoadSingleSheet(configuration, csvBytes, out var sheetBytes))
{
csvBytes = sheetBytes;
}
return(ParseDataFile(configuration, csvBytes));
}
catch (Exception exception)
{
Log.Error($"Configuration '{configuration.Id}' failed to parse file: {exception.Message}");
return(ParseFileResult.CannotParse(exception));
}
}
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
var csvBytes = ReadBytesFromStream(stream);
if (csvBytes == null)
{
return(ParseFileResult.CannotParse());
}
try
{
LocationInfo = locationInfo;
var configurations = LoadConfigurations();
using (ResultsAppender)
{
var result = ParseFileResult.CannotParse();
foreach (var configuration in configurations)
{
result = ParseDataFileWithLocale(configuration, csvBytes);
if (result.Status == ParseFileStatus.CannotParse)
{
continue;
}
return(result);
}
return(result);
}
}
catch (Exception exception)
{
return(ParseFileResult.CannotParse(exception));
}
}
private bool TryParseFile(byte[] dataBytes, out ParseFileResultWithAttachments result)
{
result = new ParseFileResultWithAttachments();
try
{
using (var stream = new MemoryStream(dataBytes))
{
result.Result = LocationInfo != null
? Plugin.ParseFile(stream, LocationInfo, Appender, Logger)
: Plugin.ParseFile(stream, Appender, Logger);
}
return(true);
}
catch (Exception exception)
{
result.Result = ParseFileResult.CannotParse(exception);
return(false);
}
}
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger)
{
_log = logger;
try
{
using (var delayedAppender = new DelayedAppender(fieldDataResultsAppender))
{
_fieldDataResultsAppender = delayedAppender;
var parsedRecords = _parser.ParseInputData(fileStream);
if (parsedRecords == null)
{
return(ParseFileResult.CannotParse(NoRecordsInInputFile));
}
if (_parser.Errors.Any())
{
if (_parser.ValidRecords > 0)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(
$"{InputFileContainsInvalidRecords}: {_parser.Errors.Length} errors:\n{string.Join("\n", _parser.Errors.Take(3))}"));
}
return(ParseFileResult.CannotParse());
}
_log.Info($"Parsed {_parser.ValidRecords} rows from input file.");
SaveRecords(parsedRecords);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
catch (Exception e)
{
_log.Error($"Failed to parse file; {e.Message}\n{e.StackTrace}");
return(ParseFileResult.CannotParse(e));
}
}
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null)
{
var csvText = ReadTextFromStream(stream);
if (csvText == null)
{
return(ParseFileResult.CannotParse());
}
try
{
LocationInfo = locationInfo;
Surveys = LoadSurveys();
using (ResultsAppender)
{
foreach (var survey in Surveys)
{
Survey = survey;
var result = ParseSurvey(csvText);
if (result.Status == ParseFileStatus.CannotParse)
{
continue;
}
return(result);
}
return(ParseFileResult.CannotParse());
}
}
catch (Exception exception)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception));
}
}
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger)
{
try
{
Config = new ConfigLoader(appender)
.Load();
var summary = GetSummary(fileStream, logger);
if (summary == null)
{
return(ParseFileResult.CannotParse());
}
if (targetLocation == null)
{
var locationIdentifier = ExtractLocationIdentifier(summary.StationName);
if (string.IsNullOrEmpty(locationIdentifier))
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing station name"));
}
targetLocation = appender.GetLocationByIdentifier(locationIdentifier);
}
var parser = new Parser(Config, targetLocation, appender, logger);
parser.Parse(summary);
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
catch (Exception e)
{
return(ParseFileResult.SuccessfullyParsedButDataInvalid(e.Message));
}
}
private ParseFileResult ParseDataFile(Configuration configuration, byte[] csvBytes)
{
var csvText = ReadTextFromBytes(configuration, csvBytes);
var rowParser = new RowParser(configuration, ResultsAppender, LocationInfo);
var(prefaceLines, dataRowReader) = ExtractPrefaceLines(configuration, csvText);
rowParser.AddPrefaceLines(prefaceLines);
if (!rowParser.IsPrefaceValid())
{
return(ParseFileResult.CannotParse());
}
var dataRowCount = 0;
using (var reader = dataRowReader)
{
var separator = configuration.Separator ?? DefaultFieldSeparator;
var fieldParser = GetCsvParser(reader, separator);
while (!fieldParser.EndOfData)
{
rowParser.LineNumber = rowParser.PrefaceLineCount + fieldParser.LineNumber;
string[] fields = null;
try
{
fields = fieldParser.ReadFields();
}
catch (Exception)
{
if (dataRowCount == 0)
{
// We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document.
return(ParseFileResult.CannotParse());
}
}
if (fields == null)
{
continue;
}
if (fields.Length > 0 && fields.All(string.IsNullOrEmpty))
{
continue;
}
if (fields.Length > 0 && !string.IsNullOrEmpty(configuration.CommentLinePrefix) && fields[0].StartsWith(configuration.CommentLinePrefix))
{
continue;
}
if (dataRowCount == 0 && configuration.IsHeaderRowRequired)
{
try
{
if (rowParser.IsHeaderFullyParsed(fields))
{
++dataRowCount;
}
continue;
}
catch (Exception exception)
{
// Most CSV files have a header.
// So a problem mapping the header is a strong indicator that this CSV file is not intended for us.
if (exception is AllHeadersMissingException)
{
// When all headers are missing, then we should exit without logging anything special.
// We'll just let the other plugins have a turn
}
else
{
// Some of the headers matched, so log a warning before returning CannotParse.
// This might be the only hint in the log that the configuration is incorrect.
Log.Info($"Partial headers detected: {exception.Message}");
}
return(ParseFileResult.CannotParse());
}
}
if (IsFooterDetected(configuration, () => string.Join(separator, fields)))
{
break;
}
try
{
rowParser.Parse(fields);
}
catch (Exception exception)
{
if (!ResultsAppender.AnyResultsAppended)
{
throw;
}
Log.Error($"Ignoring invalid CSV row: {exception.Message}");
}
++dataRowCount;
}
if (dataRowCount == 0)
{
if (configuration.NoDataRowsExpected)
{
// When all the data comes from the preface, then this will parse the preface context
rowParser.Parse(new string[0]);
}
else
{
// No rows were parsed.
if (rowParser.RemainingHeaderLines > 0)
{
return(ParseFileResult.CannotParse("No matching header row was detected."));
}
return(ParseFileResult.CannotParse());
}
}
Log.Info($"Configuration='{configuration.Id}' Priority={configuration.Priority} parsed {ResultsAppender.SummaryInfo()}.");
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}
private ParseFileResult ParseSurvey(string csvText)
{
var validator = new SurveyValidator {
Survey = Survey
};
var lineCount = 0;
using (var reader = new StringReader(csvText))
{
var rowParser = GetCsvParser(reader);
for (; !rowParser.EndOfData; ++lineCount)
{
LineNumber = rowParser.LineNumber;
try
{
Fields = rowParser.ReadFields();
}
catch (Exception)
{
if (lineCount == 0)
{
// We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document.
return(ParseFileResult.CannotParse());
}
}
if (Fields == null)
{
continue;
}
if (lineCount == 0 && Survey.FirstLineIsHeader)
{
try
{
HeaderMap = validator.BuildHeaderMap(Fields);
continue;
}
catch (Exception exception)
{
// Most Survey123 files have a header.
// So a problem mapping the header is a strong indicator that this CSV file is not intended for us.
if (exception is AllHeadersMissingException)
{
// When all headers are missing, then we should exit without logging anything special.
// We'll just let the other plugins have a turn
}
else
{
// Some of the headers matched, so log a warning before returning CannotParse.
// This might be the only hint in the log that the survey configuration JSON is incorrect.
Log.Info($"Partial headers detected: {exception.Message}");
}
return(ParseFileResult.CannotParse());
}
}
try
{
ParseRow();
}
catch (Exception exception)
{
if (!ResultsAppender.AnyResultsAppended)
{
throw;
}
Log.Error($"Ignoring invalid CSV row: {exception.Message}");
}
}
return(ParseFileResult.SuccessfullyParsedAndDataValid());
}
}