private ParseFileResult ParseArchive(ZipArchive zipArchive, LocationInfo locationInfo, List <IFieldDataPlugin> plugins) { foreach (var entry in zipArchive.Entries) { var isParsed = false; foreach (var plugin in plugins) { var result = ParseEntry(entry, plugin, locationInfo); if (result.Status == ParseFileStatus.CannotParse) { continue; } if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid) { Log.Error($"Plugin {PluginLoader.GetPluginNameAndVersion(plugin)} failed to parse '{entry.FullName}' with {result.Status}('{result.ErrorMessage}')"); return(result); } isParsed = true; break; } if (!isParsed) { return(ParseFileResult.CannotParse()); } } // All entries were parsed by one of the plugins return(ParseFileResult.SuccessfullyParsedAndDataValid()); }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger) { try { var parser = new Parser(fieldDataResultsAppender, logger); var eHsn = parser.LoadFromStream(fileStream); if (eHsn == null) { return(ParseFileResult.CannotParse()); } try { parser.Parse(eHsn); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } catch (Exception exception) { logger.Error($"File can be parsed but an error occurred: {exception.Message}\n{exception.StackTrace}"); return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception)); } } catch (Exception exception) { logger.Error($"Something went wrong: {exception.Message}\n{exception.StackTrace}"); return(ParseFileResult.CannotParse(exception)); } }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger) { try { var gaugingSummary = ParseGaugingSummaryFromFile(fileStream); var gaugingSummaryProcessor = new GaugingSummaryProcessor(fieldDataResultsAppender, logger); gaugingSummaryProcessor.ProcessGaugingSummary(gaugingSummary); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } catch (Exception ex) when(ex is PocketGaugerZipFileMissingRequiredContentException || ex is PocketGaugerZipFileException) { return(ParseFileResult.CannotParse()); } catch (PocketGaugerDataFormatException e) { return(ParseFileResult.SuccessfullyParsedButDataInvalid(e)); } catch (Exception e) { return(ParseFileResult.CannotParse(e)); } }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender appender, ILog logger) { var channel = QRevSerializer.DeserializeNoThrow(fileStream, logger); if (channel == null) { return(ParseFileResult.CannotParse()); } var locationIdentifier = channel.SiteInformation?.SiteID?.Value; if (string.IsNullOrWhiteSpace(locationIdentifier)) { logger.Error("File can be parsed but there is no SiteID specified."); return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing <Channel/SiteInformation/SiteID>")); } try { var trimmedLocationIdentifier = locationIdentifier.Trim(); var location = appender.GetLocationByIdentifier(trimmedLocationIdentifier); return(ParseXmlRootNoThrow(location, channel, appender, logger)); } catch (Exception exception) { logger.Error($"Cannot find location with identifier '{locationIdentifier}'."); return(ParseFileResult.CannotParse(exception)); } }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender appender, ILog logger) { var xmlRoot = SectionBySectionSerializer.DeserializeNoThrow(fileStream, logger); if (xmlRoot == null) { return(ParseFileResult.CannotParse()); } var stationNo = xmlRoot.Summary?.WinRiver_II_Section_by_Section_Summary?.Station_No; if (string.IsNullOrWhiteSpace(stationNo)) { logger.Error("File can be parsed but there is no Station_No specified."); return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing Station_No.")); } try { var trimmedLocationIdentifier = stationNo.Trim(); var location = appender.GetLocationByIdentifier(trimmedLocationIdentifier); return(ParseXmlRootNoThrow(location, xmlRoot, appender, logger)); } catch (Exception exception) { logger.Error($"Cannot find location with identifier {stationNo}."); return(ParseFileResult.CannotParse(exception)); } }
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null) { var jsonText = ReadTextFromStream(stream); if (jsonText == null) { return(ParseFileResult.CannotParse()); } try { LocationInfo = locationInfo; AppendedResults = ParseJson(jsonText); if (AppendedResults == null) { return(ParseFileResult.CannotParse()); } return(MapToFramework()); } catch (Exception exception) { return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception)); } }
private ParseFileResult ParseEntry(ZipArchiveEntry entry, IFieldDataPlugin plugin, LocationInfo locationInfo) { using (var entryStream = entry.Open()) using (var reader = new BinaryReader(entryStream)) using (var memoryStream = new MemoryStream(reader.ReadBytes((int)entry.Length))) { var proxyLog = ProxyLog.Create(Log, plugin, entry); var result = locationInfo == null ? plugin.ParseFile(memoryStream, ResultsAppender, proxyLog) : plugin.ParseFile(memoryStream, locationInfo, ResultsAppender, proxyLog); switch (result.Status) { case ParseFileStatus.CannotParse: result = ParseFileResult.CannotParse($"{proxyLog.Prefix}: {result.ErrorMessage}"); break; case ParseFileStatus.SuccessfullyParsedButDataInvalid: result = ParseFileResult.SuccessfullyParsedButDataInvalid($"{proxyLog.Prefix}: {result.ErrorMessage}"); break; } return(result); } }
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger) { var parser = new AquaCalcCsvParser(fileStream) { Settings = appender.GetPluginConfigurations() }; if (!parser.CanParse()) { return(ParseFileResult.CannotParse()); } var parsedData = parser.Parse(); if (targetLocation == null) { targetLocation = appender.GetLocationByIdentifier(parsedData.GageId); } var visitDetails = new VisitMapper(parsedData).GetVisitDetails(targetLocation.UtcOffset); logger.Info($"Got visit details: '{visitDetails.StartDate:s}@{targetLocation.LocationIdentifier}'"); var visitInfo = appender.AddFieldVisit(targetLocation, visitDetails); AppendActivity(appender, parsedData, visitInfo, logger); return(ParseFileResult.SuccessfullyParsedAndDataValid()); }
public ParseFileResult ParseFile(Stream fileStream) { CrossSectionSurvey parsedFileContents; try { parsedFileContents = ProcessFileStream(fileStream); } catch (CrossSectionCsvFormatException) { return(ParseFileResult.CannotParse()); } catch (Exception e) { return(ParseFileResult.CannotParse(e)); } try { return(ProcessParsedFileContents(parsedFileContents)); } catch (Exception e) { return(ParseFileResult.SuccessfullyParsedButDataInvalid(e)); } }
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger) { var xmlRoot = QRevSerializer.DeserializeNoThrow(fileStream, logger); return(xmlRoot == null ? ParseFileResult.CannotParse() : ParseXmlRootNoThrow(targetLocation, xmlRoot, appender, logger)); }
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger) { var fieldVisit = ReadFile(fileStream); //Only save data if it matches the targetLocation. if (!targetLocation.LocationIdentifier.Equals(fieldVisit.LocationIdentifier)) { return(ParseFileResult.CannotParse()); } var resultsGenerator = new FieldDataResultsGenerator(fieldDataResultsAppender, targetLocation, logger); return(GetFieldDataResults(resultsGenerator, fieldVisit)); }
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null) { try { var zipArchive = GetZipArchiveOrNull(stream); if (zipArchive == null) { return(ParseFileResult.CannotParse()); } using (zipArchive) { if (zipArchive.Entries.Any(e => e.Name != e.FullName)) { return(ParseFileResult.CannotParse()); } if (!zipArchive.Entries.Any()) { return(ParseFileResult.CannotParse()); } var plugins = LoadPlugins(); if (!plugins.Any()) { return(ParseFileResult.CannotParse()); } var result = ParseArchive(zipArchive, locationInfo, plugins); if (result.Status != ParseFileStatus.SuccessfullyParsedAndDataValid) { return(result); } // Now append all the visits ResultsAppender.AppendAllResults(); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } } catch (Exception exception) { return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception)); } }
private ParseFileResult ParseDataFileWithLocale(Configuration configuration, byte[] csvBytes) { using (LocaleScope.WithLocale(configuration.LocaleName)) { try { return(ParseDataFile(configuration, csvBytes)); } catch (Exception exception) { Log.Error($"Configuration '{configuration.Id}' failed to parse file: {exception.Message}"); return(ParseFileResult.CannotParse(exception)); } } }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger) { var fieldVisit = ReadFile(fileStream); try { var location = fieldDataResultsAppender.GetLocationByIdentifier(fieldVisit.LocationIdentifier); logger.Info($"Parsing field data for location {location.LocationIdentifier}"); var resultsGenerator = new FieldDataResultsGenerator(fieldDataResultsAppender, location, logger); return(GetFieldDataResults(resultsGenerator, fieldVisit)); } catch (Exception) { logger.Error($"Cannot parse file, location {fieldVisit.LocationIdentifier} not found"); return(ParseFileResult.CannotParse()); } }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger) { var eHsn = new Parser() .LoadFromStream(fileStream); if (eHsn == null) { return(ParseFileResult.CannotParse()); } var targetLocation = fieldDataResultsAppender.GetLocationByIdentifier("MyDummyLocation"); var now = DateTimeOffset.UtcNow; var visit = fieldDataResultsAppender.AddFieldVisit(targetLocation, new FieldVisitDetails(new DateTimeInterval(now.AddHours(-1), now))); return(ParseFileResult.SuccessfullyParsedAndDataValid()); }
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo) { DataFile = GetDataFile(stream); if (DataFile == null) { return(ParseFileResult.CannotParse()); } if (locationInfo == null) { if (string.IsNullOrEmpty(DataFile.Properties.SiteNumber)) { return(ParseFileResult.SuccessfullyParsedButDataInvalid($"No {nameof(DataFile.Properties.SiteNumber)} property is set, so no AQUARIUS location can be inferred. Try uploading the file directly to a location.")); } locationInfo = _resultsAppender.GetLocationByIdentifier(DataFile.Properties.SiteNumber); } return(AppendResults(locationInfo)); }
private ParseFileResult ParseDataFileWithLocale(Configuration configuration, byte[] csvBytes) { using (LocaleScope.WithLocale(configuration.LocaleName)) { try { if (new ExcelParser().TryLoadSingleSheet(configuration, csvBytes, out var sheetBytes)) { csvBytes = sheetBytes; } return(ParseDataFile(configuration, csvBytes)); } catch (Exception exception) { Log.Error($"Configuration '{configuration.Id}' failed to parse file: {exception.Message}"); return(ParseFileResult.CannotParse(exception)); } } }
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null) { var csvBytes = ReadBytesFromStream(stream); if (csvBytes == null) { return(ParseFileResult.CannotParse()); } try { LocationInfo = locationInfo; var configurations = LoadConfigurations(); using (ResultsAppender) { var result = ParseFileResult.CannotParse(); foreach (var configuration in configurations) { result = ParseDataFileWithLocale(configuration, csvBytes); if (result.Status == ParseFileStatus.CannotParse) { continue; } return(result); } return(result); } } catch (Exception exception) { return(ParseFileResult.CannotParse(exception)); } }
private bool TryParseFile(byte[] dataBytes, out ParseFileResultWithAttachments result) { result = new ParseFileResultWithAttachments(); try { using (var stream = new MemoryStream(dataBytes)) { result.Result = LocationInfo != null ? Plugin.ParseFile(stream, LocationInfo, Appender, Logger) : Plugin.ParseFile(stream, Appender, Logger); } return(true); } catch (Exception exception) { result.Result = ParseFileResult.CannotParse(exception); return(false); } }
public ParseFileResult ParseFile(Stream fileStream, IFieldDataResultsAppender fieldDataResultsAppender, ILog logger) { _log = logger; try { using (var delayedAppender = new DelayedAppender(fieldDataResultsAppender)) { _fieldDataResultsAppender = delayedAppender; var parsedRecords = _parser.ParseInputData(fileStream); if (parsedRecords == null) { return(ParseFileResult.CannotParse(NoRecordsInInputFile)); } if (_parser.Errors.Any()) { if (_parser.ValidRecords > 0) { return(ParseFileResult.SuccessfullyParsedButDataInvalid( $"{InputFileContainsInvalidRecords}: {_parser.Errors.Length} errors:\n{string.Join("\n", _parser.Errors.Take(3))}")); } return(ParseFileResult.CannotParse()); } _log.Info($"Parsed {_parser.ValidRecords} rows from input file."); SaveRecords(parsedRecords); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } } catch (Exception e) { _log.Error($"Failed to parse file; {e.Message}\n{e.StackTrace}"); return(ParseFileResult.CannotParse(e)); } }
public ParseFileResult Parse(Stream stream, LocationInfo locationInfo = null) { var csvText = ReadTextFromStream(stream); if (csvText == null) { return(ParseFileResult.CannotParse()); } try { LocationInfo = locationInfo; Surveys = LoadSurveys(); using (ResultsAppender) { foreach (var survey in Surveys) { Survey = survey; var result = ParseSurvey(csvText); if (result.Status == ParseFileStatus.CannotParse) { continue; } return(result); } return(ParseFileResult.CannotParse()); } } catch (Exception exception) { return(ParseFileResult.SuccessfullyParsedButDataInvalid(exception)); } }
public ParseFileResult ParseFile(Stream fileStream, LocationInfo targetLocation, IFieldDataResultsAppender appender, ILog logger) { try { Config = new ConfigLoader(appender) .Load(); var summary = GetSummary(fileStream, logger); if (summary == null) { return(ParseFileResult.CannotParse()); } if (targetLocation == null) { var locationIdentifier = ExtractLocationIdentifier(summary.StationName); if (string.IsNullOrEmpty(locationIdentifier)) { return(ParseFileResult.SuccessfullyParsedButDataInvalid("Missing station name")); } targetLocation = appender.GetLocationByIdentifier(locationIdentifier); } var parser = new Parser(Config, targetLocation, appender, logger); parser.Parse(summary); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } catch (Exception e) { return(ParseFileResult.SuccessfullyParsedButDataInvalid(e.Message)); } }
private ParseFileResult ParseDataFile(Configuration configuration, byte[] csvBytes) { var csvText = ReadTextFromBytes(configuration, csvBytes); var rowParser = new RowParser(configuration, ResultsAppender, LocationInfo); var(prefaceLines, dataRowReader) = ExtractPrefaceLines(configuration, csvText); rowParser.AddPrefaceLines(prefaceLines); if (!rowParser.IsPrefaceValid()) { return(ParseFileResult.CannotParse()); } var dataRowCount = 0; using (var reader = dataRowReader) { var separator = configuration.Separator ?? DefaultFieldSeparator; var fieldParser = GetCsvParser(reader, separator); while (!fieldParser.EndOfData) { rowParser.LineNumber = rowParser.PrefaceLineCount + fieldParser.LineNumber; string[] fields = null; try { fields = fieldParser.ReadFields(); } catch (Exception) { if (dataRowCount == 0) { // We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document. return(ParseFileResult.CannotParse()); } } if (fields == null) { continue; } if (fields.Length > 0 && fields.All(string.IsNullOrEmpty)) { continue; } if (fields.Length > 0 && !string.IsNullOrEmpty(configuration.CommentLinePrefix) && fields[0].StartsWith(configuration.CommentLinePrefix)) { continue; } if (dataRowCount == 0 && configuration.IsHeaderRowRequired) { try { if (rowParser.IsHeaderFullyParsed(fields)) { ++dataRowCount; } continue; } catch (Exception exception) { // Most CSV files have a header. // So a problem mapping the header is a strong indicator that this CSV file is not intended for us. if (exception is AllHeadersMissingException) { // When all headers are missing, then we should exit without logging anything special. // We'll just let the other plugins have a turn } else { // Some of the headers matched, so log a warning before returning CannotParse. // This might be the only hint in the log that the configuration is incorrect. Log.Info($"Partial headers detected: {exception.Message}"); } return(ParseFileResult.CannotParse()); } } if (IsFooterDetected(configuration, () => string.Join(separator, fields))) { break; } try { rowParser.Parse(fields); } catch (Exception exception) { if (!ResultsAppender.AnyResultsAppended) { throw; } Log.Error($"Ignoring invalid CSV row: {exception.Message}"); } ++dataRowCount; } if (dataRowCount == 0) { if (configuration.NoDataRowsExpected) { // When all the data comes from the preface, then this will parse the preface context rowParser.Parse(new string[0]); } else { // No rows were parsed. if (rowParser.RemainingHeaderLines > 0) { return(ParseFileResult.CannotParse("No matching header row was detected.")); } return(ParseFileResult.CannotParse()); } } Log.Info($"Configuration='{configuration.Id}' Priority={configuration.Priority} parsed {ResultsAppender.SummaryInfo()}."); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } }
private ParseFileResult ParseSurvey(string csvText) { var validator = new SurveyValidator { Survey = Survey }; var lineCount = 0; using (var reader = new StringReader(csvText)) { var rowParser = GetCsvParser(reader); for (; !rowParser.EndOfData; ++lineCount) { LineNumber = rowParser.LineNumber; try { Fields = rowParser.ReadFields(); } catch (Exception) { if (lineCount == 0) { // We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document. return(ParseFileResult.CannotParse()); } } if (Fields == null) { continue; } if (lineCount == 0 && Survey.FirstLineIsHeader) { try { HeaderMap = validator.BuildHeaderMap(Fields); continue; } catch (Exception exception) { // Most Survey123 files have a header. // So a problem mapping the header is a strong indicator that this CSV file is not intended for us. if (exception is AllHeadersMissingException) { // When all headers are missing, then we should exit without logging anything special. // We'll just let the other plugins have a turn } else { // Some of the headers matched, so log a warning before returning CannotParse. // This might be the only hint in the log that the survey configuration JSON is incorrect. Log.Info($"Partial headers detected: {exception.Message}"); } return(ParseFileResult.CannotParse()); } } try { ParseRow(); } catch (Exception exception) { if (!ResultsAppender.AnyResultsAppended) { throw; } Log.Error($"Ignoring invalid CSV row: {exception.Message}"); } } return(ParseFileResult.SuccessfullyParsedAndDataValid()); } }