private void AppendVisit(FieldVisitInfo importedVisit) { var locationInfo = LocationInfo ?? ResultsAppender.GetLocationByIdentifier(importedVisit.LocationInfo.LocationIdentifier); var visit = ResultsAppender.AddFieldVisit(locationInfo, importedVisit.FieldVisitDetails); foreach (var controlCondition in importedVisit.ControlConditions) { ResultsAppender.AddControlCondition(visit, controlCondition); } foreach (var crossSectionSurvey in importedVisit.CrossSectionSurveys) { UpgradeCrossSectionSurvey(crossSectionSurvey); ResultsAppender.AddCrossSectionSurvey(visit, crossSectionSurvey); } foreach (var dischargeActivity in importedVisit.DischargeActivities) { ResultsAppender.AddDischargeActivity(visit, dischargeActivity); } foreach (var levelSurvey in importedVisit.LevelSurveys) { ResultsAppender.AddLevelSurvey(visit, levelSurvey); } foreach (var reading in importedVisit.Readings) { ResultsAppender.AddReading(visit, reading); } }
private List <Configuration> LoadConfigurations() { var pluginConfigurations = ResultsAppender.GetPluginConfigurations(); if (!pluginConfigurations.Any()) { Log.Error($"No configurations loaded."); return(new List <Configuration>()); } var configurationLoader = new ConfigurationLoader { Log = Log }; var allConfigurations = pluginConfigurations .Select(kvp => configurationLoader.Load(kvp.Key, kvp.Value)) .Where(configuration => configuration != null) .ToList(); var configurations = allConfigurations .Where(configuration => !configuration.IsDisabled) .OrderBy(configuration => configuration.Priority) .ToList(); if (!configurations.Any()) { Log.Error($"No enabled configurations found. ({pluginConfigurations.Count} were disabled or invalid)."); return(new List <Configuration>()); } configurations = configurations .Where(configuration => LocationInfo != null || configuration.Location != null) .ToList(); foreach (var configuration in configurations) { new ConfigurationValidator { LocationInfo = LocationInfo, Configuration = configuration } .Validate(); } return(configurations); }
private void ParseRow() { var locationIdentifier = GetColumnValue(Survey.LocationColumn); if (Survey.LocationAliases.TryGetValue(locationIdentifier, out var aliasedIdentifier)) { locationIdentifier = aliasedIdentifier; } var locationInfo = LocationInfo ?? ResultsAppender.GetLocationByIdentifier(locationIdentifier); var comments = MergeTextColumns(Survey.CommentColumns); var party = MergeTextColumns(Survey.PartyColumns); var timestamp = ParseTimestamp(locationInfo); var readings = Survey .ReadingColumns .Select(ParseReading) .Where(r => r != null) .Select(r => { r.DateTimeOffset = timestamp; return(r); }) .ToList(); var fieldVisitInfo = ResultsAppender.AddFieldVisit(locationInfo, new FieldVisitDetails(new DateTimeInterval(timestamp, TimeSpan.Zero)) { Comments = comments, Party = party }); foreach (var reading in readings) { ResultsAppender.AddReading(fieldVisitInfo, reading); } }
private ParseFileResult ParseDataFile(Configuration configuration, byte[] csvBytes) { var csvText = ReadTextFromBytes(configuration, csvBytes); var rowParser = new RowParser(configuration, ResultsAppender, LocationInfo); var(prefaceLines, dataRowReader) = ExtractPrefaceLines(configuration, csvText); rowParser.AddPrefaceLines(prefaceLines); if (!rowParser.IsPrefaceValid()) { return(ParseFileResult.CannotParse()); } var dataRowCount = 0; using (var reader = dataRowReader) { var separator = configuration.Separator ?? DefaultFieldSeparator; var fieldParser = GetCsvParser(reader, separator); while (!fieldParser.EndOfData) { rowParser.LineNumber = rowParser.PrefaceLineCount + fieldParser.LineNumber; string[] fields = null; try { fields = fieldParser.ReadFields(); } catch (Exception) { if (dataRowCount == 0) { // We'll hit this when the plugin tries to parse a text file that is not CSV, like a JSON document. return(ParseFileResult.CannotParse()); } } if (fields == null) { continue; } if (fields.Length > 0 && fields.All(string.IsNullOrEmpty)) { continue; } if (fields.Length > 0 && !string.IsNullOrEmpty(configuration.CommentLinePrefix) && fields[0].StartsWith(configuration.CommentLinePrefix)) { continue; } if (dataRowCount == 0 && configuration.IsHeaderRowRequired) { try { if (rowParser.IsHeaderFullyParsed(fields)) { ++dataRowCount; } continue; } catch (Exception exception) { // Most CSV files have a header. // So a problem mapping the header is a strong indicator that this CSV file is not intended for us. if (exception is AllHeadersMissingException) { // When all headers are missing, then we should exit without logging anything special. // We'll just let the other plugins have a turn } else { // Some of the headers matched, so log a warning before returning CannotParse. // This might be the only hint in the log that the configuration is incorrect. Log.Info($"Partial headers detected: {exception.Message}"); } return(ParseFileResult.CannotParse()); } } if (IsFooterDetected(configuration, () => string.Join(separator, fields))) { break; } try { rowParser.Parse(fields); } catch (Exception exception) { if (!ResultsAppender.AnyResultsAppended) { throw; } Log.Error($"Ignoring invalid CSV row: {exception.Message}"); } ++dataRowCount; } if (dataRowCount == 0) { if (configuration.NoDataRowsExpected) { // When all the data comes from the preface, then this will parse the preface context rowParser.Parse(new string[0]); } else { // No rows were parsed. if (rowParser.RemainingHeaderLines > 0) { return(ParseFileResult.CannotParse("No matching header row was detected.")); } return(ParseFileResult.CannotParse()); } } Log.Info($"Configuration='{configuration.Id}' Priority={configuration.Priority} parsed {ResultsAppender.SummaryInfo()}."); return(ParseFileResult.SuccessfullyParsedAndDataValid()); } }