public void Export()
{
Validate();
using (Client = Connect())
using (var progressReporter = new ProgressBarReporter())
{
ExportAll(progressReporter);
}
}
private void ExportAll(ProgressBarReporter progressBarReporter)
{
var analyticalGroups = Client.Get(new GetAnalyticalGroups())
.DomainObjects;
var(requests, summary) = BuildRequestsAndSummary(analyticalGroups);
Log.Info($"Exporting observations {summary} ...");
var responses = requests
.Select(request => Client.LazyGet <Observation, GetObservationsV2, SearchResultObservation>(request, progressReporter: progressBarReporter))
.ToList();
var observedPropertiesByGroup = Context
.AnalyticalGroupIds
.Select((name, i) =>
{
var group = analyticalGroups
.First(ag => ag.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase));
return(new AnalyticalGroupInfo
{
Group = group,
GroupOrder = i,
Properties = group.AnalyticalGroupItems.Select(item => item.ObservedProperty.CustomId).ToList()
});
})
.ToList();
progressBarReporter.ExpectedCount = responses.Sum(response => response.TotalCount);
Log.Info($"Loading all {progressBarReporter.ExpectedCount} matching observations ...");
var stopwatch = Stopwatch.StartNew();
var items = responses
.SelectMany(response => response
.DomainObjects
.Where(item => item.NumericResult != null)
)
.ToList();
progressBarReporter.Dispose();
Log.Info($"{items.Count} numeric observations loaded in {stopwatch.Elapsed.Humanize(2)}.");
var resultColumns = items
.Select(item =>
{
var observedPropertyName = item.ObservedProperty.CustomId;
var groupInfo = observedPropertiesByGroup
.FirstOrDefault(gi => gi.Properties.Contains(observedPropertyName));
return(new ResultColumn
{
ObservedPropertyId = item.ObservedProperty.CustomId,
Unit = item.ObservedProperty.DefaultUnit?.CustomId,
MethodId = item.AnalysisMethod?.MethodId,
AnalyticalGroup = groupInfo?.Group?.Name,
AnalyticalGroupOrder = groupInfo?.GroupOrder ?? 0,
ObservedPropertyOrder = groupInfo?.Properties?.IndexOf(observedPropertyName) ?? 0
});
})
.DistinctBy(rc => new{ rc.ObservedPropertyId, rc.Unit, rc.MethodId, rc.AnalyticalGroup })
.OrderBy(column => column.AnalyticalGroupOrder)
.ThenBy(column => column.ObservedPropertyOrder)
.ThenBy(column => column.ObservedPropertyId)
.ThenBy(column => column.MethodId)
.ThenBy(column => column.Unit)
.ToList();
if (Context.Overwrite && File.Exists(Context.CsvOutputPath))
{
Log.Warn($"Overwriting existing file '{Context.CsvOutputPath}'.");
}
var headers = new[]
{
"Lab: Sample ID",
"Location ID",
"Observed Date",
$"Observed Time_UTC{Context.UtcOffset:m}",
"Latitude",
"Longitude",
"County",
"Medium",
"Location Groups",
}
.Concat(resultColumns.SelectMany(column => new[]
{
EscapeCsvColumn($"{column.ObservedPropertyId}_Value"),
EscapeCsvColumn($"{column.ObservedPropertyId}_{DistinctColumnUnitLabel(resultColumns, column)}")
}))
.ToList();
Log.Info($"Exporting observations to '{Context.CsvOutputPath}' ...");
var rowCount = 0;
var observationCount = 0;
using (var stream = File.OpenWrite(Context.CsvOutputPath))
using (var writer = new StreamWriter(stream, new UTF8Encoding(true)))
{
var headerRow = string.Join(",", headers);
writer.WriteLine(headerRow);
var utcOffset = Context.UtcOffset.ToTimeSpan();
foreach (var group in items.GroupBy(item => new
{
item.FieldVisit.StartTime,
item.LabResultDetails?.LabSampleId,
item.SamplingLocation.CustomId
})
.OrderBy(g => g.Key.StartTime)
.ThenBy(g => g.Key.LabSampleId)
.ThenBy(g => g.Key.CustomId))
{
var observations = group.ToList();
var first = observations.First();
var location = first.SamplingLocation;
if (!first.FieldVisit?.StartTime.HasValue ?? true)
{
throw new ExpectedException($"{"observation".ToQuantity(observations.Count)} at location '{location.CustomId}' have no timestamp.");
}
var time = first.FieldVisit.StartTime.Value.ToDateTimeOffset().Add(utcOffset);
var columns = new List <string>(headerRow.Length)
{
EscapeCsvColumn(first.LabResultDetails?.LabSampleId),
EscapeCsvColumn(location.CustomId),
$"{time:yyyy-MM-dd}",
$"{time:HH:mm:ss}",
location.Latitude,
location.Longitude,
EscapeCsvColumn(location.Address?.CountyCode),
EscapeCsvColumn(first.Medium.CustomId),
EscapeCsvColumn(string.Join(";", location.SamplingLocationGroups.Select(sg => sg.Name)))
};
do
{
var(row, remainingObservations, writtenObservations) = WriteRow(resultColumns, observations, columns);
writer.WriteLine(row);
++rowCount;
observationCount += writtenObservations;
observations = remainingObservations;
} while (observations.Any());
}
}
Log.Info($"Exported {"distinct observation".ToQuantity(observationCount)} observations over {"row".ToQuantity(rowCount)} to '{Context.CsvOutputPath}'.");
}
