public void ToMeasurement_ValidDataMetV2_ReturnCorrectMeasurementsV2()
{
//# Arrange
MapFromToa5DataTableToCafStandards map =
new MapFromToa5DataTableToCafStandards();
TOA5 toa5 = LoggerNetArranger.GetToa5MeteorologyDerivedFromActualDataV2();
List <MeasurementV2> expected =
LoggerNetArranger.GetMeasurementsV2DerivedFromActualDataMeteorologyV2();
CosmosDBSqlApiV2Transformer sut =
new CosmosDBSqlApiV2Transformer(
map,
"http://files.cafltar.org/data/schema/documentDb/v2/measurement.json",
"CosmosDBSqlApiTransformer",
"Measurement",
"CafMeteorologyEcTower",
900);
//# Act
var actual = sut.ToMeasurements(toa5);
//# Assert
Assert.Equal(expected.Count, actual.Count);
Assert.True(ComparerUtil.AreMeasurementsRoughlyEqual(expected, actual));
}
public async Task PipeItAsync()
{
EtlEvent etlEvent = new EtlEvent(
"EtlEvent",
"AzureFunction",
"http://files.cafltar.org/data/schema/documentDb/v2/etlEvent.json",
"CafMeteorologyEcTower",
version, functionName,
DateTime.UtcNow);
etlEvent.Inputs.Add(blobPath);
StreamReader reader = new StreamReader(myBlob);
string contents = "";
log.LogInformation("About to read contents");
try
{
contents = reader.ReadToEnd();
}
catch (Exception e)
{
etlEvent.Logs.Add(
$"Error reading Blob: {e.Message}");
}
DocumentLoader loader = new DocumentLoader(
client,
"cafdb",
"items");
log.LogInformation("Created loader");
if (!String.IsNullOrEmpty(contents))
{
try
{
log.LogInformation("Attempting extract and transform");
TOA5Extractor extractor = new TOA5Extractor(
name,
contents,
-8);
TOA5 toa5 = extractor.GetTOA5(observation);
CosmosDBSqlApiV2Transformer transformer =
new CosmosDBSqlApiV2Transformer(
new MapFromToa5DataTableToCafStandards(),
"http://files.cafltar.org/data/schema/documentDb/v2/measurement.json",
etlEvent.Id,
"Measurement",
"CafMeteorologyEcTower",
timestep);
List <MeasurementV2> measurements =
transformer.ToMeasurements(toa5);
log.LogInformation("Attempting load");
int docsLoaded = 0;
int docsError = 0;
foreach (MeasurementV2 measurement in measurements)
{
try
{
ResourceResponse <Document> result =
await loader.LoadNoReplace(measurement);
if (
result.StatusCode == HttpStatusCode.Created ||
result.StatusCode == HttpStatusCode.OK)
{
etlEvent.Outputs.Add(result.Resource.Id);
docsLoaded++;
}
else
{
etlEvent.Logs.Add(
$"StatusCode: {result.StatusCode} on MeasurementV2: {measurement.Id.ToString()}");
docsError++;
}
}
catch (Exception e)
{
etlEvent.Logs.Add(
$"Error loading {measurement.Id.ToString()} MeasurementV2: {e.Message}");
log.LogError($"Error loading MeasurementV2: {e.Message}");
docsError++;
}
}
log.LogInformation(
$"Loaded {docsLoaded.ToString()} MeasurementV2s.");
log.LogInformation(
$"Error loading {docsError.ToString()} MeasurementV2s.");
etlEvent.Logs.Add(
$"Loaded {docsLoaded.ToString()} MeasurementV2s");
etlEvent.Logs.Add(
$"Error loading {docsError.ToString()} MeasurementV2s");
}
catch (Exception e)
{
etlEvent.Logs.Add(
$"Error in ETL pipeline: {e.Message}");
log.LogError($"Error in ETL pipeline: {e.Message}");
throw new Exception("Error in ETL pipeline", e);
}
finally
{
log.LogInformation("Loading etlEvent to db");
etlEvent.DateTimeEnd = DateTime.UtcNow;
ResourceResponse <Document> result = await loader.LoadNoReplace(etlEvent);
log.LogInformation($"Result of writing EtlEvent: {result.StatusCode.ToString()}");
}
}
}
