Exemple #1
0
        public void ToMeasurement_ValidDataMetV2_ReturnCorrectMeasurementsV2()
        {
            //# Arrange
            MapFromToa5DataTableToCafStandards map =
                new MapFromToa5DataTableToCafStandards();
            TOA5 toa5 = LoggerNetArranger.GetToa5MeteorologyDerivedFromActualDataV2();

            List <MeasurementV2> expected =
                LoggerNetArranger.GetMeasurementsV2DerivedFromActualDataMeteorologyV2();

            CosmosDBSqlApiV2Transformer sut =
                new CosmosDBSqlApiV2Transformer(
                    map,
                    "http://files.cafltar.org/data/schema/documentDb/v2/measurement.json",
                    "CosmosDBSqlApiTransformer",
                    "Measurement",
                    "CafMeteorologyEcTower",
                    900);

            //# Act
            var actual = sut.ToMeasurements(toa5);

            //# Assert
            Assert.Equal(expected.Count, actual.Count);
            Assert.True(ComparerUtil.AreMeasurementsRoughlyEqual(expected, actual));
        }
        public async Task PipeItAsync()
        {
            EtlEvent etlEvent = new EtlEvent(
                "EtlEvent",
                "AzureFunction",
                "http://files.cafltar.org/data/schema/documentDb/v2/etlEvent.json",
                "CafMeteorologyEcTower",
                version, functionName,
                DateTime.UtcNow);

            etlEvent.Inputs.Add(blobPath);

            StreamReader reader   = new StreamReader(myBlob);
            string       contents = "";

            log.LogInformation("About to read contents");
            try
            {
                contents = reader.ReadToEnd();
            }
            catch (Exception e)
            {
                etlEvent.Logs.Add(
                    $"Error reading Blob: {e.Message}");
            }

            DocumentLoader loader = new DocumentLoader(
                client,
                "cafdb",
                "items");

            log.LogInformation("Created loader");
            if (!String.IsNullOrEmpty(contents))
            {
                try
                {
                    log.LogInformation("Attempting extract and transform");
                    TOA5Extractor extractor = new TOA5Extractor(
                        name,
                        contents,
                        -8);

                    TOA5 toa5 = extractor.GetTOA5(observation);

                    CosmosDBSqlApiV2Transformer transformer =
                        new CosmosDBSqlApiV2Transformer(
                            new MapFromToa5DataTableToCafStandards(),
                            "http://files.cafltar.org/data/schema/documentDb/v2/measurement.json",
                            etlEvent.Id,
                            "Measurement",
                            "CafMeteorologyEcTower",
                            timestep);

                    List <MeasurementV2> measurements =
                        transformer.ToMeasurements(toa5);
                    log.LogInformation("Attempting load");

                    int docsLoaded = 0;
                    int docsError  = 0;
                    foreach (MeasurementV2 measurement in measurements)
                    {
                        try
                        {
                            ResourceResponse <Document> result =
                                await loader.LoadNoReplace(measurement);

                            if (
                                result.StatusCode == HttpStatusCode.Created ||
                                result.StatusCode == HttpStatusCode.OK)
                            {
                                etlEvent.Outputs.Add(result.Resource.Id);
                                docsLoaded++;
                            }
                            else
                            {
                                etlEvent.Logs.Add(
                                    $"StatusCode: {result.StatusCode} on MeasurementV2: {measurement.Id.ToString()}");
                                docsError++;
                            }
                        }
                        catch (Exception e)
                        {
                            etlEvent.Logs.Add(
                                $"Error loading {measurement.Id.ToString()} MeasurementV2: {e.Message}");
                            log.LogError($"Error loading MeasurementV2: {e.Message}");
                            docsError++;
                        }
                    }
                    log.LogInformation(
                        $"Loaded {docsLoaded.ToString()} MeasurementV2s.");
                    log.LogInformation(
                        $"Error loading {docsError.ToString()} MeasurementV2s.");
                    etlEvent.Logs.Add(
                        $"Loaded {docsLoaded.ToString()} MeasurementV2s");
                    etlEvent.Logs.Add(
                        $"Error loading {docsError.ToString()} MeasurementV2s");
                }
                catch (Exception e)
                {
                    etlEvent.Logs.Add(
                        $"Error in ETL pipeline: {e.Message}");
                    log.LogError($"Error in ETL pipeline: {e.Message}");
                    throw new Exception("Error in ETL pipeline", e);
                }
                finally
                {
                    log.LogInformation("Loading etlEvent to db");
                    etlEvent.DateTimeEnd = DateTime.UtcNow;
                    ResourceResponse <Document> result = await loader.LoadNoReplace(etlEvent);

                    log.LogInformation($"Result of writing EtlEvent: {result.StatusCode.ToString()}");
                }
            }
        }