public void TestUnexpectedFields2()
{
//if an exception is thrown one cause it there were messaged in the file we didn't expect
using (GribFile file = new GribFile(@"..\..\TestFiles\unexpected_fields2.grib2"))
{
var rowList = GribUtilities.ParseNamGribFile(file);
Assert.AreEqual(31313, rowList.Count, "Expected count to be equal to 31313");
}
}
public void TestForecastTimeGreaterThan24()
{
//if an exception is thrown one cause it there were messaged in the file we didn't expect
using (GribFile file = new GribFile(@"..\..\TestFiles\dateover12.grib2"))
{
var rowList = GribUtilities.ParseNamGribFile(file);
Assert.AreEqual("20171130T84forecastHour00", rowList[0].PartitionKey, "Incorrect date processing accounting for incorrect hours");
Assert.AreEqual(31313, rowList.Count, "Expected count to be equal to 31313");
}
}
public void TestExpectedFieldsSecondHour()
{
//if an exception is thrown one cause it there were messaged in the file we didn't expect
using (GribFile file = new GribFile(@"..\..\TestFiles\expected_fields_hour2.grib2"))
{
var rowList = GribUtilities.ParseNamGribFile(file);
Assert.AreEqual(31313, rowList.Count, "Expected count to be equal to 31313");
var firstRow = rowList[0];
Assert.AreEqual(Math.Round(firstRow.RH2mAboveGround.Value, 2), 76.8, "Expected first element returned to have RH at 2m of this value.");
Assert.AreEqual(Math.Round(firstRow.UGRD10m.Value, 2), -1.15, "Expected first element returned to have UGRD at 10m of this value.");
}
}
public void TestForOffsetStepStart()
{
//if an exception is thrown one cause it there were messaged in the file we didn't expect
using (GribFile file = new GribFile(@"..\..\TestFiles\checkfornullapcp.grib2"))
{
var rowList = GribUtilities.ParseNamGribFile(file);
Assert.AreEqual(31313, rowList.Count, "Expected count to be equal to 31313");
var secondRow = rowList[1];
Assert.AreEqual(secondRow.APCPStepSize.Value, 11, "Incorrect step size.");
Assert.AreEqual(Math.Round(secondRow.APCPSurface.Value, 4), .0625, "Incorrect APCPSurface value");
Assert.AreEqual(secondRow.Date, new DateTime(2017, 12, 4, 23, 0, 0), "Incorrect date");
}
}
public void TestUnexpectedFields()
{
//if an exception is thrown one cause it there were messaged in the file we didn't expect
using (GribFile file = new GribFile(@"..\..\TestFiles\unexpected_fields.grib2"))
{
try
{
var rowList = GribUtilities.ParseNamGribFile(file);
}
catch (Exception e)
{
Assert.IsInstanceOfType(e, typeof(NotSupportedException), "Didn't get the exception type we expected");
return;
}
Assert.Fail("Expected to get an exception and we didn't");
}
}
public static FileProcessedTracker Run([BlobTrigger("nam-grib-westus-v1/{name}", Connection = "AzureWebJobsStorage")] Stream myBlob, string name, TraceWriter log)
{
log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes");
log.Info($"Double Checking if {name} already exists.");
var exists = AzureUtilities.CheckIfFileProcessedRowExistsInTableStorage(Constants.NamTrackerTable, Constants.NamTrackerPartitionKey, name, log);
if (exists)
{
log.Info($"{name} Already exists in double check, skipping");
return(null);
}
log.Info($"Have env: {Environment.GetEnvironmentVariable("GRIB_API_DIR_ROOT")}");
log.Info($"In dir: {Assembly.GetExecutingAssembly().Location}");
string attemptPath = "";
GribUtilities.TryFindBootstrapLibrary(out attemptPath);
log.Info($"Attemping to find lib: {attemptPath}");
GribEnvironment.Init();
#if DEBUG == false
GribEnvironment.DefinitionsPath = @"D:\home\site\wwwroot\bin\Grib.Api\definitions";
#endif
//1. Download stream to temp
//TODO: there is supposedly now an ability to read a stream direction in GRIBAPI.Net; investigate to see if its better than storing a temp file
string localFileName = AzureUtilities.DownloadBlobToTemp(myBlob, name, log);
var rowList = new List <NamTableRow>();
//2. Get values from file
using (GribFile file = new GribFile(localFileName))
{
log.Info($"Parsing file {name}");
rowList = GribUtilities.ParseNamGribFile(file);
}
//3. Format in correct table format
log.Info($"Attempting to sign in to ad for datalake upload");
var adlsAccountName = CloudConfigurationManager.GetSetting("ADLSAccountName");
//auth secrets
var domain = CloudConfigurationManager.GetSetting("Domain");
var webApp_clientId = CloudConfigurationManager.GetSetting("WebAppClientId");
var clientSecret = CloudConfigurationManager.GetSetting("ClientSecret");
var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
// Create client objects and set the subscription ID
var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
try
{
adlsFileSystemClient.FileSystem.UploadFile(adlsAccountName, localFileName, "/nam-grib-westus-v1/" + name, uploadAsBinary: true, overwrite: true);
log.Info($"Uploaded file: {localFileName}");
}
catch (Exception e)
{
log.Error($"Upload failed: {e.Message}");
}
MemoryStream s = new MemoryStream();
StreamWriter csvWriter = new StreamWriter(s, Encoding.UTF8);
csvWriter.WriteLine(NamTableRow.Columns);
MemoryStream sLocations = new MemoryStream();
StreamWriter csvLocationsWriter = new StreamWriter(sLocations, Encoding.UTF8);
csvLocationsWriter.WriteLine("Lat, Lon");
string fileName = null;
foreach (var row in rowList)
{
if (fileName == null)
{
fileName = row.PartitionKey + ".csv";
}
csvLocationsWriter.WriteLine(row.Lat + "," + row.Lon);
csvWriter.WriteLine(row.ToString());
}
csvWriter.Flush();
csvLocationsWriter.Flush();
s.Position = 0;
sLocations.Position = 0;
AzureUtilities.UploadLocationsFile(sLocations, log);
sLocations.Flush();
sLocations.Close();
log.Info($"Completed csv creation--attempting to upload to ADLS");
try
{
adlsFileSystemClient.FileSystem.Create(adlsAccountName, "/nam-csv-westus-v1/" + fileName, s, overwrite: true);
log.Info($"Uploaded csv stream: {localFileName}");
}
catch (Exception e)
{
log.Info($"Upload failed: {e.Message}");
}
s.Flush();
s.Close();
//delete local temp file
File.Delete(localFileName);
DateTime date = DateTime.ParseExact(name.Split('.')[0], "yyyyMMdd", null);
return(new FileProcessedTracker {
ForecastDate = date, PartitionKey = "nam-grib-westus-v1", RowKey = name, Url = "unknown"
});
}
static void Main(string[] args)
{
//using (GribFile file = new GribFile("../../SampleFiles/20171030.nam.t00z.awphys00.tm00.grib2"))
using (GribFile file = new GribFile(@"C:\Users\scott\AppData\Local\Temp\20171106.nam.t00z.awphys00.tm00.grib2"))
GribUtilities.ParseNamGribFile(file);
}
