public void TestRemoveBardCodes()
{
SnodasUtilities.RemoveBardCodesFromHdr("../../TestFiles/us_ssmv11038wS__A0024TTNATS2017121005DP001.Hdr", "./FixedBardCodes.Hdr");
GdalConfiguration.ConfigureGdal();
string datFileName = "us_ssmv11038wS__A0024TTNATS2017121005DP001.dat";
File.Copy(@"../../TestFiles/" + datFileName, "./" + datFileName, true);
var dataSet = Gdal.Open("./FixedBardCodes.Hdr", Access.GA_ReadOnly);
double[] geoTransform = new double[6];
dataSet.GetGeoTransform(geoTransform);
Assert.AreEqual(-124.73, Math.Round(geoTransform[0], 2), "First value of geoTransform is not as expected");
}
public void TestSnowdasListForCoordinates()
{
GdalConfiguration.ConfigureGdal();
double Lat = 47.45871;
double Lon = -121.4170309;
var coordinateList = new List <(double, double)>
{
(Lat, Lon)
};
var filePath = "../../TestFiles/us_ssmv11036tS__T0001TTNATS2017121005HP001.Hdr";
var results = new Dictionary <(double Lat, double Lon), SnodasRow>();
SnodasUtilities.SnowdasListForCoordinates(coordinateList, results, filePath);
var result = results.Values.ToList()[0];
Assert.AreEqual(844, result.SNOWDAS_SnowDepth_mm, "Incorrect snow depth value");
Assert.AreEqual(new DateTime(2017, 12, 10), result.Date.Date, "Incorrect date value");
}
public void TestGetValuesForCoordinates()
{
GdalConfiguration.ConfigureGdal();
double Lat = 47.45871;
double Lon = -121.4170309;
double Lat1 = 48.45871;
double Lon1 = -120.4170309;
var coordinateList = new List <(double, double)>
{
(Lat, Lon), (Lat1, Lon1)
};
var filePaths = new List <string>()
{
"../../TestFiles/us_ssmv11036tS__T0001TTNATS2017121005HP001.Hdr"
};
var results = SnodasUtilities.GetValuesForCoordinates(coordinateList, filePaths);
Assert.AreEqual(844, results.Where(r => r.Lat == Lat).First().SNOWDAS_SnowDepth_mm, "Incorrect snow depth value");
Assert.AreEqual(664, results.Where(r => r.Lat == Lat1).First().SNOWDAS_SnowDepth_mm, "Incorrect snow depth value");
}
}
public void TestExtractFiles()
{
string snodasFileToUncompress = "../../TestFiles/SNODAS_20171210.tar";
var files = SnodasUtilities.UnpackSnodasArchive(snodasFileToUncompress);
Assert.AreEqual(16, files.Count, "Incorrect total number of files");
Assert.IsTrue(File.Exists(files[0]), $"File {files[0]} doesn't exist");
int countHdr = 0, countDat = 0;
foreach (var f in files)
{
var extension = f.Split('.')[1];
if (extension == "Hdr")
{
countHdr++;
}
else if (extension == "dat")
{
countDat++;
}
}
Assert.AreEqual(8, countHdr, "Expected 8 hdr files");
Assert.AreEqual(8, countDat, "Expected 8 dat files");
}
public static FileProcessedTracker Run([QueueTrigger("downloadandunpacksnodas", Connection = "AzureWebJobsStorage")] FileReadyToDownloadQueueMessage myQueueItem,
TraceWriter log)
{
log.Info($"C# Queue trigger function processed snodas date: {myQueueItem.FileDate}");
string partitionName = myQueueItem.Filetype;
var urlToDownload = myQueueItem.Url;
log.Info($"Downloading Url {urlToDownload}");
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(urlToDownload);
request.Method = WebRequestMethods.Ftp.DownloadFile;
// This example assumes the FTP site uses anonymous logon.
request.Credentials = new NetworkCredential("anonymous", "");
List <string> listOfUnpackedFiles = null;
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
log.Info($"File {urlToDownload} downloaded");
Stream responseStream = response.GetResponseStream();
listOfUnpackedFiles = SnodasUtilities.UnpackSnodasStream(responseStream);
log.Info($"File {urlToDownload} unpacked");
//fix the bard codes in the hdr files
foreach (var f in listOfUnpackedFiles.Where(s => s.ToLower().Contains(".hdr")))
{
SnodasUtilities.RemoveBardCodesFromHdr(f);
}
log.Info($"Attempting to sign in to ad for datalake upload");
var adlsAccountName = CloudConfigurationManager.GetSetting("ADLSAccountName");
//auth secrets
var domain = CloudConfigurationManager.GetSetting("Domain");
var webApp_clientId = CloudConfigurationManager.GetSetting("WebAppClientId");
var clientSecret = CloudConfigurationManager.GetSetting("ClientSecret");
var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
// Create client objects and set the subscription ID
var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
log.Info($"Attempting to upload unpacked files to adls");
#if DEBUG
// listOfUnpackedFiles = listOfUnpackedFiles.Where(f => f.Contains(".Hdr")).Take(1).ToList();
#endif
foreach (var file in listOfUnpackedFiles)
{
try
{
adlsFileSystemClient.FileSystem.UploadFile(adlsAccountName, file, "/snodas-dat-us-v1/" + file.Split('\\').Last(), uploadAsBinary: true, overwrite: true);
log.Info($"Uploaded file: {file}");
}
catch (Exception e)
{
log.Error($"Upload failed: {e.Message}");
}
}
//1: Get values for lat/lon
var locations = AzureUtilities.DownloadLocations(log);
#if DEBUG
var executingAssemblyFile = new Uri(Assembly.GetExecutingAssembly().GetName().CodeBase).LocalPath;
var executingDirectory = Path.GetDirectoryName(executingAssemblyFile);
if (string.IsNullOrEmpty(executingDirectory))
{
throw new InvalidOperationException("cannot get executing directory");
}
executingDirectory = Directory.GetParent(executingDirectory).FullName;
var gdalPath = Path.Combine(executingDirectory, "gdal");
log.Info($"Have gdal path {gdalPath}");
#endif
log.Info($"Configuring gdal");
GdalConfiguration.ConfigureGdal();
var results = SnodasUtilities.GetValuesForCoordinates(locations, listOfUnpackedFiles.Where(f => f.Contains(".Hdr")).ToList());
log.Info($"Have {results.Count} results for coordinates.");
DateTime fileDate;
string fileName;
using (MemoryStream s = new MemoryStream())
using (StreamWriter csvWriter = new StreamWriter(s, Encoding.UTF8))
{
csvWriter.WriteLine(SnodasRow.GetHeader);
foreach (var row in results)
{
csvWriter.WriteLine(row.ToString());
}
csvWriter.Flush();
s.Position = 0;
fileDate = results[0].Date;
fileName = fileDate.ToString("yyyyMMdd") + "Snodas.csv";
try
{
adlsFileSystemClient.FileSystem.Create(adlsAccountName, "/snodas-csv-westus-v1/" + fileName, s, overwrite: true);
log.Info($"Uploaded csv stream: {fileName}");
}
catch (Exception e)
{
log.Info($"Upload failed: {e.Message}");
}
}
log.Info($"Removing unpacked files");
foreach (var f in listOfUnpackedFiles)
{
//delete local temp file
File.Delete(f);
}
return(new FileProcessedTracker {
ForecastDate = fileDate, PartitionKey = "snodas-westus-v1", RowKey = fileName, Url = "unknown"
});
}
