public static void Run([TimerTrigger("0 20 * * * *", RunOnStartup = true)] TimerInfo myTimer,
[Queue("filereadytodownloadqueue", Connection = "AzureWebJobsStorage")] ICollector <FileReadyToDownloadQueueMessage> outputQueueItem,
TraceWriter log)
{
#if DEBUG
int numberOfDaysToCheck = 1;
#else
int numberOfDaysToCheck = 5;
#endif
log.Info($"C# DetectSnotelReadyForDownload Timer trigger function executed at: {DateTime.Now}");
//%HOUR% & %STATE% to be populated
string[] stateList = { "WA", "OR", "CA", "ID", "UT", "NV", "MT", "WY", "CO", "AZ", "NM", "AK" };
string partitionName = "snotel-csv-westus-v1";
// Retrieve storage account from connection string.
var storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureWebJobsStorage"));
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("snoteltracker");
table.CreateIfNotExists();
//look back eight days and fill in any missing values;
TableQuery <FileProcessedTracker> dateQuery = new TableQuery <FileProcessedTracker>().Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionName),
TableOperators.And,
TableQuery.GenerateFilterConditionForDate("ForecastDate", QueryComparisons.GreaterThan, DateTime.UtcNow.AddDays(-1 * numberOfDaysToCheck))
)
);
var results = table.ExecuteQuery(dateQuery);
//1. Are there any missing dates for the last n days we should backfill
var checkDate = DateTime.UtcNow.AddDays(-1 * numberOfDaysToCheck);;
#if DEBUG
stateList = new string[] { stateList[0] };
#endif
while (checkDate < DateTime.UtcNow)
{
foreach (var state in stateList)
{
string fileName = SnotelUtilities.CreateSnotelFileDate(checkDate) + "." + state + ".snotel.csv";
if (results.Where(r => r.RowKey == fileName).Count() == 0)
{
//If file doesn't exist enter a new item
log.Info($"backfill: adding item {fileName} to download queue");
CreateQueueItem(outputQueueItem, log, partitionName, checkDate, state);
}
else
{
log.Info($"Skipping item {fileName} as it already exists");
}
}
checkDate = checkDate.AddHours(1);
}
}
private static void CreateQueueItem(ICollector <FileReadyToDownloadQueueMessage> outputQueueItem,
TraceWriter log,
string partitionName,
DateTime readingDateUtc, string state)
{
string snotelTemplate = @"https://wcc.sc.egov.usda.gov/reportGenerator/view_csv/customMultipleStationReport/hourly/start_of_period/state=%22%STATE%%22%20AND%20network=%22SNTLT%22,%22SNTL%22%20AND%20element=%22SNWD%22%20AND%20outServiceDate=%222100-01-01%22%7Cname/%yyyy-MM-dd%,%yyyy-MM-dd%:H%7C%HOUR%/name,elevation,latitude,longitude,WTEQ::value,PREC::value,SNWD::value,TOBS::value";
string snotelUrl = CreateSnotelUrl(readingDateUtc, state, snotelTemplate);
//keep the file date utc; we'll correct the times in the file to UTC in the ADSL upload
var fileDate = SnotelUtilities.CreateSnotelFileDate(readingDateUtc);
log.Info($"Adding file {fileDate} with state {state} to download queue.");
//enter a new queue item
outputQueueItem.Add(new FileReadyToDownloadQueueMessage {
FileName = state + ".snotel.csv", FileDate = fileDate, Url = snotelUrl, Filetype = partitionName
});
}
public static void Run([TimerTrigger("0 30 * * * *", RunOnStartup = true)] TimerInfo myTimer, TraceWriter log)
{
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
int numberOfStates = 12;
// Retrieve storage account from connection string.
var storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureWebJobsStorage"));
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("snotelmergetracker");
table.CreateIfNotExists();
#if DEBUG
int numberOfHoursToCheck = 3 * 7 * 24; //was 1;
#else
int numberOfHoursToCheck = 7 * 24; //one week
#endif
//look back x days and fill in any missing values;
TableQuery <FileProcessedTracker> dateQuery = new TableQuery <FileProcessedTracker>().Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionName),
TableOperators.And,
TableQuery.GenerateFilterConditionForDate("ForecastDate", QueryComparisons.GreaterThan, DateTime.UtcNow.AddDays(-1 * numberOfHoursToCheck))
)
);
var results = table.ExecuteQuery(dateQuery);
//refactoring the below code to a shared method can cause an .net issue
//related to binding redirect to arise; leave this here for now. See AzureUtilities.cs
//for more info
log.Info($"Attempting to sign in to ad for datalake upload");
var adlsAccountName = CloudConfigurationManager.GetSetting("ADLSAccountName");
//auth secrets
var domain = CloudConfigurationManager.GetSetting("Domain");
var webApp_clientId = CloudConfigurationManager.GetSetting("WebAppClientId");
var clientSecret = CloudConfigurationManager.GetSetting("ClientSecret");
var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result;
var fullAdlsAccountName = CloudConfigurationManager.GetSetting("ADLSFullAccountName");
// Create client objects and set the subscription ID
var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
var adlsClient = AdlsClient.CreateClient(fullAdlsAccountName, creds);
var checkDate = DateTime.UtcNow.AddHours(-1 * numberOfHoursToCheck);
while (checkDate < DateTime.UtcNow)
{
//has it already been marked as complete in the table
string nameToCheck = SnotelUtilities.CreateSnotelFileDate(checkDate) + ".snotel.csv";
if (results.Where(r => r.RowKey == nameToCheck).Count() == 0)
{
log.Info($"{nameToCheck} doesn't exist in completed table, need to see if all files exist to concat");
var lexStartAndEnd = SnotelUtilities.CreateSnotelFileDate(checkDate);
var hourlyFilesOnAdls = adlsClient.EnumerateDirectory(csvDirectory).Where(f => f.Name.StartsWith(lexStartAndEnd)).Select(f => f.Name).ToList();
if (hourlyFilesOnAdls.Count == numberOfStates)
{
if (ConcatFiles(adlsClient, nameToCheck, hourlyFilesOnAdls))
{
//mark file as finished in table
FileProcessedTracker tracker = new FileProcessedTracker {
ForecastDate = checkDate, PartitionKey = partitionName, RowKey = nameToCheck, Url = "unknown"
};
table.Execute(TableOperation.Insert(tracker));
}
else
{
log.Error($"Missing data for {checkDate} need to manually backfill, can't concat");
}
}
else
{
log.Info($"all state files don't exist for {checkDate}, waiting until next run");
}
}
else
{
log.Info($"{nameToCheck} marked as already concated");
}
checkDate = checkDate.AddHours(1);
}
}
