public static void Run([TimerTrigger("0 20 * * * *", RunOnStartup = true)] TimerInfo myTimer, [Queue("filereadytodownloadqueue", Connection = "AzureWebJobsStorage")] ICollector <FileReadyToDownloadQueueMessage> outputQueueItem, TraceWriter log) { #if DEBUG int numberOfDaysToCheck = 1; #else int numberOfDaysToCheck = 5; #endif log.Info($"C# DetectSnotelReadyForDownload Timer trigger function executed at: {DateTime.Now}"); //%HOUR% & %STATE% to be populated string[] stateList = { "WA", "OR", "CA", "ID", "UT", "NV", "MT", "WY", "CO", "AZ", "NM", "AK" }; string partitionName = "snotel-csv-westus-v1"; // Retrieve storage account from connection string. var storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureWebJobsStorage")); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference("snoteltracker"); table.CreateIfNotExists(); //look back eight days and fill in any missing values; TableQuery <FileProcessedTracker> dateQuery = new TableQuery <FileProcessedTracker>().Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionName), TableOperators.And, TableQuery.GenerateFilterConditionForDate("ForecastDate", QueryComparisons.GreaterThan, DateTime.UtcNow.AddDays(-1 * numberOfDaysToCheck)) ) ); var results = table.ExecuteQuery(dateQuery); //1. Are there any missing dates for the last n days we should backfill var checkDate = DateTime.UtcNow.AddDays(-1 * numberOfDaysToCheck);; #if DEBUG stateList = new string[] { stateList[0] }; #endif while (checkDate < DateTime.UtcNow) { foreach (var state in stateList) { string fileName = SnotelUtilities.CreateSnotelFileDate(checkDate) + "." + state + ".snotel.csv"; if (results.Where(r => r.RowKey == fileName).Count() == 0) { //If file doesn't exist enter a new item log.Info($"backfill: adding item {fileName} to download queue"); CreateQueueItem(outputQueueItem, log, partitionName, checkDate, state); } else { log.Info($"Skipping item {fileName} as it already exists"); } } checkDate = checkDate.AddHours(1); } }
private static void CreateQueueItem(ICollector <FileReadyToDownloadQueueMessage> outputQueueItem, TraceWriter log, string partitionName, DateTime readingDateUtc, string state) { string snotelTemplate = @"https://wcc.sc.egov.usda.gov/reportGenerator/view_csv/customMultipleStationReport/hourly/start_of_period/state=%22%STATE%%22%20AND%20network=%22SNTLT%22,%22SNTL%22%20AND%20element=%22SNWD%22%20AND%20outServiceDate=%222100-01-01%22%7Cname/%yyyy-MM-dd%,%yyyy-MM-dd%:H%7C%HOUR%/name,elevation,latitude,longitude,WTEQ::value,PREC::value,SNWD::value,TOBS::value"; string snotelUrl = CreateSnotelUrl(readingDateUtc, state, snotelTemplate); //keep the file date utc; we'll correct the times in the file to UTC in the ADSL upload var fileDate = SnotelUtilities.CreateSnotelFileDate(readingDateUtc); log.Info($"Adding file {fileDate} with state {state} to download queue."); //enter a new queue item outputQueueItem.Add(new FileReadyToDownloadQueueMessage { FileName = state + ".snotel.csv", FileDate = fileDate, Url = snotelUrl, Filetype = partitionName }); }
public static void Run([TimerTrigger("0 30 * * * *", RunOnStartup = true)] TimerInfo myTimer, TraceWriter log) { log.Info($"C# Timer trigger function executed at: {DateTime.Now}"); int numberOfStates = 12; // Retrieve storage account from connection string. var storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureWebJobsStorage")); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference("snotelmergetracker"); table.CreateIfNotExists(); #if DEBUG int numberOfHoursToCheck = 3 * 7 * 24; //was 1; #else int numberOfHoursToCheck = 7 * 24; //one week #endif //look back x days and fill in any missing values; TableQuery <FileProcessedTracker> dateQuery = new TableQuery <FileProcessedTracker>().Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionName), TableOperators.And, TableQuery.GenerateFilterConditionForDate("ForecastDate", QueryComparisons.GreaterThan, DateTime.UtcNow.AddDays(-1 * numberOfHoursToCheck)) ) ); var results = table.ExecuteQuery(dateQuery); //refactoring the below code to a shared method can cause an .net issue //related to binding redirect to arise; leave this here for now. See AzureUtilities.cs //for more info log.Info($"Attempting to sign in to ad for datalake upload"); var adlsAccountName = CloudConfigurationManager.GetSetting("ADLSAccountName"); //auth secrets var domain = CloudConfigurationManager.GetSetting("Domain"); var webApp_clientId = CloudConfigurationManager.GetSetting("WebAppClientId"); var clientSecret = CloudConfigurationManager.GetSetting("ClientSecret"); var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result; var fullAdlsAccountName = CloudConfigurationManager.GetSetting("ADLSFullAccountName"); // Create client objects and set the subscription ID var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); var adlsClient = AdlsClient.CreateClient(fullAdlsAccountName, creds); var checkDate = DateTime.UtcNow.AddHours(-1 * numberOfHoursToCheck); while (checkDate < DateTime.UtcNow) { //has it already been marked as complete in the table string nameToCheck = SnotelUtilities.CreateSnotelFileDate(checkDate) + ".snotel.csv"; if (results.Where(r => r.RowKey == nameToCheck).Count() == 0) { log.Info($"{nameToCheck} doesn't exist in completed table, need to see if all files exist to concat"); var lexStartAndEnd = SnotelUtilities.CreateSnotelFileDate(checkDate); var hourlyFilesOnAdls = adlsClient.EnumerateDirectory(csvDirectory).Where(f => f.Name.StartsWith(lexStartAndEnd)).Select(f => f.Name).ToList(); if (hourlyFilesOnAdls.Count == numberOfStates) { if (ConcatFiles(adlsClient, nameToCheck, hourlyFilesOnAdls)) { //mark file as finished in table FileProcessedTracker tracker = new FileProcessedTracker { ForecastDate = checkDate, PartitionKey = partitionName, RowKey = nameToCheck, Url = "unknown" }; table.Execute(TableOperation.Insert(tracker)); } else { log.Error($"Missing data for {checkDate} need to manually backfill, can't concat"); } } else { log.Info($"all state files don't exist for {checkDate}, waiting until next run"); } } else { log.Info($"{nameToCheck} marked as already concated"); } checkDate = checkDate.AddHours(1); } }