private static void WriteMasterBlobCsv(StatFinDataset dataset, StatFinDatasetMeta datasetDetails, CloudBlobContainer blobContainer) { var blobPath = GetMasterBlobPathCsv(datasetDetails); var blockBlob = blobContainer.GetBlockBlobReference(blobPath); var datasetText = ToCsv(dataset); blockBlob.UploadTextAsync(datasetText); }
private static Uri ConstructRequestUri(StatFinDatasetMeta datasetDetails, string apiVersion, string resultsLanguage) { var baseAddress = new Uri(baseAddressString); var tablePath = new Uri( apiPath + "/" + apiVersion + "/" + resultsLanguage + "/" + datasetDetails.Database + "/" + datasetDetails.SubjectRealm + "/" + datasetDetails.SubjectSubRealm + "/" + datasetDetails.TableName + ".px", UriKind.Relative ); return(new Uri(baseAddress, tablePath)); }
public static async Task <HttpResponseMessage> RunAsync( [HttpTrigger(AuthorizationLevel.Function, "get", Route = "datafetch/{blobContainerName}/{database}/{subjectRealm}/{subjectSubRealm}/{tableName}/{yearMin}/{yearMax}")] HttpRequestMessage triggerRequest, string blobContainerName, string database, string subjectRealm, string subjectSubRealm, string tableName, string yearMin, string yearMax, TraceWriter log ) { string apiVersion = "v1"; string resultsLanguage = "fi"; // TODO: find available languages and loop over (or include language variable in the dataset) log.Info("C# HTTP trigger function begins to process a request."); var datasetDetails = new StatFinDatasetMeta { Database = database, SubjectRealm = subjectRealm, SubjectSubRealm = subjectSubRealm, TableName = tableName, ResultsLanguage = resultsLanguage }; var dataFetchUri = ConstructRequestUri(datasetDetails, apiVersion, resultsLanguage); // Get data for each year individually and save to Blob var years = GetYearRange(yearMin, yearMax); foreach (var year in years) { var dataFetchRequestBody = ConstructRequestBody(new string[] { year }); var dataFetchResponse = await httpClient.PostAsJsonAsync(dataFetchUri, dataFetchRequestBody); var dataFetchResponseString = await dataFetchResponse.Content.ReadAsStringAsync(); string blobFilePath = GetStageBlobPath(datasetDetails, year); WriteToBlob(blobContainerName, blobFilePath, dataFetchResponseString); } log.Info("C# HTTP trigger function processed a request successfully."); return(triggerRequest.CreateResponse(HttpStatusCode.OK)); }
public static HttpResponseMessage RunAsync( [HttpTrigger(AuthorizationLevel.Function, "get", Route = "updatemasterdata/{blobContainerName}/{database}/{subjectRealm}/{subjectSubRealm}/{tableName}")] HttpRequestMessage triggerRequest, string blobContainerName, string database, string subjectRealm, string subjectSubRealm, string tableName, TraceWriter logger ) { var resultsLanguage = "fi"; // TODO: find available languages and loop over (or include language variable in the dataset) logger.Info("Beginning to process a HTTP trigger request."); var storageAccount = CloudStorageAccount.Parse(connectionString); // TODO: check for existence var blobClient = storageAccount.CreateCloudBlobClient(); var blobContainer = blobClient.GetContainerReference(blobContainerName); // TODO: check for existence var datasetDetails = new StatFinDatasetMeta { Database = database, SubjectRealm = subjectRealm, SubjectSubRealm = subjectSubRealm, TableName = tableName, ResultsLanguage = resultsLanguage }; var rawDatasets = ReadStageBlob(blobContainer, datasetDetails); var masterDataset = ToMasterDataset(rawDatasets, logger); WriteMasterBlobCsv(masterDataset, datasetDetails, blobContainer); // TODO: Save in JSON as well logger.Info("HTTP trigger request processed successfully."); return(triggerRequest.CreateResponse(HttpStatusCode.OK)); }
// TODO: Move this somewhere so that it can be used from other classes as well private static string GetStageBlobPath(StatFinDatasetMeta datasetDetails, string year) { return("stage/" + datasetDetails.Database + "/" + datasetDetails.SubjectRealm + "/" + datasetDetails.SubjectSubRealm + "/" + datasetDetails.TableName + "/" + datasetDetails.ResultsLanguage + "/" + year + ".json"); }
private static StatFinDatasetRaw[] ReadStageBlob(CloudBlobContainer blobContainer, StatFinDatasetMeta datasetDetails) { var datasets = new List <StatFinDatasetRaw>(); var blobDir = GetStageDirectoryPath(datasetDetails); var blobDirItems = blobContainer.ListBlobs(blobDir); // ListBlobs (for container) returns the directory itself, not its contents foreach (var blobDirItem in blobDirItems) { var directory = (CloudBlobDirectory)blobDirItem; var blobs = directory.ListBlobs(); foreach (var blobItem in blobs) { var blob = (CloudBlockBlob)blobItem; var blobContentText = string.Empty; // Read the Blob contents to memory using (var memoryStream = new MemoryStream()) { blob.DownloadToStream(memoryStream); blobContentText = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray()); } // TODO: try catch var blobContent = JsonConvert.DeserializeObject <StatFinDatasetRaw>(blobContentText); datasets.Add(blobContent); } } return(datasets.ToArray()); }
// TODO: Move this somewhere so that it can be used from other classes as well private static string GetMasterBlobPathJson(StatFinDatasetMeta blobFolder) { return("master/" + blobFolder.Database + "/" + blobFolder.SubjectRealm + "/" + blobFolder.SubjectSubRealm + "/" + blobFolder.ResultsLanguage + "/json/" + blobFolder.TableName + ".json"); }
// TODO: Move this somewhere so that it can be used from other classes as well private static string GetStageDirectoryPath(StatFinDatasetMeta blobFolder) { return("stage/" + blobFolder.Database + "/" + blobFolder.SubjectRealm + "/" + blobFolder.SubjectSubRealm + "/" + blobFolder.TableName + "/" + blobFolder.ResultsLanguage); }
private static string ToJson(StatFinDataset dataset, StatFinDatasetMeta datasetDetails, CloudBlobContainer blobContainer) { // TODO return(String.Empty); }