private static void WriteMasterBlobCsv(StatFinDataset dataset, StatFinDatasetMeta datasetDetails, CloudBlobContainer blobContainer)
{
var blobPath = GetMasterBlobPathCsv(datasetDetails);
var blockBlob = blobContainer.GetBlockBlobReference(blobPath);
var datasetText = ToCsv(dataset);
blockBlob.UploadTextAsync(datasetText);
}
private static Uri ConstructRequestUri(StatFinDatasetMeta datasetDetails, string apiVersion, string resultsLanguage)
{
var baseAddress = new Uri(baseAddressString);
var tablePath = new Uri(
apiPath +
"/" + apiVersion +
"/" + resultsLanguage +
"/" + datasetDetails.Database +
"/" + datasetDetails.SubjectRealm +
"/" + datasetDetails.SubjectSubRealm +
"/" + datasetDetails.TableName + ".px",
UriKind.Relative
);
return(new Uri(baseAddress, tablePath));
}
public static async Task <HttpResponseMessage> RunAsync(
[HttpTrigger(AuthorizationLevel.Function, "get", Route = "datafetch/{blobContainerName}/{database}/{subjectRealm}/{subjectSubRealm}/{tableName}/{yearMin}/{yearMax}")] HttpRequestMessage triggerRequest,
string blobContainerName,
string database,
string subjectRealm,
string subjectSubRealm,
string tableName,
string yearMin,
string yearMax,
TraceWriter log
)
{
string apiVersion = "v1";
string resultsLanguage = "fi"; // TODO: find available languages and loop over (or include language variable in the dataset)
log.Info("C# HTTP trigger function begins to process a request.");
var datasetDetails = new StatFinDatasetMeta
{
Database = database,
SubjectRealm = subjectRealm,
SubjectSubRealm = subjectSubRealm,
TableName = tableName,
ResultsLanguage = resultsLanguage
};
var dataFetchUri = ConstructRequestUri(datasetDetails, apiVersion, resultsLanguage);
// Get data for each year individually and save to Blob
var years = GetYearRange(yearMin, yearMax);
foreach (var year in years)
{
var dataFetchRequestBody = ConstructRequestBody(new string[] { year });
var dataFetchResponse = await httpClient.PostAsJsonAsync(dataFetchUri, dataFetchRequestBody);
var dataFetchResponseString = await dataFetchResponse.Content.ReadAsStringAsync();
string blobFilePath = GetStageBlobPath(datasetDetails, year);
WriteToBlob(blobContainerName, blobFilePath, dataFetchResponseString);
}
log.Info("C# HTTP trigger function processed a request successfully.");
return(triggerRequest.CreateResponse(HttpStatusCode.OK));
}
public static HttpResponseMessage RunAsync(
[HttpTrigger(AuthorizationLevel.Function, "get", Route = "updatemasterdata/{blobContainerName}/{database}/{subjectRealm}/{subjectSubRealm}/{tableName}")] HttpRequestMessage triggerRequest,
string blobContainerName,
string database,
string subjectRealm,
string subjectSubRealm,
string tableName,
TraceWriter logger
)
{
var resultsLanguage = "fi"; // TODO: find available languages and loop over (or include language variable in the dataset)
logger.Info("Beginning to process a HTTP trigger request.");
var storageAccount = CloudStorageAccount.Parse(connectionString); // TODO: check for existence
var blobClient = storageAccount.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference(blobContainerName); // TODO: check for existence
var datasetDetails = new StatFinDatasetMeta
{
Database = database,
SubjectRealm = subjectRealm,
SubjectSubRealm = subjectSubRealm,
TableName = tableName,
ResultsLanguage = resultsLanguage
};
var rawDatasets = ReadStageBlob(blobContainer, datasetDetails);
var masterDataset = ToMasterDataset(rawDatasets, logger);
WriteMasterBlobCsv(masterDataset, datasetDetails, blobContainer);
// TODO: Save in JSON as well
logger.Info("HTTP trigger request processed successfully.");
return(triggerRequest.CreateResponse(HttpStatusCode.OK));
}
// TODO: Move this somewhere so that it can be used from other classes as well
private static string GetStageBlobPath(StatFinDatasetMeta datasetDetails, string year)
{
return("stage/" + datasetDetails.Database + "/" + datasetDetails.SubjectRealm + "/" + datasetDetails.SubjectSubRealm + "/" + datasetDetails.TableName + "/" + datasetDetails.ResultsLanguage + "/" + year + ".json");
}
private static StatFinDatasetRaw[] ReadStageBlob(CloudBlobContainer blobContainer, StatFinDatasetMeta datasetDetails)
{
var datasets = new List <StatFinDatasetRaw>();
var blobDir = GetStageDirectoryPath(datasetDetails);
var blobDirItems = blobContainer.ListBlobs(blobDir); // ListBlobs (for container) returns the directory itself, not its contents
foreach (var blobDirItem in blobDirItems)
{
var directory = (CloudBlobDirectory)blobDirItem;
var blobs = directory.ListBlobs();
foreach (var blobItem in blobs)
{
var blob = (CloudBlockBlob)blobItem;
var blobContentText = string.Empty;
// Read the Blob contents to memory
using (var memoryStream = new MemoryStream())
{
blob.DownloadToStream(memoryStream);
blobContentText = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray());
}
// TODO: try catch
var blobContent = JsonConvert.DeserializeObject <StatFinDatasetRaw>(blobContentText);
datasets.Add(blobContent);
}
}
return(datasets.ToArray());
}
// TODO: Move this somewhere so that it can be used from other classes as well
private static string GetMasterBlobPathJson(StatFinDatasetMeta blobFolder)
{
return("master/" + blobFolder.Database + "/" + blobFolder.SubjectRealm + "/" + blobFolder.SubjectSubRealm + "/" + blobFolder.ResultsLanguage + "/json/" + blobFolder.TableName + ".json");
}
// TODO: Move this somewhere so that it can be used from other classes as well
private static string GetStageDirectoryPath(StatFinDatasetMeta blobFolder)
{
return("stage/" + blobFolder.Database + "/" + blobFolder.SubjectRealm + "/" + blobFolder.SubjectSubRealm + "/" + blobFolder.TableName + "/" + blobFolder.ResultsLanguage);
}
private static string ToJson(StatFinDataset dataset, StatFinDatasetMeta datasetDetails, CloudBlobContainer blobContainer)
{
// TODO
return(String.Empty);
}
