public async Task Run([CosmosDBTrigger(
databaseName: "TelementryDB",
collectionName: "DeviceReading",
ConnectionStringSetting = "COSMOS_CONNECTION_STRING",
LeaseCollectionName = "leases",
CreateLeaseCollectionIfNotExists = true)] IReadOnlyList <Document> input)
{
try
{
List <DeviceReading> backupDocuments = new List <DeviceReading>();
CloudBlobClient cloudBlobClient = _azureStorageHelpers.ConnectToBlobClient(
_config[Constants.STORAGE_ACCOUNT_NAME],
_config[Constants.STORAGE_ACCOUNT_KEY]);
CloudBlobContainer blobContainer = _azureStorageHelpers.GetBlobContainer(
cloudBlobClient,
_config[Constants.STORAGE_CONTAINER]);
string backupFile = Path.Combine($"{DateTime.Now.ToString("dd-MM-yyyy")}-backup.json");
if (input != null && input.Count > 0)
{
foreach (var document in input)
{
// Persist to blob storage
var deviceReading = JsonConvert.DeserializeObject <DeviceReading>(document.ToString());
backupDocuments.Add(deviceReading);
_logger.LogInformation($"{document.Id} has been added to list");
}
}
var jsonData = JsonConvert.SerializeObject(backupDocuments);
using (StreamWriter file = File.CreateText(backupFile))
{
JsonSerializer serializer = new JsonSerializer();
serializer.Serialize(file, jsonData);
}
await _azureStorageHelpers.UploadBlobToStorage(blobContainer, backupFile);
}
catch (Exception ex)
{
_logger.LogWarning($"Something went wrong. Exception thrown: {ex.Message}");
throw;
}
}
private async Task TrainAndSaveModel(MLContext mlContext, string trainFilePath, string testFilePath, string modelPath, CloudBlobContainer cloudBlobContainer)
{
// Read flat file from local folder
_logger.LogInformation("Loading the file into the pipeline");
IDataView dataView = mlContext.Data.LoadFromTextFile <TaxiTrip>(trainFilePath, hasHeader: true, separatorChar: ',');
// Create the pipeline
_logger.LogInformation("Training pipeline");
var pipeline = mlContext.Transforms.CopyColumns(outputColumnName: "Label", inputColumnName: "FareAmount")
.Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "VendorIdEncoded", inputColumnName: "VendorId"))
.Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "RateCodeEncoded", inputColumnName: "RateCode"))
.Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "PaymentTypeEncoded", inputColumnName: "PaymentType"))
.Append(mlContext.Transforms.Concatenate("Features", "VendorIdEncoded", "RateCodeEncoded", "PassengerCount", "TripDistance", "PaymentTypeEncoded"))
.Append(mlContext.Regression.Trainers.FastTree());
// Fit the model
_logger.LogInformation("Fitting model");
var model = pipeline.Fit(dataView);
// Test the model
_logger.LogInformation("Evaluating the model");
var modelRSquaredValue = Evaluate(_mlContext, model, testFilePath);
_logger.LogInformation($"R-Squared value for model is {modelRSquaredValue}");
if (modelRSquaredValue >= 0.7)
{
_logger.LogInformation("Good fit! Saving model");
mlContext.Model.Save(model, dataView.Schema, modelPath);
// Upload Model to Blob Storage
_logger.LogInformation("Uploading model to Blob Storage");
await _azureStorageHelpers.UploadBlobToStorage(cloudBlobContainer, modelPath);
}
else
{
_logger.LogInformation("The model is a poor fit");
}
}
