public HttpResponseMessage PostRunUEB()
{
HttpResponseMessage response = new HttpResponseMessage();
ServiceContext db = new ServiceContext();
string modelRunRootPath = string.Empty;
string uebInputPackageZipFileName = "ueb_input_package.zip";
string uebInputPackageZipFile = string.Empty;
string uebExecutableFilesPath = string.Empty;
string uebExecutionControlFileName = UEB.UEBSettings.UEB_EXECUTION_CONTROL_FILE_NAME;
string msg = string.Empty;
ServiceLog serviceLog = null;
// get the ueb model package zip file from the request
Stream uebPackageZipFileFromClient = null;
try
{
var t_stream = this.Request.Content.ReadAsStreamAsync().ContinueWith(s =>
{
uebPackageZipFileFromClient = s.Result;
});
t_stream.Wait();
msg = string.Format("Input model package zip file was read from the http request.");
logger.Info(msg);
}
catch (Exception ex)
{
logger.Fatal(ex.Message);
return Request.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message);
}
if (uebPackageZipFileFromClient == null)
{
string errMsg = "No model package file was received from the client.";
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(errMsg);
return response;
}
// generate a guid to pass on to the client as a job ID and use this as part of creating a unique folder
// for model run output
Guid jobGuid = Guid.NewGuid();
modelRunRootPath = Path.Combine(UEB.UEBSettings.WORKING_DIR_PATH, jobGuid.ToString(), UEB.UEBSettings.UEB_RUN_FOLDER_NAME);
uebInputPackageZipFile = Path.Combine(modelRunRootPath, uebInputPackageZipFileName);
Directory.CreateDirectory(modelRunRootPath);
msg = string.Format("Directory ({0}) was created for running ueb model.", modelRunRootPath);
logger.Info(msg);
try
{
var service = db.Services.First(s => s.APIName == "RunUEBController.PostRunUEB");
serviceLog = new ServiceLog
{
JobID = jobGuid.ToString(),
ServiceID = service.ServiceID,
CallTime = DateTime.Now,
StartTime = DateTime.Now,
RunStatus = RunStatus.Processing
};
db.ServiceLogs.Add(serviceLog);
db.SaveChanges();
serviceLog = db.ServiceLogs.First(s => s.JobID == serviceLog.JobID);
// save the recieved package file locally for ueb to use
using (var fileStream = File.Create(uebInputPackageZipFile))
{
uebPackageZipFileFromClient.CopyTo(fileStream);
}
msg = string.Format("Input model package zip file was saved: {0}.", uebInputPackageZipFile);
logger.Info(msg);
}
catch (Exception ex)
{
serviceLog.RunStatus = RunStatus.Error;
serviceLog.FinishTime = DateTime.Now;
serviceLog.Error = ex.Message;
db.SaveChanges();
logger.Fatal(ex.Message);
return Request.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message);
}
// unzip the request zip file
ZipFile.ExtractToDirectory(uebInputPackageZipFile, modelRunRootPath);
// copy the UEB executables and dlls to model run folder
uebExecutableFilesPath = UEB.UEBSettings.UEB_EXECUTABLE_DIR_PATH;
string[] files = Directory.GetFiles(uebExecutableFilesPath);
string fileName = string.Empty;
string destFile = string.Empty;
foreach (string file in files)
{
// Use static Path methods to extract only the file name from the path.
fileName = Path.GetFileName(file);
destFile = Path.Combine(modelRunRootPath, fileName);
// Copy the file and overwrite destination file if they already exist.
File.Copy(file, destFile, true);
}
//Run ueb in async mode
RunUEB(modelRunRootPath, uebInputPackageZipFile, jobGuid.ToString());
UebRunStatusResponse uebRunStatus = new UebRunStatusResponse();
uebRunStatus.Message = "UEB execution has started.";
uebRunStatus.RunJobID = jobGuid.ToString();
string jsonResponse = JsonConvert.SerializeObject(uebRunStatus, Formatting.Indented);
response.StatusCode = HttpStatusCode.OK;
response.Content = new StringContent(jsonResponse);
return response;
}
/// <summary>
/// Sends the package zip file to the client (browser)
/// Pacakge zip file must pre-exists
/// </summary>
/// <param name="packageID">Id of the package</param>
/// <returns>A zip file containing all input data files and control files</returns>
public HttpResponseMessage GetPackageDownload(string packageID)
{
HttpResponseMessage response = new HttpResponseMessage();
ServiceContext db = new ServiceContext();
string targetPackageRootDirPath = Path.Combine(UEB.UEBSettings.WORKING_DIR_PATH, packageID);
string targetPackageDirPath = Path.Combine(targetPackageRootDirPath, UEB.UEBSettings.PACKAGE_OUTPUT_SUB_DIR_PATH);
string packageZipFileName = UEB.UEBSettings.UEB_PACKAGE_FILE_NAME;
if (string.IsNullOrEmpty(packageID))
{
string errMsg = string.Format("No package ID was provided");
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(errMsg);
return response;
}
ServiceLog serviceLog = db.ServiceLogs.FirstOrDefault(sl => sl.JobID == packageID);
if (serviceLog == null)
{
string errMsg = string.Format("No UEB package build request was found for the provided package ID: {0}.", packageID);
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.NotFound;
response.Content = new StringContent(errMsg);
return response;
}
if (serviceLog.RunStatus != RunStatus.Success)
{
string errMsg = string.Empty;
if (serviceLog.RunStatus == RunStatus.InQueue)
{
errMsg = string.Format("UEB package build request still in a queue to be processed for package ID: {0}.", packageID);
}
else if (serviceLog.RunStatus == RunStatus.Processing)
{
errMsg = string.Format("UEB package build request is currently being processed for package ID: {0}.", packageID);
}
else if (serviceLog.RunStatus == RunStatus.Error)
{
errMsg = string.Format("UEB package build request failed processing for package ID: {0}.", packageID);
}
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.NotFound;
response.Content = new StringContent(errMsg);
return response;
}
if (Directory.Exists(targetPackageRootDirPath) == false) // if the folder doesnot exist at this point, it means package has been already sent to client in the past
{
string errMsg = string.Format("UEB model package data no more available for the provided package ID: {0}.", packageID);
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.NotFound;
response.Content = new StringContent(errMsg);
return response;
}
if (Directory.Exists(targetPackageDirPath) == false) // if the folder doesnot exist at this point, we got the logic wrong
{
string errMsg = string.Format("No UEB package was found for the provided package ID: {0}.", packageID);
logger.Error("Internal logic error. " + errMsg);
response.StatusCode = HttpStatusCode.InternalServerError;
response.Content = new StringContent(errMsg);
return response;
}
try
{
string zipUEBPackageFile = Path.Combine(targetPackageDirPath, packageZipFileName);
// make sure the package zip file exists
if (!File.Exists(zipUEBPackageFile))
{
string errMsg = string.Format("No package file ({0}) was found.", zipUEBPackageFile);
logger.Error(errMsg);
return Request.CreateErrorResponse(HttpStatusCode.NotFound, errMsg);
}
FileStream fileStream = File.Open(zipUEBPackageFile, FileMode.Open, FileAccess.Read);
response.StatusCode = HttpStatusCode.OK;
response.Content = new StreamContent(fileStream);
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
response.Content.Headers.ContentLength = fileStream.Length;
// Ref: http://weblogs.asp.net/cibrax/archive/2011/04/25/implementing-caching-in-your-wcf-web-apis.aspx
// set the browser to cache this response for 120 secs only
response.Content.Headers.Expires = DateTime.Now.AddSeconds(120);
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = packageZipFileName
};
logger.Info(string.Format("UEB package zip file for PackageID: {0} was sent to the client.", packageID));
// delete temporary working folder used for creating ueb model package
Task deleteTask = new Task(() =>
{
// the zip file may be locked untl the client finish reading the
// data from the file stream object
while (FileManager.IsFileLocked(new FileInfo(zipUEBPackageFile)))
{
// wait 3 seconds
Thread.Sleep(TimeSpan.FromSeconds(3));
}
if (Directory.Exists(targetPackageRootDirPath))
{
Directory.Delete(targetPackageRootDirPath, true);
logger.Info(string.Format("UEB build package temporary folder: {0} was deleted.", targetPackageRootDirPath));
}
});
deleteTask.Start();
}
catch (Exception ex)
{
logger.Fatal(ex.Message);
return Request.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message);
}
return response;
}
private void RunUEB(string modelRunRootPath, string uebInputPackageZipFile, string runJobID)
{
ServiceContext db = new ServiceContext();
ServiceLog serviceLog = db.ServiceLogs.First(s => s.JobID == runJobID);
try
{
Task uebRunTask = new Task(() =>
{
System.Diagnostics.Process proc = null;
List<string> arguments = new List<string>();
arguments.Add(Path.Combine(modelRunRootPath, "UEBGrid.exe"));
arguments.Add(UEB.UEBSettings.UEB_EXECUTION_CONTROL_FILE_NAME);
// create a string containing all the argument items separated by a space
string commandString = string.Join(" ", arguments);
object command = commandString;
// Create the ProcessStartInfo using "cmd" as the program to be run,
// and "/c " as the parameters.
// "/c" tells cmd that you want it to execute the command that follows,
// then exit.
System.Diagnostics.ProcessStartInfo procStartInfo = new
System.Diagnostics.ProcessStartInfo("cmd", "/c " + command);
procStartInfo.WorkingDirectory = modelRunRootPath;
// The following commands are needed to redirect the standard output.
// redirect to the Process.StandardOutput StreamReader.
procStartInfo.RedirectStandardOutput = true;
procStartInfo.RedirectStandardError = true;
procStartInfo.UseShellExecute = false;
// Do not create the black window.
procStartInfo.CreateNoWindow = true;
// Now you create a process, assign its ProcessStartInfo, and start it.
logger.Info(string.Format("Starting UEB run for job ID: {0}.", runJobID));
proc = new System.Diagnostics.Process();
proc.StartInfo = procStartInfo;
proc.Start();
// Get the output into a string.
string result = proc.StandardOutput.ReadToEnd();
string errors = proc.StandardError.ReadToEnd();
logger.Info(string.Format("Ending UEB run for job ID: {0}.", runJobID));
if (result.Contains("successfully performed") == false || result.Length == 0)
{
string errMsg = string.Format("UEB run failed for job ID:{0}.", runJobID);
logger.Error(errMsg);
logger.Error(errors);
logger.Error(result);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.FinishTime = DateTime.Now;
serviceLog.Error = errors;
db.SaveChanges();
}
else
{
string msg = string.Format("UEB run was successful for job ID:{0}", runJobID);
logger.Info(msg);
string modelOutputZipPath = Path.Combine(modelRunRootPath, "outputszip");
string modelOutputPath = Path.Combine(modelRunRootPath, UEB.UEBSettings.MODEL_OUTPUT_FOLDER_NAME);
string modelOutputZipFile = Path.Combine(modelOutputZipPath, UEB.UEBSettings.UEB_RUN_OUTPUT_ZIP_FILE_NAME);
Directory.CreateDirectory(modelOutputZipPath);
ZipFile.CreateFromDirectory(modelOutputPath, modelOutputZipFile);
msg = string.Format("UEB run output zip file was created for UEB run job ID:{0}", runJobID);
logger.Info(msg);
serviceLog.RunStatus = RunStatus.Success;
serviceLog.FinishTime = DateTime.Now;
db.SaveChanges();
}
});
uebRunTask.Start();
}
catch (Exception ex)
{
string errMsg = string.Format("UEB run failed for job ID:{0}.", runJobID);
logger.Error(errMsg);
logger.Error(ex.Message);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.FinishTime = DateTime.Now;
serviceLog.Error = ex.Message;
db.SaveChanges();
}
}
/// <summary>
/// Get a zip file containing all shape files
/// </summary>
/// <remarks>
/// HTTP GET call format:http://{server}/api/EPADelineate?watershedOutletLat=latValue&watershedOutletLon=lonValue
/// </remarks>
/// <param name="watershedOutletLat">Latidue of the watershed outlet location based on WGS84 datum</param>
/// <param name="watershedOutletLon">Longitude of the watershed outlet location based on WGS84 datum</param>
/// <returns>A zip file of all shape files, shape files data based on WGS84 datum</returns>
public HttpResponseMessage GetShapeFiles(double watershedOutletLat, double watershedOutletLon)
{
HttpResponseMessage response = new HttpResponseMessage();
ServiceContext db = new ServiceContext();
ServiceLog serviceLog = null;
try
{
var service = db.Services.First(s => s.APIName == "EPADelineateController.GetShapeFiles");
serviceLog = new ServiceLog
{
JobID = Guid.NewGuid().ToString(),
ServiceID = service.ServiceID,
CallTime = DateTime.Now,
StartTime = DateTime.Now,
RunStatus = RunStatus.Processing
};
db.ServiceLogs.Add(serviceLog);
db.SaveChanges();
serviceLog = db.ServiceLogs.First(s => s.JobID == serviceLog.JobID);
Delineate(watershedOutletLat, watershedOutletLon);
// Ref:http://stackoverflow.com/questions/12145390/how-to-set-downloading-file-name-in-asp-net-mvc-web-api
// create a zip file of all shapes files
string zipFilePath = Path.Combine(_targetShapeFileZipDirPath, "shapefiles.zip");
if (File.Exists(zipFilePath))
{
File.Delete(zipFilePath);
}
ZipFile.CreateFromDirectory(_targetShapeFilesDirPath, zipFilePath);
// load the zip file to memory
MemoryStream ms = new MemoryStream();
using (FileStream file = new FileStream(zipFilePath, FileMode.Open, FileAccess.Read))
{
file.CopyTo(ms);
file.Close();
if (ms.Length == 0)
{
string errMsg = "No watershed exists for the selected outlet location.";
logger.Error(errMsg);
return Request.CreateErrorResponse(HttpStatusCode.NotFound, errMsg);
}
response.StatusCode = HttpStatusCode.OK;
ms.Position = 0;
response.Content = new StreamContent(ms);
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
// Ref: http://weblogs.asp.net/cibrax/archive/2011/04/25/implementing-caching-in-your-wcf-web-apis.aspx
// set the browser to cache this response for 10 secs only
response.Content.Headers.Expires = DateTime.Now.AddSeconds(10);
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = UEB.UEBSettings.WATERSHED_SHAPE_ZIP_FILE_NAME
};
logger.Info("All shape files were zipped and sent to the client as a zip file.");
}
// clean up the temporary folders for shape files
DirectoryInfo dir = new DirectoryInfo(_targetShapeFileGuidDirPath);
dir.Delete(true);
serviceLog.RunStatus = RunStatus.Success;
serviceLog.FinishTime = DateTime.Now;
db.SaveChanges();
}
catch (Exception ex)
{
logger.Fatal(ex.Message);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.FinishTime = DateTime.Now;
serviceLog.Error = ex.Message;
db.SaveChanges();
return Request.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message);
}
return response;
}
public int Run()
{
HttpResponseMessage response = new HttpResponseMessage();
ResponseMessage daymetResponse;
var tokenSource = new CancellationTokenSource();
CancellationToken cancellationToken = tokenSource.Token;
ServiceContext db = new ServiceContext();
int numberOfJobsStarted = 0;
// check if a ueb build package process is currently running
ServiceLog serviceInProcess = db.ServiceLogs.OrderBy(s => s.CallTime).FirstOrDefault(s => s.RunStatus == RunStatus.Processing & s.Service.APIName == "GenerateUEBPackageController.PostUEBPackageCreate");
if (serviceInProcess != null)
{
// In order to avoid a situtation where the status of UEB package build
// did not get changed from Processing to some other status, no other build package
// process could start. So here we change the status of a package build process from Processing to
// Error if the process has started more than 10 hours ago
// TODO: the 10 hour limit need to be specified in the ueb.cfig file
// The JobRunner console application that runs this function using the
// windows task scheduler has been configured to stop the JobRunner application if it runs more than 10 hours
if (DateTime.Now.Subtract((DateTime)serviceInProcess.StartTime).TotalHours > 10)
{
serviceInProcess.FinishTime = DateTime.Now;
serviceInProcess.RunStatus = RunStatus.Error;
serviceInProcess.Error = "Processing took too long.";
db.SaveChanges();
}
else
{
return numberOfJobsStarted;
}
}
ServiceLog serviceInQueue = db.ServiceLogs.OrderBy(s => s.CallTime).FirstOrDefault(s => s.RunStatus == RunStatus.InQueue & s.Service.APIName == "GenerateUEBPackageController.PostUEBPackageCreate");
if (serviceInQueue == null) return numberOfJobsStarted;
serviceInQueue.RunStatus = RunStatus.Processing;
serviceInQueue.StartTime = DateTime.Now;
db.SaveChanges();
// set the package request processing root dir path
string jobGuid = serviceInQueue.JobID;
_packageRequestProcessRootDirPath = Path.Combine(UEB.UEBSettings.WORKING_DIR_PATH, serviceInQueue.JobID);
// set other directory paths necessary for processing the request to build ueb package
_sourceFilePath = _packageRequestProcessRootDirPath;
_targetTempPackageFilePath = Path.Combine(_packageRequestProcessRootDirPath, UEB.UEBSettings.PACKAGE_FILES_OUTPUT_SUB_DIR_PATH);
_targetPackageDirPath = Path.Combine(_packageRequestProcessRootDirPath, UEB.UEBSettings.PACKAGE_OUTPUT_SUB_DIR_PATH);
// set the path for saving the client request zip file
_clientPackageRequestDirPath = Path.Combine(_packageRequestProcessRootDirPath, UEB.UEBSettings.PACKAGE_BUILD_REQUEST_SUB_DIR_PATH);
// read the file with json extension to a string - uebPackageBuildRequestJson
string[] jsonFiles = Directory.GetFiles(_clientPackageRequestDirPath, "*.json");
string uebPackageBuildRequestJson = string.Empty;
using (var fileReader = new StreamReader(jsonFiles[0]))
{
uebPackageBuildRequestJson = fileReader.ReadToEnd();
}
UEB.UEBPackageRequest uebPkgRequest;
uebPkgRequest = JsonConvert.DeserializeObject<UEB.UEBPackageRequest>(uebPackageBuildRequestJson);
string scriptErrMsg = string.Empty;
numberOfJobsStarted = 1;
try
{
Task mainTask = new Task(() =>
{
float constantWindSpeed = UEB.UEBSettings.WATERSHED_CONSTANT_WIND_SPEED;
response = GenerateBufferedWatershedFiles(cancellationToken, uebPkgRequest.BufferSize);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate buffered watershed shape files seems to have failed.";
logger.Error( scriptErrMsg + " for job id:" + jobGuid);
}
response = GenerateWatershedDEMFile(cancellationToken, uebPkgRequest.GridCellSize);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate watershed DEM file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
response = GenerateWatershedNetCDFFile(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate waterhsed netcdf domain file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
if (uebPkgRequest.SiteInitialConditions.is_apr_derive_from_elevation)
{
response = GetWatershedAtmosphericPressure(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate atmospheric pressure netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
}
response = GetWatershedSlopeNetCDFFile(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate Slope netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
response = GetWatershedAspectNetCDFFile(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate Aspect netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
response = GetWatershedLatLonValues(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate Lat/Lon data points netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
response = GetWatershedLandCoverData(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate land cover data seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
response = GetWatershedLandCoverVariablesData(cancellationToken);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate land cover variables data points netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
string outputTminDataVarName = UEB.UEBSettings.WATERSHED_SINGLE_TEMP_MIN_NETCDF_VARIABLE_NAME; // "tmin";
string inputDaymetTminFileNamePattern = UEB.UEBSettings.DAYMET_RESOURCE_TEMP_MIN_FILE_NAME_PATTERN; // "tmin*.nc";
daymetResponse = DataProcessor.GetWatershedSingleTempDataPointNetCDFFile(cancellationToken, outputTminDataVarName,
inputDaymetTminFileNamePattern, _packageRequestProcessRootDirPath, uebPkgRequest.StartDate, uebPkgRequest.EndDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate single Temp Min data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
string outputTmaxDataVarName = UEB.UEBSettings.WATERSHED_SINGLE_TEMP_MAX_NETCDF_VARIABLE_NAME; ; // "tmax";
string inputDaymetTmaxFileNamePattern = UEB.UEBSettings.DAYMET_RESOURCE_TEMP_MAX_FILE_NAME_PATTERN; ; // "tmax*.nc";
daymetResponse = DataProcessor.GetWatershedSingleTempDataPointNetCDFFile(cancellationToken, outputTmaxDataVarName,
inputDaymetTmaxFileNamePattern, _packageRequestProcessRootDirPath, uebPkgRequest.StartDate, uebPkgRequest.EndDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate single Temp Max data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
daymetResponse = DataProcessor.GetWatershedMultipleTempDataPointsNetCDFFile(cancellationToken, uebPkgRequest.TimeStep,
_packageRequestProcessRootDirPath, uebPkgRequest.StartDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate multiple Temperature data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
string inputDaymetVpFileNamePattern = UEB.UEBSettings.DAYMET_RESOURCE_VP_FILE_NAME_PATTERN; // "vp*.nc";
daymetResponse = DataProcessor.GetWatershedSingleVaporPresDataPointNetCDFFile(cancellationToken, inputDaymetVpFileNamePattern,
_packageRequestProcessRootDirPath, uebPkgRequest.StartDate, uebPkgRequest.EndDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate single VP data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
daymetResponse = DataProcessor.GetWatershedMultipleVaporPresDataPointsNetCDFFile(cancellationToken, uebPkgRequest.TimeStep,
_packageRequestProcessRootDirPath, uebPkgRequest.StartDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate multiple VP data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
string inputDaymetPrcpFileNamePattern = UEB.UEBSettings.DAYMET_RESOURCE_PRECP_FILE_NAME_PATTERN; // "prcp*.nc";
daymetResponse = DataProcessor.GetWatershedSinglePrecpDataPointNetCDFFile(cancellationToken, inputDaymetPrcpFileNamePattern,
_packageRequestProcessRootDirPath, uebPkgRequest.StartDate, uebPkgRequest.EndDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate single Prec data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
daymetResponse = DataProcessor.GetWatershedMultiplePrecpDataPointsNetCDFFile(cancellationToken, uebPkgRequest.TimeStep,
_packageRequestProcessRootDirPath, uebPkgRequest.StartDate);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate multiple Prec data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
daymetResponse = DataProcessor.GetWatershedMultipleWindDataPointsNetCDFFile(cancellationToken, constantWindSpeed, _packageRequestProcessRootDirPath);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate multiple Wind data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
daymetResponse = DataProcessor.GetWatershedMultipleRHDataPointsNetCDFFile(cancellationToken, uebPkgRequest.TimeStep, _packageRequestProcessRootDirPath);
if (daymetResponse.StatusCode != ResponseStatus.OK)
{
tokenSource.Cancel();
scriptErrMsg = "Python script to generate multiple RH data points per day netcdf file seems to have failed.";
logger.Error(scriptErrMsg + " for job id:" + jobGuid);
}
if (tokenSource.IsCancellationRequested == false)
{
response = CreateUEBpackageZipFile(jobGuid, uebPkgRequest.StartDate, uebPkgRequest.EndDate, uebPkgRequest.TimeStep, uebPkgRequest);
if (response.StatusCode != HttpStatusCode.OK)
{
tokenSource.Cancel();
}
}
else
{
logger.Error("One of the python scripts execution got cancelled due to execution error. No package was created for job id:" + jobGuid);
}
}, tokenSource.Token);
mainTask.Start();
Helpers.TaskDataStore.SetTask(jobGuid, mainTask);
mainTask.ContinueWith((t) =>
{
if (mainTask.IsCanceled || mainTask.IsFaulted || tokenSource.IsCancellationRequested)
{
CleanUpOnFailure();
string errorMsg = "UEB pacakage could not be created for job id:" + jobGuid;
logger.Error(errorMsg);
serviceInQueue.RunStatus = RunStatus.Error;
serviceInQueue.FinishTime = DateTime.Now;
serviceInQueue.Error = scriptErrMsg;
db.SaveChanges();
}
else
{
HttpResponseMessage mainTaskResponse = new HttpResponseMessage();
mainTaskResponse.StatusCode = HttpStatusCode.OK;
mainTaskResponse.Content = new StringContent("UEB package creation is complete for job id:" + jobGuid);
Helpers.TaskDataStore.SetTaskResult(jobGuid, mainTaskResponse);
logger.Info(mainTaskResponse.Content.ToString());
serviceInQueue.RunStatus = RunStatus.Success;
serviceInQueue.FinishTime = DateTime.Now;
db.SaveChanges();
}
});
}
catch (Exception ex)
{
HttpResponseMessage mainTaskResponse = new HttpResponseMessage();
mainTaskResponse.StatusCode = HttpStatusCode.Forbidden;
mainTaskResponse.Content = new StringContent("UEB package creation was unsuccessful for job id:" + jobGuid + "\n" + ex.Message);
Helpers.TaskDataStore.SetTaskResult(jobGuid, mainTaskResponse);
logger.Error(mainTaskResponse.Content.ToString());
serviceInQueue.RunStatus = RunStatus.Error;
serviceInQueue.FinishTime = DateTime.Now;
serviceInQueue.Error = ex.Message;
db.SaveChanges();
CleanUpOnFailure();
}
return numberOfJobsStarted;
}
public HttpResponseMessage GetModelRunOutput(string uebRunJobID)
{
HttpResponseMessage response = new HttpResponseMessage();
ServiceContext db = new ServiceContext();
string modelRunRootPath = string.Empty;
string modelRunOutputZipFile = string.Empty;
string uebRunStatusFileName = UEB.UEBSettings.UEB_RUN_STATUS_FILE_NAME;
string uebRunStatusFile = string.Empty;
if (string.IsNullOrEmpty(uebRunJobID))
{
string errMsg = string.Format("No UEB run job ID was provided");
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(errMsg);
return response;
}
modelRunRootPath = Path.Combine(UEB.UEBSettings.WORKING_DIR_PATH, uebRunJobID, UEB.UEBSettings.UEB_RUN_FOLDER_NAME);
modelRunOutputZipFile = Path.Combine(modelRunRootPath, "outputszip", UEB.UEBSettings.UEB_RUN_OUTPUT_ZIP_FILE_NAME);
ServiceLog serviceLog = db.ServiceLogs.FirstOrDefault(sl => sl.JobID == uebRunJobID);
if (serviceLog == null)
{
string errMsg = string.Format("No UEB run job was found for the provided job ID: {0}.", uebRunJobID);
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.NotFound;
response.Content = new StringContent(errMsg);
return response;
}
if (serviceLog.RunStatus != RunStatus.Success)
{
string errMsg = string.Empty;
if (serviceLog.RunStatus == RunStatus.InQueue)
{
errMsg = string.Format("UEB run request still in a queue to be processed for job ID: {0}.", uebRunJobID);
}
else if (serviceLog.RunStatus == RunStatus.Processing)
{
errMsg = string.Format("UEB run request is currently being processed for job ID: {0}.", uebRunJobID);
}
else if (serviceLog.RunStatus == RunStatus.Error)
{
errMsg = string.Format("UEB run request failed processing for job ID: {0}.", uebRunJobID);
}
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.NotFound;
response.Content = new StringContent(errMsg);
return response;
}
// check model run root path for the given run job exists
if (Directory.Exists(modelRunRootPath) == false) // if the folder doesnot exist at this point, it means model run output has been already sent to client in the past
{
string errMsg = string.Format("UEB model run output has been deleted for the provided job ID: {0}.", uebRunJobID);
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.NotFound;
response.Content = new StringContent(errMsg);
return response;
}
try
{
FileStream fileStream = File.Open(modelRunOutputZipFile, FileMode.Open, FileAccess.Read);
response.StatusCode = HttpStatusCode.OK;
response.Content = new StreamContent(fileStream);
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
response.Content.Headers.ContentLength = fileStream.Length;
// Ref: http://weblogs.asp.net/cibrax/archive/2011/04/25/implementing-caching-in-your-wcf-web-apis.aspx
// set the browser to cache this response for 120 secs only
response.Content.Headers.Expires = DateTime.Now.AddSeconds(120);
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = UEB.UEBSettings.UEB_RUN_OUTPUT_ZIP_FILE_NAME
};
logger.Info(string.Format("UEB run output zip file for UEB run job ID: {0} was sent to the client.", uebRunJobID));
// delete temporary working folder used for running ueb model
Task deleteTask = new Task(() =>
{
// the zip file may be locked untl the client finish reading the
// data from the file stream object
while (FileManager.IsFileLocked(new FileInfo(modelRunOutputZipFile)))
{
// wait a second
Thread.Sleep(TimeSpan.FromSeconds(1));
}
string modelRunJobIDPath = Path.Combine(UEB.UEBSettings.WORKING_DIR_PATH, uebRunJobID);
if (Directory.Exists(modelRunJobIDPath))
{
Directory.Delete(modelRunJobIDPath, true);
logger.Info(string.Format("UEB run temporary folder: {0} was deleted.", modelRunJobIDPath));
}
});
deleteTask.Start();
}
catch (Exception ex)
{
logger.Fatal(ex.Message);
return Request.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message);
}
return response;
}
/// <summary>
/// Creates UEB model package
/// <remarks>
/// POST is used since the input data needs to come in as a zip file in the body of the request
/// and can't be sent as query parameter
/// </remarks>
/// </summary>
/// <returns></returns>
public HttpResponseMessage PostUEBPackageCreate()
{
string uebPackageBuildRequestJson = string.Empty;
HttpResponseMessage response = new HttpResponseMessage();
Stream uebPackageBuildRequestZipFile = null;
ServiceContext db = new ServiceContext();
ServiceLog serviceLog = null;
var t_stream = this.Request.Content.ReadAsStreamAsync().ContinueWith(s =>
{
uebPackageBuildRequestZipFile = s.Result;
});
t_stream.Wait();
// generate a guid to pass on to the client as a job ID
Guid jobGuid = Guid.NewGuid();
// set the package request processing root dir path
_packageRequestProcessRootDirPath = Path.Combine(UEB.UEBSettings.WORKING_DIR_PATH, jobGuid.ToString());
// set other directory paths necessary for processing the request to build ueb package
_sourceFilePath = _packageRequestProcessRootDirPath;
_targetTempPackageFilePath = Path.Combine(_packageRequestProcessRootDirPath, UEB.UEBSettings.PACKAGE_FILES_OUTPUT_SUB_DIR_PATH);
_targetPackageDirPath = Path.Combine(_packageRequestProcessRootDirPath, UEB.UEBSettings.PACKAGE_OUTPUT_SUB_DIR_PATH);
// set the path for saving the client request zip file
_clientPackageRequestDirPath = Path.Combine(_packageRequestProcessRootDirPath, UEB.UEBSettings.PACKAGE_BUILD_REQUEST_SUB_DIR_PATH);
string packageRequestZipFile = Path.Combine(_clientPackageRequestDirPath, UEB.UEBSettings.PACKAGE_BUILD_REQUEST_ZIP_FILE_NAME);
try
{
var service = db.Services.First(s => s.APIName == "GenerateUEBPackageController.PostUEBPackageCreate");
serviceLog = new ServiceLog
{
JobID = jobGuid.ToString(),
ServiceID = service.ServiceID,
CallTime = DateTime.Now,
RunStatus = RunStatus.InQueue
};
db.ServiceLogs.Add(serviceLog);
db.SaveChanges();
serviceLog = db.ServiceLogs.First(s => s.JobID == serviceLog.JobID);
if (Directory.Exists(_packageRequestProcessRootDirPath))
{
Directory.Delete(_packageRequestProcessRootDirPath);
}
Directory.CreateDirectory(_packageRequestProcessRootDirPath);
if (Directory.Exists(_targetTempPackageFilePath))
{
Directory.Delete(_targetTempPackageFilePath);
}
Directory.CreateDirectory(_targetTempPackageFilePath);
if (Directory.Exists(_clientPackageRequestDirPath))
{
Directory.Delete(_clientPackageRequestDirPath);
}
Directory.CreateDirectory(_clientPackageRequestDirPath);
using (var fileStream = File.Create(packageRequestZipFile))
{
uebPackageBuildRequestZipFile.CopyTo(fileStream);
}
// unzip the request zip file
ZipFile.ExtractToDirectory(packageRequestZipFile, _clientPackageRequestDirPath);
}
catch (Exception ex)
{
logger.Error(ex.Message);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(ex.Message);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.Error = ex.Message;
db.SaveChanges();
return response;
}
// read the file with json extension to a string - uebPackageBuildRequestJson
string[] jsonFiles = Directory.GetFiles(_clientPackageRequestDirPath, "*.json");
if (jsonFiles.Length != 1)
{
string errMsg = "Either no json file was found in the package request or there are multiple json files";
logger.Error(errMsg);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(errMsg);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.Error = errMsg;
db.SaveChanges();
return response;
}
using (var fileReader = new StreamReader(jsonFiles[0]))
{
uebPackageBuildRequestJson = fileReader.ReadToEnd();
}
UEB.UEBPackageRequest pkgRequest;
try
{
pkgRequest = JsonConvert.DeserializeObject<UEB.UEBPackageRequest>(uebPackageBuildRequestJson);
}
catch (Exception ex)
{
logger.Error(ex.Message);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(ex.Message);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.Error = ex.Message;
db.SaveChanges();
return response;
}
// check if the client request is valid
string validationResult = ValidateUEBPackageRequest(pkgRequest);
if (string.IsNullOrEmpty(validationResult) == false)
{
logger.Error(validationResult);
response.StatusCode = HttpStatusCode.BadRequest;
response.Content = new StringContent(validationResult);
serviceLog.RunStatus = RunStatus.Error;
serviceLog.Error = validationResult;
db.SaveChanges();
return response;
}
// if the domain file is a shape file, extract the domain zip file to the root processing dir path
string domainFile = Path.Combine(_clientPackageRequestDirPath, pkgRequest.DomainFileName);
if (Path.GetExtension(domainFile) == ".zip")
{
// unzip the domain shape zip file
ZipFile.ExtractToDirectory(domainFile, _packageRequestProcessRootDirPath);
}
// copy all the files with extensions (.dat, .nc) that came with the client uebpackagerequest zip file
// to the package root processing folder
string[] files = Directory.GetFiles(_clientPackageRequestDirPath);
string fileName = string.Empty;
string destFile = string.Empty;
foreach (string file in files)
{
if (Path.GetExtension(file) == ".nc" || Path.GetExtension(file) == ".dat")
{
// Use static Path methods to extract only the file name from the path.
fileName = Path.GetFileName(file);
destFile = Path.Combine(_packageRequestProcessRootDirPath, fileName);
// Copy the file and overwrite destination file if they already exist.
File.Copy(file, destFile, true);
}
}
PackageCreationStatus pkgStatus = new PackageCreationStatus();
pkgStatus.Message = "UEB package build request is now in a job queue.";
pkgStatus.PackageID = jobGuid.ToString();
string jsonResponse = JsonConvert.SerializeObject(pkgStatus, Formatting.Indented);
response.StatusCode = HttpStatusCode.OK;
response.Content = new StringContent(jsonResponse);
//PK:2/11/2014 (Added): start the package build process - this would start the package build process for this request
// if no other requests are in queue. Othewise it will start the build process for the build request that
// has been in queue for the longest time (first-in and first-out principle)
PackageBuilder uebPkgBuilder = new PackageBuilder();
int numberOfJobsStarted = uebPkgBuilder.Run();
logger.Info("Number of queued jobs started:" + numberOfJobsStarted);
return response;
}
