public void Abort(Resource resource)
{
lock (_taskLock)
{
if (State == TaskState.Started)
{
State = TaskState.Aborted;
Time.AddToOverheads(TaskTimeOverheads.Provider, () =>
{
//var provider = Broker.ProviderByName(resource.ProviderName);
var controller = Discovery.GetControllerFarm(resource);
try
{
//provider.Abort(this.Incarnation.ProvidedTaskId, resource, CurrentSchedule.Nodes);
controller.Abort(this.TaskId); // service's method
controller.Close();
}
catch (Exception e)
{
controller.Abort(); // drop connection
Log.Error(String.Format("Exception while aborting task {0} on {1}: {2}",
TaskId, resource.ResourceName, e
));
}
});
Time.Finished(TaskTimeMetric.Calculation);
//Time.Finished(TaskTimeMetric.Brokering);
}
else
{
Log.Warn("Aborting non-started task " + TaskId.ToString());
State = TaskState.Aborted;
}
//todo: _lastEvent = Aborted; !!!!!!!!!!!!!!!!!!111
_failReason = "Aborted";
_lastEvent = Eventing.EventType.TaskFailed;
//Over();
Log.Info(String.Format("Task {0} aborted", TaskId));
}
}
public void Update(IEnumerable <Resource> resources)
{
lock (_taskLock)
{
string failReason = null;
ServiceProxies.ControllerFarmService.TaskStateInfo stateInfo = null;
var resource = resources.FirstOrDefault(r => r.ResourceName == CurrentSchedule.ResourceName);
if (resource == null) // resource was available, but now isn't
{
failReason = "Resource became unreachable (check if ControllerFarm is still running)"; // todo : retries on Task.Update
}
else
{
//var provider = Broker.ProviderByName(resource.ProviderName);
var controller = Discovery.GetControllerFarm(resource);
try
{
/*
* Tuple<TaskState, string> stateTuple = null;
* //Time.AddExpenses(TaskTimeExpenses.Provider, () => // todo: time measures on other provider's actions
* //{
* stateTuple = provider.GetTaskState(TaskId, Incarnation.ProvidedTaskId, resource, CurrentSchedule.Nodes);
* //});
*
* var state = stateTuple.Item1;
* string reason = stateTuple.Item2;
*/
stateInfo = controller.GetTaskStateInfo(TaskId);
controller.Close();
}
catch (Exception e)
{
controller.Abort();
Log.Warn(String.Format("Exception on updating task's {0} state: {1}", this.TaskId, e));
failReason = "Controller failed while updating task: " + e.Message;
}
}
if (stateInfo != null)
{
var state = stateInfo.State;
string reason = stateInfo.StateComment;
if (state != this.State)
{
Log.Debug(String.Format("Task {0}: new state = {1}{2}", TaskId, state, String.IsNullOrEmpty(reason) ? "" : ", reason = " + reason));
}
// todo : log new task state
if (state == TaskState.Completed)
{
try
{
Time.Finished(TaskTimeMetric.Calculation);
string incarnatedFtpFolder = GetFtpFolder(CurrentSchedule.Nodes.First(), resource, CopyPhase.Out);
Complete(incarnatedFtpFolder);
}
catch (Exception e)
{
Log.Error(String.Format(
"Couldn't complete task {0}: {1}",
TaskId, e.ToString()
));
//Fail(errMsg);
failReason = e.Message;
}
}
else
if (state == TaskState.Failed)
{
failReason = reason;
}
}
if (!String.IsNullOrEmpty(failReason))
{
Time.Finished(TaskTimeMetric.Calculation);
Fail(failReason);
}
}
}
public void Run(TaskSchedule schedule, IEnumerable <Resource> resources)
{
lock (_taskLock)
{
try
{
var execStarted = DateTime.Now;
CurrentSchedule = schedule;
Params[CONST.Params.Method] = Method;
var resource = resources.First(r => r.ResourceName == schedule.ResourceName);
string incarnatedFtpFolder = GetFtpFolder(schedule.Nodes.First(), resource, CopyPhase.In);
if (PackageBaseProxy.GetSupportedPackageNames()
.Any(name => String.Equals(name, Package, StringComparison.InvariantCultureIgnoreCase))
)
{
//ProcessInputs();
Time.AddToOverheads(TaskTimeOverheads.InputFilesCopy, () =>
{
Log.Debug("Uploading incarnated inputs");
foreach (var file in Incarnation.FilesToCopy)
{
Log.Debug(file.FileName + ": started");
IOProxy.Ftp.MakePath(incarnatedFtpFolder + Path.GetDirectoryName(file.FileName).Replace("\\", "/"));
Log.Debug(file.FileName + ": path been made");
IOProxy.Storage.Download(file.StorageId, incarnatedFtpFolder + file.FileName);
Log.Debug(file.FileName + ": downloaded");
}
Log.Debug("Uploading incarnated inputs done");
});
}
else
{
//ApplyAdapters(Broker.Adapters.Where(a => a.Type == AdapterType.Machine), incarnatedFtpFolder);
//ApplyAdapters(Broker.Adapters.Where(a => a.Type == AdapterType.Package), incarnatedFtpFolder);
//ApplyAdapters(Broker.Adapters.Where(a => a.Type == AdapterType.Mixed), incarnatedFtpFolder);
}
Incarnation.PackageName = Package;
Incarnation.UserCert = UserCert;
if (String.IsNullOrWhiteSpace(Incarnation.CommandLine))
{
throw new Exception("Impossible to run task with empty command line");
}
if (!Incarnation.CommandLine.Contains("{0}") &&
Incarnation.CommandLine.StartsWith(Package, StringComparison.InvariantCultureIgnoreCase))
{
Incarnation.CommandLine = "{0}" + Incarnation.CommandLine.Substring(Package.Length);
}
Log.Stats("T_adapters", this.WfId, this.TaskId, DateTime.Now - execStarted);
Time.AddToOverheads(TaskTimeOverheads.Provider, () =>
{
//var provider = Broker.ProviderByName(resource.ProviderName);
var controller = Discovery.GetControllerFarm(resource);
try
{
//Incarnation.ProvidedTaskId = provider.Run(this.TaskId, this.Incarnation, resource, schedule.Nodes);
var runContext = new ServiceProxies.ControllerFarmService.TaskRunContext()
{
TaskId = this.TaskId,
//Incarnation = this.Incarnation,
UserCert = this.UserCert,
PackageName = this.Incarnation.PackageName,
CommandLine = this.Incarnation.CommandLine,
InputFiles = this.Incarnation.FilesToCopy.Select(f => new ServiceProxies.ControllerFarmService.FileContext()
{
FileName = f.FileName,
StorageId = f.StorageId,
}).ToArray(),
ExpectedOutputFileNames = this.Incarnation.ExpectedOutputFileNames.ToArray(),
NodesConfig = schedule.Nodes.Select(n => new ServiceProxies.ControllerFarmService.NodeRunConfig()
{
ResourceName = n.ResourceName,
NodeName = n.NodeName,
Cores = n.Cores
}).ToArray()
};
Log.Debug("Running task on controller: " + TaskId.ToString());
controller.Run(runContext);
Log.Debug("Run done: " + TaskId.ToString());
controller.Close();
}
catch (Exception e)
{
controller.Abort();
Log.Error("Exception on Task.Run for task " + this.TaskId + ": " + e.ToString());
throw;
}
});
State = TaskState.Started;
Time.Started(TaskTimeMetric.Calculation);
Log.Stats("T_clust_start", this.WfId, this.TaskId, DateTime.Now);
_lastEvent = Eventing.EventType.TaskStarted;
}
catch (Exception e)
{
Log.Error(String.Format("Error on executing task {0}: {1}\n{2}",
TaskId, e.Message, e.StackTrace
));
Fail(reason: e.Message);
}
}
}