public BlockRequestBase(TaskId taskId, string taskExecutionId, BlockType blockType, string blockExecutionId)
{
TaskId = taskId;
TaskExecutionId = taskExecutionId;
BlockExecutionId = blockExecutionId;
BlockType = blockType;
}
public ListBlockCreateRequest(TaskId taskId,
string taskExecutionId,
List<string> serializedValues)
: base(taskId, taskExecutionId, BlockType.List)
{
SerializedValues = serializedValues;
}
public DequeueForcedBlocksRequest(TaskId taskId,
string taskExecutionId,
BlockType blockType,
List<string> forcedBlockQueueIds)
: base(taskId, taskExecutionId, blockType)
{
ForcedBlockQueueIds = forcedBlockQueueIds;
}
public RangeBlockCreateRequest(TaskId taskId,
string taskExecutionId,
long from,
long to)
: base(taskId, taskExecutionId, Taskling.Blocks.Common.BlockType.NumericRange)
{
From = from;
To = to;
}
public RangeBlockCreateRequest(TaskId taskId,
string taskExecutionId,
DateTime fromDate,
DateTime toDate)
: base(taskId, taskExecutionId, Taskling.Blocks.Common.BlockType.DateRange)
{
From = fromDate.Ticks;
To = toDate.Ticks;
}
public BlockExecutionChangeStatusRequest(TaskId taskId,
string taskExecutionId,
BlockType blockType,
string blockExecutionId,
BlockExecutionStatus blockExecutionStatus)
: base(taskId, taskExecutionId, blockType, blockExecutionId)
{
BlockExecutionStatus = blockExecutionStatus;
}
public StartCriticalSectionRequest(TaskId taskId,
string taskExecutionId,
TaskDeathMode taskDeathMode,
CriticalSectionType criticalSectionType)
: base(taskId, taskExecutionId)
{
TaskDeathMode = taskDeathMode;
Type = criticalSectionType;
}
public BlockExecutionCreateRequest(TaskId taskId,
string taskExecutionId,
BlockType blockType,
string blockId,
int attempt)
: base(taskId, taskExecutionId, blockType)
{
BlockId = blockId;
Attempt = attempt;
}
public FindBlocksOfTaskRequest(TaskId taskId,
string taskExecutionId,
BlockType blockType,
string referenceValueOfTask,
ReprocessOption reprocessOption)
: base(taskId, taskExecutionId, blockType)
{
ReferenceValueOfTask = referenceValueOfTask;
ReprocessOption = reprocessOption;
}
public ListBlockCreateRequest(TaskId taskId,
string taskExecutionId,
List<string> serializedValues,
string serializedHeader,
int compressionThreshold)
: base(taskId, taskExecutionId, BlockType.List)
{
SerializedValues = serializedValues;
SerializedHeader = serializedHeader;
CompressionThreshold = compressionThreshold;
}
/// <summary>
/// Get task data from xml node "task"
/// </summary>
private static Task GetTaskInfo(XElement xTask)
{
TaskId id = new TaskId(xTask.Attribute("id").Value);
TaskStatus status = StatusFromString(xTask.Attribute("status").Value);
Task task = new Task();
task.Id = id;
task.Status = status;
XAttribute xRegistrationTime = xTask.Attribute("registrationTime");
if (xRegistrationTime != null)
{
DateTime time;
if (DateTime.TryParse(xRegistrationTime.Value, out time))
task.RegistrationTime = time;
}
XAttribute xStatusChangeTime = xTask.Attribute("statusChangeTime");
if (xStatusChangeTime != null)
{
DateTime time;
if (DateTime.TryParse(xStatusChangeTime.Value, out time))
task.StatusChangeTime = time;
}
XAttribute xPagesCount = xTask.Attribute("filesCount");
if (xPagesCount != null)
{
int pagesCount;
if (Int32.TryParse(xPagesCount.Value, out pagesCount))
task.PagesCount = pagesCount;
}
XAttribute xCredits = xTask.Attribute("credits");
if (xCredits != null)
{
int credits;
if( Int32.TryParse( xCredits.Value, out credits ))
task.Credits = credits;
}
XAttribute xDescription = xTask.Attribute("description");
if (xDescription != null)
task.Description = xDescription.Value;
XAttribute xResultUrl = xTask.Attribute("resultUrl");
if (xResultUrl != null)
{
task.DownloadUrl = xResultUrl.Value;
}
return task;
}
protected void AssertContainsException(TaskId task, string expectedType,
string expectedExceptionMessage, string expectedStackTrace)
{
var expectedString = string.Format("{0}\n{1}\n{2}", expectedType, expectedExceptionMessage,
expectedStackTrace);
var messages = from e in messageElements
where e.Name == TaskAction.Exception && task.MatchesTaskElement(e)
select string.Format("{0}\n{1}\n{2}", e.Attribute("type").Value,
GetElementValue(e, "message").Replace("\n", Environment.NewLine),
GetElementValue(e, "stack-trace").Trim());
var output = messages.Join("\n---\n");
StringAssert.Contains(expectedString, output);
}
public TaskExecutionStartRequest(TaskId taskId,
TaskDeathMode taskDeathMode,
int concurrencyLimit,
short failedTaskRetryLimit,
short deadTaskRetryLimit
)
: base(taskId)
{
TaskDeathMode = taskDeathMode;
ConcurrencyLimit = concurrencyLimit;
FailedTaskRetryLimit = failedTaskRetryLimit;
DeadTaskRetryLimit = deadTaskRetryLimit;
}
protected void AssertContainsErrorFinal(TaskId task, string expectedMessage)
{
var messages = from e in messageElements
where e.Name == TaskAction.Finish && task.MatchesTaskElement(e)
select new
{
Result = e.Attribute("result").Value,
Message = GetElementValue(e, "message")
};
var result = messages.Last();
Assert.AreEqual(TaskResult.Error, result.Result);
if (!string.IsNullOrEmpty(expectedMessage))
Assert.AreEqual(expectedMessage, result.Message);
}
public FindFailedBlocksRequest(TaskId taskId,
string taskExecutionId,
BlockType blockType,
DateTime searchPeriodBegin,
DateTime searchPeriodEnd,
int blockCountLimit,
int retryLimit)
: base(taskId, taskExecutionId, blockType)
{
SearchPeriodBegin = searchPeriodBegin;
SearchPeriodEnd = searchPeriodEnd;
BlockCountLimit = blockCountLimit;
RetryLimit = retryLimit;
}
public void Enqueue(TaskId id, Action task) {
var entry = new TaskEntry {
Id = id,
EnqueuedDateTimeUtc = _dateTimeProvider.UtcNow,
Action = task,
StopWatch = new Stopwatch(),
};
Logger.LogInfo("Queue \"{0}\": Enqueing task \"{1}\"", _description, entry.Id.Description);
bool isFirstTask;
lock (_lock) {
if (_runningTask == null) {
_runningTask = entry;
isFirstTask = true;
} else {
_tasks.Enqueue(entry);
isFirstTask = false;
}
}
if (isFirstTask)
RunTaskAsync(entry);
}
public static void TaskId(this ScenarioContext scenarioContext, TaskId taskId)
{
scenarioContext.Set(taskId, "taskId");
}
protected void AssertSingle(TaskId task, string messageType)
{
var messages = (from m in messageElements
where m.Name == messageType && task.MatchesTaskElement(m)
select m.Name.ToString()).ToList();
Assert.AreEqual(1, messages.Count, "Expected one item of {0} for task {1}", messageType, task);
}
protected void AssertContainsCreate(TaskId taskId)
{
AssertSingle(taskId, TaskAction.Create);
}
/// <exception cref="System.Exception"/>
public virtual void TestKillJob()
{
JobConf conf = new JobConf();
AppContext context = Org.Mockito.Mockito.Mock <AppContext>();
// a simple event handler solely to detect the container cleaned event
CountDownLatch isDone = new CountDownLatch(1);
EventHandler handler = new _EventHandler_106(isDone);
Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler);
// create and start the launcher
LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock
<TaskUmbilicalProtocol>());
launcher.Init(conf);
launcher.Start();
// create mocked job, task, and task attempt
// a single-mapper job
JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobs[jobId] = job;
// app context returns the one and only job
Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs);
Task ytask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map);
Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask);
// create a sleeping mapper that runs beyond the test timeout
MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>();
Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true);
Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true);
TaskAttemptID taskID = TypeConverter.FromYarn(taId);
Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID);
Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID()
));
Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf
>(), Matchers.IsA <TaskUmbilicalProtocol>());
// sleep for a long time
// pump in a task attempt launch event
ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer
(), mapTask);
launcher.Handle(launchEvent);
Sharpen.Thread.Sleep(200);
// now pump in a container clean-up event
ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null
, null, ContainerLauncher.EventType.ContainerRemoteCleanup);
launcher.Handle(cleanupEvent);
// wait for the event to fire: this should be received promptly
isDone.Await();
launcher.Close();
}
protected void AssertDoesNotContain(TaskId task, string action)
{
var messages = from e in messageElements
where e.Name == action && task.MatchesTaskElement(e)
select e;
CollectionAssert.IsEmpty(messages);
}
public override void UpdateAttempt(TaskAttemptStatusUpdateEvent.TaskAttemptStatus
status, long timestamp)
{
base.UpdateAttempt(status, timestamp);
TaskAttemptId attemptID = status.id;
TaskId taskID = attemptID.GetTaskId();
JobId jobID = taskID.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobID);
if (job == null)
{
return;
}
Task task = job.GetTask(taskID);
if (task == null)
{
return;
}
TaskAttempt taskAttempt = task.GetAttempt(attemptID);
if (taskAttempt == null)
{
return;
}
long boxedStart = startTimes[attemptID];
long start = boxedStart == null ? long.MinValue : boxedStart;
// We need to do two things.
// 1: If this is a completion, we accumulate statistics in the superclass
// 2: If this is not a completion, we learn more about it.
// This is not a completion, but we're cooking.
//
if (taskAttempt.GetState() == TaskAttemptState.Running)
{
// See if this task is already in the registry
AtomicLong estimateContainer = attemptRuntimeEstimates[taskAttempt];
AtomicLong estimateVarianceContainer = attemptRuntimeEstimateVariances[taskAttempt
];
if (estimateContainer == null)
{
if (attemptRuntimeEstimates[taskAttempt] == null)
{
attemptRuntimeEstimates[taskAttempt] = new AtomicLong();
estimateContainer = attemptRuntimeEstimates[taskAttempt];
}
}
if (estimateVarianceContainer == null)
{
attemptRuntimeEstimateVariances.PutIfAbsent(taskAttempt, new AtomicLong());
estimateVarianceContainer = attemptRuntimeEstimateVariances[taskAttempt];
}
long estimate = -1;
long varianceEstimate = -1;
// This code assumes that we'll never consider starting a third
// speculative task attempt if two are already running for this task
if (start > 0 && timestamp > start)
{
estimate = (long)((timestamp - start) / Math.Max(0.0001, status.progress));
varianceEstimate = (long)(estimate * status.progress / 10);
}
if (estimateContainer != null)
{
estimateContainer.Set(estimate);
}
if (estimateVarianceContainer != null)
{
estimateVarianceContainer.Set(varianceEstimate);
}
}
}
public TaskId GetNextIdentity()
{
return(TaskId.New());
}
public RecordCollector(string logPrefix, IStreamConfig configuration, TaskId id)
{
this.logPrefix = $"{logPrefix}";
this.configuration = configuration;
this.id = id;
}
public void StreamTaskSuspendResume()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper()))
.To("topic2");
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var topology = builder.Build();
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
producer);
task.GroupMetadata = consumer as SyncConsumer;
task.InitializeStateStores();
task.InitializeTopology();
List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >();
int offset = 0;
for (int i = 0; i < 5; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i+1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(part, offset++)
});
}
task.AddRecords(messages);
Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds()));
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
Assert.IsTrue(task.CommitNeeded);
task.Commit();
}
task.Suspend();
task.Resume();
task.AddRecords(messages);
Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds()));
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
Assert.IsTrue(task.CommitNeeded);
task.Commit();
}
// CHECK IN TOPIC topic2
consumer.Subscribe("topic2");
List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >();
ConsumeResult <byte[], byte[]> result = null;
do
{
result = consumer.Consume(100);
if (result != null)
{
results.Add(result);
consumer.Commit(result);
}
} while (result != null);
Assert.AreEqual(10, results.Count);
task.Close();
}
public virtual Task GetTask(TaskId taskID)
{
return(mockJob.GetTask(taskID));
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolSize()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
containerLauncher.Init(new Configuration());
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// No events yet
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, MRJobConfig.DefaultMrAmContainerlauncherThreadpoolInitialSize
);
NUnit.Framework.Assert.AreEqual(0, threadPool.GetPoolSize());
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, threadPool.GetCorePoolSize
());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Same set of hosts, so no change
containerLauncher.finishEventHandling = true;
int timeOut = 0;
while (containerLauncher.numEventsProcessed.Get() < 10 && timeOut++ < 200)
{
Log.Info("Waiting for number of events processed to become " + 10 + ". It is now "
+ containerLauncher.numEventsProcessed.Get() + ". Timeout is " + timeOut);
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual(10, containerLauncher.numEventsProcessed.Get());
containerLauncher.finishEventHandling = false;
for (int i_1 = 0; i_1 < 10; i_1++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i_1 + 10);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i_1 + 10);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 20);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Different hosts, there should be an increase in core-thread-pool size to
// 21(11hosts+10buffer)
// Core pool size should be 21 but the live pool size should be only 11.
containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
containerLauncher.finishEventHandling = false;
ContainerId containerId_1 = ContainerId.NewContainerId(appAttemptId, 21);
TaskAttemptId taskAttemptId_1 = MRBuilderUtils.NewTaskAttemptId(taskId, 21);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId_1, containerId_1
, "host11:1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
WaitForEvents(containerLauncher, 21);
NUnit.Framework.Assert.AreEqual(11, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
// change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
// and verify initialPoolSize value.
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadpoolInitialSize, 20);
containerLauncher = new TestContainerLauncher.CustomContainerLauncher(this, context
);
containerLauncher.Init(conf);
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, 20);
}
public DeleteCommand(string args) : base(args)
{
_taskId = new TaskId(args);
}
public override int GetHashCode()
{
return(TaskId.GetHashCode());
}
protected bool Equals(TaskReopened other)
{
return(TaskId.Equals(other.TaskId));
}
private void Over()
{
WriteModelCoefs();
Log.Debug("Task " + TaskId.ToString() + " is over");
var duration = this.Time.Duration.ToArray();
var overTotals = this.Time.OverheadTotals.ToArray().ToDictionary(p => p.Key, p => p.Value);
var overAvgs = this.Time.OverheadAverages.ToArray().ToDictionary(p => p.Key, p => p.Value);
Time.Finished(TaskTimeMetric.Brokering);
Log.Debug("Finished brokering");
string timeStat = "";
timeStat += "Duration:" + Environment.NewLine;
timeStat += String.Join(Environment.NewLine, duration.Select(pair => " " + pair.Key.ToString() + " = " + pair.Value.ToString())) + Environment.NewLine;
timeStat += "Overheads:" + Environment.NewLine;
timeStat += String.Join(Environment.NewLine, overTotals.Select(pair => " " + pair.Key.ToString() + " = " + pair.Value.ToString()));
Log.Over(
"{0} -- {1} {2} #{3} {4}{7}{5}{7}{6}",
DateTime.Now, Package, State.ToString(), TaskId,
(CurrentSchedule == null)?
"":
CurrentSchedule.ResourceName +
" [" + String.Join(", ", CurrentSchedule.Nodes.Select(n => n.Cores.ToString())) + "]",
timeStat,
ToJsonString(),
Environment.NewLine
);
try
{
var sb = new StringBuilder();
var head = new StringBuilder();
sb.AppendFormat("{0};", WfId);
head.Append("WfId;");
sb.AppendFormat("{0};", TaskId);
head.Append("TaskId;");
sb.AppendFormat("{0};", Package);
head.Append("Package;");
sb.AppendFormat("{0};", CurrentSchedule.ResourceName);
head.Append("ResourceName;");
sb.AppendFormat("{0};", State);
head.Append("State;");
sb.AppendFormat("{0};", duration.First(p => p.Key == TaskTimeMetric.Calculation).Value);
head.Append("T_Calculation;");
var avgSb = new StringBuilder(sb.ToString());
var overNames = new[] {
TaskTimeOverheads.WaitInQueue,
TaskTimeOverheads.PackageBase,
TaskTimeOverheads.Estimation,
TaskTimeOverheads.Scheduler,
TaskTimeOverheads.Provider,
TaskTimeOverheads.InputFilesCopy,
TaskTimeOverheads.OutputFilesCopy,
TaskTimeOverheads.Other,
TaskTimeOverheads.All,
};
foreach (var overName in overNames)
{
if (!overTotals.ContainsKey(overName))
{
overTotals[overName] = TimeSpan.Zero;
}
if (!overAvgs.ContainsKey(overName))
{
overAvgs[overName] = TimeSpan.Zero;
}
}
foreach (var overName in overNames)
{
sb.AppendFormat("{0};", overTotals[overName]);
avgSb.AppendFormat("{0};", overAvgs[overName]);
head.AppendFormat("T_{0};", overName);
}
//sb.AppendFormat("{0};", AssignedResource.ResourceName);
//sb.AppendFormat("{0};", Time.Duration[TaskTimeMetric.Calculation]);
//sb.AppendFormat("{0};", Time.Duration[TaskTimeMetric.Postponed]);
//sb.AppendFormat("{0};", Time.Duration[TaskTimeMetric.Queued]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.All]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.Other]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.PackageBase]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.Scheduler]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.Provider]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.InputFilesCopy]);
//sb.AppendFormat("{0};", Time.Overheads[TaskTimeOverheads.OutputFilesCopy]);
//sb.AppendFormat("{0};", 0);
Time.OverheadsSpecial["pb"] = Time.OverheadTotals[TaskTimeOverheads.PackageBase] + Time.OverheadTotals[TaskTimeOverheads.OutputFilesCopy];
var specialLine = new List <string>()
{
TaskId.ToString()
};
var specialHeader = new List <string>()
{
"TaskId"
};
foreach (string key in Time.OverheadsSpecial.Keys)
{
specialHeader.Add(key);
specialLine.Add(Time.OverheadsSpecial[key].TotalSeconds.ToString());
}
lock (_csvFilesLock)
{
File.AppendAllText(CONST.Path.OverCsvFile, sb.ToString() + Environment.NewLine);
File.AppendAllText(CONST.Path.OverAvgCsvFile, avgSb.ToString() + Environment.NewLine);
File.WriteAllText(CONST.Path.OverHeadersFile, head.ToString() + Environment.NewLine);
File.AppendAllText(CONST.Path.OverSpecFile, String.Join(";", specialLine) + Environment.NewLine);
File.WriteAllText(CONST.Path.OverSpecHeadersFile, String.Join(";", specialHeader) + Environment.NewLine);
}
}
catch (Exception e)
{
Log.Warn(String.Format("Problem with tasks over: {0}\n{1}", e.Message, e.StackTrace));
}
}
public CompleteCriticalSectionRequest(TaskId taskId, string taskExecutionId, CriticalSectionType criticalSectionType)
: base(taskId, taskExecutionId)
{
Type = criticalSectionType;
}
public Interval(TaskId child, float delay)
: base(child)
{
this.Delay = delay;
}
private void PopulateMembers(AppContext ctx)
{
JobId jobID = null;
TaskId taskID = null;
string tid = $(AMParams.TaskId);
if (!tid.IsEmpty())
{
taskID = MRApps.ToTaskID(tid);
jobID = taskID.GetJobId();
}
else
{
string jid = $(AMParams.JobId);
if (!jid.IsEmpty())
{
jobID = MRApps.ToJobID(jid);
}
}
if (jobID == null)
{
return;
}
job = ctx.GetJob(jobID);
if (job == null)
{
return;
}
if (taskID != null)
{
task = job.GetTask(taskID);
if (task == null)
{
return;
}
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> entry in task.GetAttempts())
{
long value = 0;
Counters counters = entry.Value.GetCounters();
CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup
)) : null;
if (group != null)
{
Counter c = group.FindCounter($(AMParams.CounterName));
if (c != null)
{
value = c.GetValue();
}
}
values[MRApps.ToString(entry.Key)] = value;
}
return;
}
// Get all types of counters
IDictionary <TaskId, Task> tasks = job.GetTasks();
foreach (KeyValuePair <TaskId, Task> entry_1 in tasks)
{
long value = 0;
Counters counters = entry_1.Value.GetCounters();
CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup
)) : null;
if (group != null)
{
Counter c = group.FindCounter($(AMParams.CounterName));
if (c != null)
{
value = c.GetValue();
}
}
values[MRApps.ToString(entry_1.Key)] = value;
}
}
internal void SetTaskId(TaskId id)
{
logPrefix = $"stream-task[{id.Id}|{id.Partition}]|processor[{Name}]- ";
}
protected void AssertContainsOutput(TaskId task, params string[] expectedOutput)
{
var messages = from e in messageElements
where e.Name == TaskAction.Output && task.MatchesTaskElement(e)
select GetElementValue(e, "message").TrimEnd();
CollectionAssert.AreEqual(expectedOutput, messages);
}
public UntilFail(TaskId child)
: base(child)
{
}
public void AllowCreation()
{
var taskId = new TaskId("id");
taskId.Value.Should().Be("id");
}
// -------------------------------------------------------------------
// Public
// -------------------------------------------------------------------
public override void ChildSuccess(TaskId task)
{
this.Condition = true;
}
public override void ChildFail(TaskId task)
{
this.Success();
this.Condition = false;
}
protected void AssertContainsOutput(TaskId task, string expectedOutput)
{
var messages = from e in messageElements
where e.Name == TaskAction.Output && task.MatchesTaskElement(e)
select GetElementValue(e, "message").TrimEnd();
var output = messages.Single();
Assert.AreEqual(expectedOutput, output);
}
private void GetCounters(AppContext ctx)
{
JobId jobID = null;
TaskId taskID = null;
string tid = $(AMParams.TaskId);
if (!tid.IsEmpty())
{
taskID = MRApps.ToTaskID(tid);
jobID = taskID.GetJobId();
}
else
{
string jid = $(AMParams.JobId);
if (jid != null && !jid.IsEmpty())
{
jobID = MRApps.ToJobID(jid);
}
}
if (jobID == null)
{
return;
}
job = ctx.GetJob(jobID);
if (job == null)
{
return;
}
if (taskID != null)
{
task = job.GetTask(taskID);
if (task == null)
{
return;
}
total = task.GetCounters();
return;
}
// Get all types of counters
IDictionary <TaskId, Task> tasks = job.GetTasks();
total = job.GetAllCounters();
bool needTotalCounters = false;
if (total == null)
{
total = new Counters();
needTotalCounters = true;
}
map = new Counters();
reduce = new Counters();
foreach (Task t in tasks.Values)
{
Counters counters = t.GetCounters();
if (counters == null)
{
continue;
}
switch (t.GetType())
{
case TaskType.Map:
{
map.IncrAllCounters(counters);
break;
}
case TaskType.Reduce:
{
reduce.IncrAllCounters(counters);
break;
}
}
if (needTotalCounters)
{
total.IncrAllCounters(counters);
}
}
}
protected void AssertContainsStart(TaskId task)
{
AssertSingle(task, TaskAction.Start);
}
protected void AssertContainsFailingChildren(TaskId task)
{
AssertContainsFinish(task, TaskResult.Exception, "One or more child tests failed");
}
protected void AssertMessageOrder(TaskId task, params string[] messageTypes)
{
var messages = from m in messageElements
where task.MatchesTaskElement(m)
select m.Name.ToString();
CollectionAssert.IsSubsetOf(messageTypes, messages, "With task {0}", task);
}
[U] public void TaskId()
{
TaskId fromString = "node-1:12";
DebugFor(fromString).Should().Be("node-1:12");
}
protected int GetMessageIndex(TaskId task, string messageType)
{
var indices = (messageElements.Select((e, i) => new {Element = e, Index = i})
.Where(x => x.Element.Name == messageType && task.MatchesTaskElement(x.Element))
.Select(x => x.Index)).ToList();
Assert.AreEqual(1, indices.Count, "Expected single message of type {0} for task {1}", messageType, task);
return indices.Single();
}
/// <summary>
/// Get task data from xml node "task"
/// </summary>
private static OcrTask GetTaskInfo(XElement xTask)
{
TaskId id = new TaskId(xTask.Attribute("id").Value);
TaskStatus status = StatusFromString(xTask.Attribute("status").Value);
var task = new OcrTask
{
TaskId = id,
Status = status
};
XAttribute xRegistrationTime = xTask.Attribute("registrationTime");
if (xRegistrationTime != null)
{
DateTime time;
if (DateTime.TryParse(xRegistrationTime.Value, out time))
{
task.RegistrationTime = time;
}
}
XAttribute xStatusChangeTime = xTask.Attribute("statusChangeTime");
if (xStatusChangeTime != null)
{
DateTime time;
if (DateTime.TryParse(xStatusChangeTime.Value, out time))
{
task.StatusChangeTime = time;
}
}
XAttribute xFilesCount = xTask.Attribute("filesCount");
if (xFilesCount != null)
{
int filesCount;
if (int.TryParse(xFilesCount.Value, out filesCount))
{
task.FilesCount = filesCount;
}
}
XAttribute xCredits = xTask.Attribute("credits");
if (xCredits != null)
{
int credits;
if (int.TryParse(xCredits.Value, out credits))
{
task.Credits = credits;
}
}
XAttribute xDescription = xTask.Attribute("description");
if (xDescription != null)
{
task.Description = xDescription.Value;
}
XAttribute xResultUrl = xTask.Attribute("resultUrl");
if (xResultUrl != null)
{
task.DownloadUrls = new List <string> {
xResultUrl.Value
};
for (int i = 2; i < 10; i++)
{
XAttribute xResultUrlI = xTask.Attribute("resultUrl" + i);
if (xResultUrlI != null)
{
task.DownloadUrls.Add(xResultUrlI.Value);
}
else
{
break;
}
}
}
XAttribute xError = xTask.Attribute("error");
if (xError != null)
{
task.Error = xError.Value;
}
return(task);
}
protected void AssertContainsError(TaskId task, string message)
{
AssertContainsFinish(task, TaskResult.Error, message);
}
public Task(TaskId id, TaskStatus status)
{
Id = id;
Status = status;
}
public static string GetTaskProducerClientId(string threadClientId, TaskId taskId)
{
return(threadClientId + "-" + taskId + "-producer");
}
public void Run(TaskSchedule schedule, IEnumerable <Resource> resources)
{
lock (_taskLock)
{
try
{
var execStarted = DateTime.Now;
CurrentSchedule = schedule;
Params[CONST.Params.Method] = Method;
var resource = resources.First(r => r.ResourceName == schedule.ResourceName);
string incarnatedFtpFolder = GetFtpFolder(schedule.Nodes.First(), resource, CopyPhase.In);
if (PackageBaseProxy.GetSupportedPackageNames()
.Any(name => String.Equals(name, Package, StringComparison.InvariantCultureIgnoreCase))
)
{
//ProcessInputs();
Time.AddToOverheads(TaskTimeOverheads.InputFilesCopy, () =>
{
Log.Debug("Uploading incarnated inputs");
foreach (var file in Incarnation.FilesToCopy)
{
Log.Debug(file.FileName + ": started");
IOProxy.Ftp.MakePath(incarnatedFtpFolder + Path.GetDirectoryName(file.FileName).Replace("\\", "/"));
Log.Debug(file.FileName + ": path been made");
IOProxy.Storage.Download(file.StorageId, incarnatedFtpFolder + file.FileName);
Log.Debug(file.FileName + ": downloaded");
}
Log.Debug("Uploading incarnated inputs done");
});
}
else
{
//ApplyAdapters(Broker.Adapters.Where(a => a.Type == AdapterType.Machine), incarnatedFtpFolder);
//ApplyAdapters(Broker.Adapters.Where(a => a.Type == AdapterType.Package), incarnatedFtpFolder);
//ApplyAdapters(Broker.Adapters.Where(a => a.Type == AdapterType.Mixed), incarnatedFtpFolder);
}
Incarnation.PackageName = Package;
Incarnation.UserCert = UserCert;
if (String.IsNullOrWhiteSpace(Incarnation.CommandLine))
{
throw new Exception("Impossible to run task with empty command line");
}
if (!Incarnation.CommandLine.Contains("{0}") &&
Incarnation.CommandLine.StartsWith(Package, StringComparison.InvariantCultureIgnoreCase))
{
Incarnation.CommandLine = "{0}" + Incarnation.CommandLine.Substring(Package.Length);
}
Log.Stats("T_adapters", this.WfId, this.TaskId, DateTime.Now - execStarted);
Time.AddToOverheads(TaskTimeOverheads.Provider, () =>
{
//var provider = Broker.ProviderByName(resource.ProviderName);
var controller = Discovery.GetControllerFarm(resource);
try
{
//Incarnation.ProvidedTaskId = provider.Run(this.TaskId, this.Incarnation, resource, schedule.Nodes);
var runContext = new ServiceProxies.ControllerFarmService.TaskRunContext()
{
TaskId = this.TaskId,
//Incarnation = this.Incarnation,
UserCert = this.UserCert,
PackageName = this.Incarnation.PackageName,
CommandLine = this.Incarnation.CommandLine,
InputFiles = this.Incarnation.FilesToCopy.Select(f => new ServiceProxies.ControllerFarmService.FileContext()
{
FileName = f.FileName,
StorageId = f.StorageId,
}).ToArray(),
ExpectedOutputFileNames = this.Incarnation.ExpectedOutputFileNames.ToArray(),
NodesConfig = schedule.Nodes.Select(n => new ServiceProxies.ControllerFarmService.NodeRunConfig()
{
ResourceName = n.ResourceName,
NodeName = n.NodeName,
Cores = n.Cores
}).ToArray()
};
Log.Debug("Running task on controller: " + TaskId.ToString());
controller.Run(runContext);
Log.Debug("Run done: " + TaskId.ToString());
controller.Close();
}
catch (Exception e)
{
controller.Abort();
Log.Error("Exception on Task.Run for task " + this.TaskId + ": " + e.ToString());
throw;
}
});
State = TaskState.Started;
Time.Started(TaskTimeMetric.Calculation);
Log.Stats("T_clust_start", this.WfId, this.TaskId, DateTime.Now);
_lastEvent = Eventing.EventType.TaskStarted;
}
catch (Exception e)
{
Log.Error(String.Format("Error on executing task {0}: {1}\n{2}",
TaskId, e.Message, e.StackTrace
));
Fail(reason: e.Message);
}
}
}
protected void AssertContainsFinish(TaskId task, string expectedTaskResult, string expectedMessage = null)
{
var messages = (from e in messageElements
where e.Name == TaskAction.Finish && task.MatchesTaskElement(e)
select new
{
Result = e.Attribute("result").Value,
Message = GetElementValue(e, "message")
}).ToList();
Assert.AreEqual(1, messages.Count, "Expected single message of type {0} for task {1}", TaskAction.Finish, task);
var result = messages.Single();
Assert.AreEqual(expectedTaskResult, result.Result);
if (!string.IsNullOrEmpty(expectedMessage))
Assert.AreEqual(expectedMessage, result.Message);
}
public void CancelTask(TaskId taskId)
{
Channel.CancelTask(taskId);
}
/// <summary>
/// Get task data from xml node "task"
/// </summary>
private static Task getTaskInfo(XElement xTask)
{
TaskId id = new TaskId(xTask.Attribute("id").Value);
TaskStatus status = statusFromString(xTask.Attribute("status").Value);
Task task = new Task();
task.Id = id;
task.Status = status;
XAttribute xRegistrationTime = xTask.Attribute("registrationTime");
if (xRegistrationTime != null)
{
DateTime time;
if (DateTime.TryParse(xRegistrationTime.Value, out time))
task.RegistrationTime = time;
}
XAttribute xStatusChangeTime = xTask.Attribute("statusChangeTime");
if (xStatusChangeTime != null)
{
DateTime time;
if (DateTime.TryParse(xStatusChangeTime.Value, out time))
task.StatusChangeTime = time;
}
XAttribute xPagesCount = xTask.Attribute("filesCount");
if (xPagesCount != null)
{
int pagesCount;
if (Int32.TryParse(xPagesCount.Value, out pagesCount))
task.PagesCount = pagesCount;
}
XAttribute xCredits = xTask.Attribute("credits");
if (xCredits != null)
{
int credits;
if( Int32.TryParse( xCredits.Value, out credits ))
task.Credits = credits;
}
XAttribute xDescription = xTask.Attribute("description");
if (xDescription != null)
task.Description = xDescription.Value;
XAttribute xResultUrl = xTask.Attribute("resultUrl");
if (xResultUrl != null)
{
task.DownloadUrls = new List<string>{xResultUrl.Value};
for (int i = 2; i < 10; i++)
{
XAttribute xResultUrlI = xTask.Attribute("resultUrl" + i);
if (xResultUrlI != null)
{
task.DownloadUrls.Add(xResultUrlI.Value);
}
else
{
break;
}
}
}
XAttribute xError = xTask.Attribute("error");
if (xError != null)
{
task.Error = xError.Value;
}
return task;
}
public string GetLongRunningTaskResult(TaskId taskId)
{
return(Channel.GetLongRunningTaskResult(taskId));
}
public Task()
{
Status = TaskStatus.Unknown;
Id = new TaskId("<unknown>");
}
public TaskStatus GetTaskStatus(TaskId taskId)
{
return(Channel.GetTaskStatus(taskId));
}
public AssociateTaskToPlan(PlanId planId, TaskId taskId)
{
PlanId = planId;
TaskId = taskId;
}
public Task_is_committed_to_sprint(SprintId sprintId, TaskId taskId)
{
SprintId = sprintId;
TaskId = taskId;
}