public void Map_null()
{
var map = new MapTask <TestItem>()
.Map <int>(null);
Assert.That(map.Maps.Count, Is.EqualTo(0));
}
public void AliasTo_null()
{
var map = new MapTask <TestItem>()
.AliasTo(null);
Assert.That(map.Alias, Is.EqualTo("testitem"));
}
public async Task <IActionResult> GetTaskFromQueue()
{
var allTasks = await this.stateManager_.GetOrAddAsync <IReliableDictionary <Guid, MapTask> >("allTasks");
var tasksQueue = await this.stateManager_.GetOrAddAsync <IReliableConcurrentQueue <MapTask> >("tasksQueue");
using (var tx = this.stateManager_.CreateTransaction())
{
var conditionalValue = await tasksQueue.TryDequeueAsync(tx);
if (conditionalValue.HasValue)
{
MapTask task = conditionalValue.Value;
task.State = MapTask.StateType.Processing;
await allTasks.SetAsync(tx, task.Uuid, task);
await tx.CommitAsync();
return(Json(conditionalValue.Value));
}
else
{
return(new NotFoundResult());
}
}
}
public void MapAction_5()
{
var map = new MapTask <TestAction>()
.MapAction <int, int, int, int, int>((x, v1, v2, v3, v4, v5) => x.Sum = v1 + v2 + v3 + v4 + v5, "value1",
"value2", "value3", "value4", "value5");
var action = map.GroupActions.First();
var first = map.CmsFields.First();
var second = map.CmsFields.Skip(1).Take(1).First();
var third = map.CmsFields.Skip(2).Take(1).First();
var forth = map.CmsFields.Skip(3).Take(1).First();
var fifth = map.CmsFields.Skip(4).Take(1).First();
var item = new TestAction();
Assert.That(map.CmsFields.Count, Is.EqualTo(5));
Assert.That(first.Key, Is.EqualTo("value1"));
Assert.That(first.Value, Is.EqualTo(typeof(int)));
Assert.That(second.Key, Is.EqualTo("value2"));
Assert.That(second.Value, Is.EqualTo(typeof(int)));
Assert.That(third.Key, Is.EqualTo("value3"));
Assert.That(third.Value, Is.EqualTo(typeof(int)));
Assert.That(forth.Key, Is.EqualTo("value4"));
Assert.That(forth.Value, Is.EqualTo(typeof(int)));
Assert.That(fifth.Key, Is.EqualTo("value5"));
Assert.That(fifth.Value, Is.EqualTo(typeof(int)));
Assert.That(action.Aliases.First(), Is.EqualTo("value1"));
Assert.That(action.Aliases.Skip(1).Take(1).First(), Is.EqualTo("value2"));
Assert.That(action.Aliases.Skip(2).Take(1).First(), Is.EqualTo("value3"));
Assert.That(action.Aliases.Skip(3).Take(1).First(), Is.EqualTo("value4"));
Assert.That(action.Aliases.Skip(4).Take(1).First(), Is.EqualTo("value5"));
action.Action(item, new object[] { 1, 2, 3, 4, 5 });
Assert.That(item.Sum, Is.EqualTo(15));
}
public void Create_new_success()
{
var map = new MapTask <TestItem>();
var result = map.Create();
Assert.That(result is TestItem);
}
public void Constructor_success()
{
var map = new MapTask <TestItem>();
Assert.That(map.Alias, Is.EqualTo("testitem"));
Assert.That(map.EntityType, Is.EqualTo(typeof(TestItem)));
}
public void AliasTo_success()
{
var map = new MapTask <TestItem>()
.AliasTo("ABC");
Assert.That(map.Alias, Is.EqualTo("abc"));
}
public async Task <IActionResult> PostNewJob([FromBody] string sourceText)
{
ServiceEventSource.Current.ServiceMessage(this.context_, "Orchestrator param: " + sourceText);
CancellationToken ct = new CancellationToken();
var jobsDictionary = await this.stateManager_.GetOrAddAsync <IReliableDictionary <Guid, JobObject> >("jobsDictionary");
var tasksQueue = await this.stateManager_.GetOrAddAsync <IReliableConcurrentQueue <MapTask> >("tasksQueue");
var allTasks = await this.stateManager_.GetOrAddAsync <IReliableDictionary <Guid, MapTask> >("allTasks");
using (ITransaction tx = this.stateManager_.CreateTransaction())
{
Guid jobGuid = System.Guid.NewGuid();
JobObject job = new JobObject(jobGuid);
MapTask task;
int cursorPos = 0;
while (cursorPos + TASK_SIZE < sourceText.Length)
{
int nextCursorPos = cursorPos + TASK_SIZE;
while (sourceText[nextCursorPos] != ' ')
{
nextCursorPos--;
}
task = new MapTask(jobGuid, sourceText.Substring(cursorPos, nextCursorPos - cursorPos));
task.State = MapTask.StateType.InQueue;
job.Tasks.Add(task.Uuid, false);
await allTasks.AddAsync(tx, task.Uuid, task);
await tasksQueue.EnqueueAsync(tx, task, ct);
cursorPos = nextCursorPos + 1;
}
task = new MapTask(jobGuid, sourceText.Substring(cursorPos));
task.State = MapTask.StateType.InQueue;
job.Tasks.Add(task.Uuid, false);
await allTasks.AddAsync(tx, task.Uuid, task);
await tasksQueue.EnqueueAsync(tx, task, ct);
try
{
await jobsDictionary.AddAsync(tx, job.Uuid, job, TimeSpan.FromSeconds(1), ct);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this.context_, e.Message);
}
await tx.CommitAsync();
return(new OkResult());
}
}
public void Constructor_without_model_maps_success()
{
var mapTask = new MapTask <TestEntity>();
var defer = new TaskDefer(mapTask.ModelMaps);
Assert.That(defer.Maps, Is.Not.Null);
Assert.That(defer.Maps.Count, Is.EqualTo(0));
}
public void Create_return_null()
{
var map = new MapTask <TestItem>();
var instance = new TestEntity();
var result = map.Create(instance);
Assert.That(result is TestItem);
}
public void Map_default()
{
var map = new MapTask <TestItem>()
.Map(x => x.ItemId, defaultVal: 1);
var mapItem = map.Maps.First();
Assert.That(mapItem.Default, Is.EqualTo(1));
}
public void Map_alias()
{
var map = new MapTask <TestItem>()
.Map(x => x.ItemId, "iD");
var mapItem = map.Maps.First();
Assert.That(mapItem.Alias, Is.EqualTo("id"));
}
TextAsset[] prefabText; //現在のモードの出現マップ
// Start is called before the first frame update
void Start()
{
mapTask = GetComponent <MapTask>();
cameraTask = GetComponent <CameraTask>();
prefabText = Resources.LoadAll <TextAsset>(GetPath.MainStage);
CreatePrefab(Vector2Int.zero, StartMapData());
CreateMap(RandomMapData());
}
protected internal override Task CreateRemoteTask()
{
//job file name is set in TaskAttempt, setting it null here
MapTask mapTask = new MapTask(string.Empty, TypeConverter.FromYarn(GetID()), partition
, splitInfo.GetSplitIndex(), 1);
// YARN doesn't have the concept of slots per task, set it as 1.
mapTask.SetUser(conf.Get(MRJobConfig.UserName));
mapTask.SetConf(conf);
return(mapTask);
}
public void Constructor_with_model_maps_success()
{
var mapTask = new MapTask <TestEntity>();
mapTask.Match(x => x.Item);
var defer = new TaskDefer(mapTask.ModelMaps);
Assert.That(defer.Maps, Is.Not.Null);
Assert.That(defer.Maps.Count, Is.EqualTo(1));
Assert.That(defer.Maps.First() is MapItem <object, string>);
}
public void MatchAction_not_valid()
{
var map = new MapTask <TestEntity>()
.MatchAction <TestItem>(null, "items")
.MatchAction <TestItem>((x, y) =>
{
x.List = y.ToList();
});
Assert.That(map.ModelMaps.Count, Is.EqualTo(1));
}
public void Create_with_instance()
{
var map = new MapTask <TestItem>();
var instance = new TestItem {
ItemId = 100
};
var result = map.Create(instance) as TestItem;
Assert.That(result, Is.EqualTo(instance));
Assert.That(result?.ItemId, Is.EqualTo(100));
}
public async Task <IActionResult> Put(Guid uuid, [FromBody] MapTask task)
{
var jobsDictionary = await this.stateManager_.GetOrAddAsync <IReliableDictionary <Guid, JobObject> >("jobsDictionary");
var completeJobsQueue = await this.stateManager_.GetOrAddAsync <IReliableConcurrentQueue <string> >("completeJobsQueue");
var allTasks = await this.stateManager_.GetOrAddAsync <IReliableDictionary <Guid, MapTask> >("allTasks");
using (ITransaction tx = this.stateManager_.CreateTransaction())
{
var jobConditionalValue = await jobsDictionary.TryGetValueAsync(tx, uuid);
if (jobConditionalValue.HasValue)
{
JobObject job = jobConditionalValue.Value;
job.SetTaskCompleted(task.Uuid);
if (job.IsComplete)
{
// Job is complete, remove it, send the result to FrontEndSvc for now
var removeConditional = await jobsDictionary.TryRemoveAsync(tx, job.Uuid);
if (!removeConditional.HasValue)
{
// Retry, I guess?
}
var counts = new List <Dictionary <string, int> >();
// Current task's Output field is null when taken from allTasks dictionary
foreach (var taskUuid in job.Tasks.Keys)
{
MapTask t = (await allTasks.TryRemoveAsync(tx, taskUuid)).Value;
counts.Add(t.Output ?? task.Output);
}
string serialized = JsonConvert.SerializeObject(counts);
await completeJobsQueue.EnqueueAsync(tx, serialized);
}
else
{
task.State = MapTask.StateType.Completed;
await jobsDictionary.SetAsync(tx, job.Uuid, job);
await allTasks.SetAsync(tx, task.Uuid, task);
}
}
await tx.CommitAsync();
}
return(new OkResult());
}
public void Map_func()
{
var map = new MapTask <TestItem>()
.Map(x => x.ItemId, mapFunc: x => int.Parse(x) * 2);
var mapItem = map.Maps.First();
var item = new TestItem();
var value = mapItem.Converter("3");
mapItem.Setter.Set(item, value);
Assert.That(item.ItemId, Is.EqualTo(6));
}
public void Ignore_ok()
{
var map = new MapTask <TestItem>()
.MapAuto();
Assert.That(map.Maps.Count, Is.EqualTo(5));
map.Ignore(x => x.ItemId);
Assert.That(map.Maps.Count, Is.EqualTo(4));
//again, no change
map.Ignore(x => x.ItemId);
Assert.That(map.Maps.Count, Is.EqualTo(4));
}
public void Map_property()
{
var map = new MapTask <TestItem>()
.Map(x => x.ItemId);
Assert.That(map.Maps.Count, Is.EqualTo(1));
var mapItem = map.Maps.First();
Assert.That(mapItem.Alias, Is.EqualTo("itemid"));
Assert.That(mapItem.Setter, Is.Not.Null);
var item = new TestItem();
mapItem.Setter.Set(item, 8);
Assert.That(item.ItemId, Is.EqualTo(8));
}
//-------------------------------------------------------------------------
public void c2sTaskGiveUp(int task_id)
{
if (IsClient)
{
return;
}
TaskBase task = null;
MapTask.TryGetValue(task_id, out task);
if (task != null)
{
task.c2sTaskGiveUp();
}
}
public void Match_alias()
{
var map = new MapTask <TestEntity>()
.Match(x => x.Item, alias: "item-single");
var modelMap = map.ModelMaps.First();
var entity = new TestEntity();
var item = new TestItem();
Assert.That(map.ModelMaps.Count, Is.EqualTo(1));
Assert.That(modelMap.Alias, Is.EqualTo("item-single"));
Assert.That(modelMap.IsMatch, Is.EqualTo(ModelMap.DefaultMatchKey));
Assert.That(!modelMap.IsMany);
modelMap.SetModel(entity, item);
Assert.That(entity.Item, Is.EqualTo(item));
}
public void MatchMany_alias()
{
var map = new MapTask <TestEntity>()
.MatchMany(x => x.Items, alias: "items-many");
var modelMap = map.ModelMaps.First();
var entity = new TestEntity();
var items = new[] { new TestItem() };
Assert.That(map.ModelMaps.Count, Is.EqualTo(1));
Assert.That(modelMap.Alias, Is.EqualTo("items-many"));
Assert.That(modelMap.IsMatch, Is.EqualTo(ModelMap.DefaultMatchKey));
Assert.That(modelMap.IsMany);
modelMap.SetModel(entity, items);
Assert.That(entity.Items, Is.EqualTo(items));
}
public void MapAction_1()
{
var map = new MapTask <TestAction>()
.MapAction <int>((x, v) => x.Sum = v + 1, "value1");
var action = map.GroupActions.First();
var field = map.CmsFields.First();
var item = new TestAction();
Assert.That(map.CmsFields.Count, Is.EqualTo(1));
Assert.That(map.Maps.Count, Is.EqualTo(0));
Assert.That(field.Key, Is.EqualTo("value1"));
Assert.That(field.Value, Is.EqualTo(typeof(int)));
Assert.That(action.Aliases.First(), Is.EqualTo("value1"));
action.Action(item, new object[] { 2 });
Assert.That(item.Sum, Is.EqualTo(3));
}
public void MapContent_Success()
{
var content = Substitute.For <IContent>();
content.Get <int>(Arg.Any <string>()).Returns(1);
var item = new TestAction();
var map = new MapTask <TestAction>()
.MapContent((c, x) =>
{
x.Agregate = "value1 = " + c.Get <int>("value1");
});
Assert.That(map.Action, Is.Not.Null);
map.Action(content, item);
Assert.That(item.Agregate, Is.EqualTo("value1 = 1"));
}
//-------------------------------------------------------------------------
public void serverUpdate(float elapsed_tm)
{
if (QueTaskDone.Count > 0)
{
foreach (var i in QueTaskDone)
{
TaskBase task = null;
MapTask.TryGetValue(i, out task);
if (task == null)
{
continue;
}
if (task.TaskData.task_state == TaskState.Release)
{
// 查询下一个任务
int next_task_id = task.TbDataTask.NextId;
// 创建下一个任务
TbDataTask next_tbdata_task = EbDataMgr.Instance.getData <TbDataTask>(next_task_id);
if (next_tbdata_task == null)
{
continue;
}
TaskFactory next_task_factory = null;
MapTaskFactory.TryGetValue(next_tbdata_task.TaskType, out next_task_factory);
if (next_task_factory != null)
{
TaskBase next_task = next_task_factory.createTask(next_task_id);
MapTask[next_task.TaskData.task_id] = next_task;
mMapDirtyTask[next_task_id] = next_task.TaskData;
}
}
}
}
while (QueTaskDone.Count > 0)
{
int task_id = QueTaskDone.Dequeue();
MapTask.Remove(task_id);
}
}
public void MatchMany_with_keyFunc()
{
Func <TestItem, string, bool> matchKey = (model, key) => model.Key == Guid.Parse(key);
var map = new MapTask <TestEntity>()
.MatchMany(x => x.Items, matchKey);
var modelMap = map.ModelMaps.First();
var entity = new TestEntity();
var items = new[] { new TestItem() };
Assert.That(map.ModelMaps.Count, Is.EqualTo(1));
Assert.That(modelMap.Alias, Is.EqualTo("items"));
Assert.That(modelMap.IsMatch, Is.Not.EqualTo(matchKey));
Assert.That(modelMap.IsMany);
modelMap.SetModel(entity, items);
Assert.That(entity.Items, Is.EqualTo(items));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
public override void Init(MapOutputCollector.Context context)
{
this.mapTask = context.GetMapTask();
this.jobConf = context.GetJobConf();
this.reporter = context.GetReporter();
numberOfPartitions = jobConf.GetNumReduceTasks();
keyClass = (Type)jobConf.GetMapOutputKeyClass();
valueClass = (Type)jobConf.GetMapOutputValueClass();
recordWriters = new TestMerge.KeyValueWriter[numberOfPartitions];
outStreams = new ByteArrayOutputStream[numberOfPartitions];
// Create output streams for partitions.
for (int i = 0; i < numberOfPartitions; i++)
{
outStreams[i] = new ByteArrayOutputStream();
recordWriters[i] = new TestMerge.KeyValueWriter <K, V>(jobConf, outStreams[i], keyClass
, valueClass);
}
}
/// <summary>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// </summary>
/// <remarks>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// protocol is played.
/// The test mocks the various steps in a failed task's
/// life-cycle using a special OutputCommitter and UmbilicalProtocol
/// implementation.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestTaskCleanupDoesNotCommit()
{
// Mimic a job with a special committer that does not cleanup
// files when a task fails.
JobConf job = new JobConf();
job.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithoutCleanup));
Path outDir = new Path(rootDir, "output");
FileOutputFormat.SetOutputPath(job, outDir);
// Mimic job setup
string dummyAttemptID = "attempt_200707121733_0001_m_000000_0";
TaskAttemptID attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
OutputCommitter committer = new TestTaskCommit.CommitterWithoutCleanup();
JobContext jContext = new JobContextImpl(job, ((JobID)attemptID.GetJobID()));
committer.SetupJob(jContext);
// Mimic a map task
dummyAttemptID = "attempt_200707121733_0001_m_000001_0";
attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
Task task = new MapTask(null, attemptID, 0, null, 1);
task.SetConf(job);
task.LocalizeConfiguration(job);
task.Initialize(job, ((JobID)attemptID.GetJobID()), Reporter.Null, false);
// Mimic the map task writing some output.
string file = "test.txt";
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat <Text, Text> theOutputFormat = new TextOutputFormat <Text, Text>();
RecordWriter <Text, Text> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, Reporter.Null);
theRecordWriter.Write(new Text("key"), new Text("value"));
theRecordWriter.Close(Reporter.Null);
// Mimic a task failure; setting up the task for cleanup simulates
// the abort protocol to be played.
// Without checks in the framework, this will fail
// as the committer will cause a COMMIT to happen for
// the cleanup task.
task.SetTaskCleanupTask();
TestTaskCommit.MyUmbilical umbilical = new TestTaskCommit.MyUmbilical(this);
task.Run(job, umbilical);
NUnit.Framework.Assert.IsTrue("Task did not succeed", umbilical.taskDone);
}
