/// <exception cref="System.IO.IOException"/>
private DeletionService.DeletionTaskRecoveryInfo ParseTaskProto(YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto
proto)
{
int taskId = proto.GetId();
string user = proto.HasUser() ? proto.GetUser() : null;
Path subdir = null;
IList <Path> basePaths = null;
if (proto.HasSubdir())
{
subdir = new Path(proto.GetSubdir());
}
IList <string> basedirs = proto.GetBasedirsList();
if (basedirs != null && basedirs.Count > 0)
{
basePaths = new AList <Path>(basedirs.Count);
foreach (string basedir in basedirs)
{
basePaths.AddItem(new Path(basedir));
}
}
DeletionService.FileDeletionTask task = new DeletionService.FileDeletionTask(taskId
, this, user, subdir, basePaths);
return(new DeletionService.DeletionTaskRecoveryInfo(task, proto.GetSuccessorIdsList
(), proto.GetDeletionTime()));
}
public DeletionTaskRecoveryInfo(DeletionService.FileDeletionTask task, IList <int>
successorTaskIds, long deletionTimestamp)
{
this.task = task;
this.successorTaskIds = successorTaskIds;
this.deletionTimestamp = deletionTimestamp;
}
public virtual void ScheduleFileDeletionTask(DeletionService.FileDeletionTask fileDeletionTask
)
{
if (debugDelay != -1)
{
RecordDeletionTaskInStateStore(fileDeletionTask);
sched.Schedule(fileDeletionTask, debugDelay, TimeUnit.Seconds);
}
}
public virtual DeletionService.FileDeletionTask[] GetSuccessorTasks()
{
lock (this)
{
DeletionService.FileDeletionTask[] successors = new DeletionService.FileDeletionTask
[successorTaskSet.Count];
return(Sharpen.Collections.ToArray(successorTaskSet, successors));
}
}
private void RecordDeletionTaskInStateStore(DeletionService.FileDeletionTask task
)
{
if (!stateStore.CanRecover())
{
// optimize the case where we aren't really recording
return;
}
if (task.taskId != DeletionService.FileDeletionTask.InvalidTaskId)
{
return;
}
// task already recorded
task.taskId = GenerateTaskId();
DeletionService.FileDeletionTask[] successors = task.GetSuccessorTasks();
// store successors first to ensure task IDs have been generated for them
foreach (DeletionService.FileDeletionTask successor in successors)
{
RecordDeletionTaskInStateStore(successor);
}
YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto.Builder builder
= YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto.NewBuilder
();
builder.SetId(task.taskId);
if (task.GetUser() != null)
{
builder.SetUser(task.GetUser());
}
if (task.GetSubDir() != null)
{
builder.SetSubdir(task.GetSubDir().ToString());
}
builder.SetDeletionTime(Runtime.CurrentTimeMillis() + TimeUnit.Milliseconds.Convert
(debugDelay, TimeUnit.Seconds));
if (task.GetBaseDirs() != null)
{
foreach (Path dir in task.GetBaseDirs())
{
builder.AddBasedirs(dir.ToString());
}
}
foreach (DeletionService.FileDeletionTask successor_1 in successors)
{
builder.AddSuccessorIds(successor_1.taskId);
}
try
{
stateStore.StoreDeletionTask(task.taskId, ((YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto
)builder.Build()));
}
catch (IOException e)
{
Log.Error("Unable to store deletion task " + task.taskId + " for " + task.GetSubDir
(), e);
}
}
/// <summary>
/// If there is a task dependency between say tasks 1,2,3 such that
/// task2 and task3 can be started only after task1 then we should define
/// task2 and task3 as successor tasks for task1.
/// </summary>
/// <remarks>
/// If there is a task dependency between say tasks 1,2,3 such that
/// task2 and task3 can be started only after task1 then we should define
/// task2 and task3 as successor tasks for task1.
/// Note:- Task dependency should be defined prior to
/// </remarks>
/// <param name="successorTask"/>
public virtual void AddFileDeletionTaskDependency(DeletionService.FileDeletionTask
successorTask)
{
lock (this)
{
if (successorTaskSet.AddItem(successorTask))
{
successorTask.IncrementAndGetPendingPredecessorTasks();
}
}
}
/// <summary>Delete the path(s) as this user.</summary>
/// <param name="user">The user to delete as, or the JVM user if null</param>
/// <param name="subDir">the sub directory name</param>
/// <param name="baseDirs">the base directories which contains the subDir's</param>
public virtual void Delete(string user, Path subDir, params Path[] baseDirs)
{
// TODO if parent owned by NM, rename within parent inline
if (debugDelay != -1)
{
IList <Path> baseDirList = null;
if (baseDirs != null && baseDirs.Length != 0)
{
baseDirList = Arrays.AsList(baseDirs);
}
DeletionService.FileDeletionTask task = new DeletionService.FileDeletionTask(this
, user, subDir, baseDirList);
RecordDeletionTaskInStateStore(task);
sched.Schedule(task, debugDelay, TimeUnit.Seconds);
}
}
/*
* This is called when
* 1) Current file deletion task ran and finished.
* 2) This can be even directly called by predecessor task if one of the
* dependent tasks of it has failed marking its success = false.
*/
private void FileDeletionTaskFinished()
{
lock (this)
{
try
{
delService.stateStore.RemoveDeletionTask(taskId);
}
catch (IOException e)
{
Log.Error("Unable to remove deletion task " + taskId + " from state store", e);
}
IEnumerator <DeletionService.FileDeletionTask> successorTaskI = this.successorTaskSet
.GetEnumerator();
while (successorTaskI.HasNext())
{
DeletionService.FileDeletionTask successorTask = successorTaskI.Next();
if (!success)
{
successorTask.SetSuccess(success);
}
int count = successorTask.DecrementAndGetPendingPredecessorTasks();
if (count == 0)
{
if (successorTask.GetSucess())
{
successorTask.delService.ScheduleFileDeletionTask(successorTask);
}
else
{
successorTask.FileDeletionTaskFinished();
}
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFileDeletionTaskDependency()
{
TestDeletionService.FakeDefaultContainerExecutor exec = new TestDeletionService.FakeDefaultContainerExecutor
();
Configuration conf = new Configuration();
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
del.Init(conf);
del.Start();
try
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
IList <Path> dirs = BuildDirs(r, @base, 2);
CreateDirs(new Path("."), dirs);
// first we will try to delete sub directories which are present. This
// should then trigger parent directory to be deleted.
IList <Path> subDirs = BuildDirs(r, dirs[0], 2);
DeletionService.FileDeletionTask dependentDeletionTask = del.CreateFileDeletionTask
(null, dirs[0], new Path[] { });
IList <DeletionService.FileDeletionTask> deletionTasks = new AList <DeletionService.FileDeletionTask
>();
foreach (Path subDir in subDirs)
{
DeletionService.FileDeletionTask deletionTask = del.CreateFileDeletionTask(null,
null, new Path[] { subDir });
deletionTask.AddFileDeletionTaskDependency(dependentDeletionTask);
deletionTasks.AddItem(deletionTask);
}
foreach (DeletionService.FileDeletionTask task in deletionTasks)
{
del.ScheduleFileDeletionTask(task);
}
int msecToWait = 20 * 1000;
while (msecToWait > 0 && (lfs.Util().Exists(dirs[0])))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsFalse(lfs.Util().Exists(dirs[0]));
// Now we will try to delete sub directories; one of the deletion task we
// will mark as failure and then parent directory should not be deleted.
subDirs = BuildDirs(r, dirs[1], 2);
subDirs.AddItem(new Path(dirs[1], "absentFile"));
dependentDeletionTask = del.CreateFileDeletionTask(null, dirs[1], new Path[] { }
);
deletionTasks = new AList <DeletionService.FileDeletionTask>();
foreach (Path subDir_1 in subDirs)
{
DeletionService.FileDeletionTask deletionTask = del.CreateFileDeletionTask(null,
null, new Path[] { subDir_1 });
deletionTask.AddFileDeletionTaskDependency(dependentDeletionTask);
deletionTasks.AddItem(deletionTask);
}
// marking one of the tasks as a failure.
deletionTasks[2].SetSuccess(false);
foreach (DeletionService.FileDeletionTask task_1 in deletionTasks)
{
del.ScheduleFileDeletionTask(task_1);
}
msecToWait = 20 * 1000;
while (msecToWait > 0 && (lfs.Util().Exists(subDirs[0]) || lfs.Util().Exists(subDirs
[1])))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(dirs[1]));
}
finally
{
del.Stop();
}
}
