private bool StartHouseKeeping(Task task)
{
var cleanThread = System.Threading.Tasks.Task.Factory.StartNew(() => {
List <P2PBackup.Common.Task> expiredBackups = new DAL.TaskDAO().GetExpiredBackups();
task.OriginalSize = expiredBackups.Sum(o => o.FinalSize);
task.TotalItems = expiredBackups.Count;
Logger.Append("HUBRN", Severity.INFO, "Started cleaning " + expiredBackups.Count + " expired backups");
//int done = 0;
try{
foreach (P2PBackup.Common.Task nodeTask in expiredBackups)
{
if (nodeTask == null)
{
continue;
}
nodeTask.RunStatus = TaskRunningStatus.Expiring;
new DAL.TaskDAO().Update(nodeTask);
PeerNode node = Hub.NodesList.GetById(nodeTask.BackupSet.NodeId);
if (node != null)
{
Logger.Append("HUBRN", Severity.INFO, "Asking node #" + node.Id + " (" + node.Name + ") to expire task " + nodeTask.Id);
//node.SendMessage("EXP "+task.Id+" "+nodeTask.Id+" "+nodeTask.IndexName+" "+nodeTask.IndexSum);
node.ManageTask(nodeTask, TaskAction.Expire);
}
else
{
Logger.Append("HUBRN", Severity.WARNING, "Can't expire task " + nodeTask.Id + " of node #" + nodeTask.BackupSet.NodeId + ", node is offline");
}
//done++;
}
}
catch (Exception e) {
Console.WriteLine("StartHouseKeeping() : " + e.Message + " ---- " + e.StackTrace);
}
}, System.Threading.Tasks.TaskCreationOptions.LongRunning);
/*cleanThread.ContinueWith(o=>{
* UpdateTask(task.Id, task.OriginalSize, task.FinalSize, "", "", new List<int>(), 100);
* }, System.Threading.Tasks.TaskContinuationOptions.OnlyOnRanToCompletion);*/
return(true);
}
private Task CreateTask(BackupSet bs, TaskStartupType taskType, User u, BackupLevel?overridenLevel)
{
if (TasksQueueExists(bs))
{
return(null);
}
Task newBackup = new Task(bs, taskType);
newBackup.Operation = bs.Operation;
newBackup.NodeId = bs.NodeId;
newBackup.CurrentAction = "Initializing";
newBackup.RunStatus = TaskRunningStatus.PendingStart;
newBackup.StartDate = DateTime.Now;
if (overridenLevel.HasValue)
{
newBackup.Level = overridenLevel.Value;
}
else if (bs.ScheduleTimes.Count == 1)
{
newBackup.Level = bs.ScheduleTimes[0].Level;
}
else
{
newBackup.Level = BackupLevel.Refresh;
}
if (u != null)
{
newBackup.UserId = u.Id;
Logger.Append("HUBRN", Severity.DEBUG, "User " + u.Id + " (" + u.Name + ") started new task for backupset " + bs.Id + " with level " + newBackup.Level + " (client #" + bs.NodeId + ")");
}
// set an encryption key
newBackup.EncryptionKey = Convert.ToBase64String(Guid.NewGuid().ToByteArray());
newBackup = new DAL.TaskDAO().Save(newBackup);
TasksQueue.Add((Task)newBackup);
Logger.Append("RUN", Severity.TRIVIA, "Created new task for scheduled backupset " + bs.ToString());
TaskPublisher.Instance().Notify(newBackup);
return(newBackup);
}
private void Schedule(CancellationToken cancelToken)
{
while (!cancelToken.IsCancellationRequested)
{
if (isFirstStart)
{
int interruptedTasks = new DAL.TaskDAO().UpdateInterrupted();
Logger.Append("HUBRN", Severity.INFO, "Updated " + interruptedTasks + " previously interrupted task(s).");
Logger.Append("HUBRN", Severity.INFO, "Waiting 20s before starting to schedule tasks...");
Thread.Sleep(20000);
isFirstStart = false;
}
Logger.Append("HUBRN", Severity.DEBUG, "Polling database for next BackupSets to run.");
var nextBS = new List <BackupSet> ();
try{
nextBS = new DAL.BackupSetDAO().GetNextToSchedule(1);
}
catch (Exception e) {
Logger.Append("HUBRN", Severity.CRITICAL, "Cannot retrieve next tasks to schedule from DB : " + e.ToString());
}
foreach (BackupSet bsn in nextBS)
{
CreateTask(bsn, TaskStartupType.Scheduled, null, null);
}
Logger.Append("HUBRN", Severity.DEBUG, "Added " + nextBS.Count + " Backup Sets to queue. Total queue size : " + TasksQueue.Count);
for (int i = TasksQueue.Count - 1; i >= 0; i--)
{
Task task = TasksQueue.GetByIndex(i);
//REACTIVER!!! //if(TimeSpan.Parse(bs.BackupTimes[0].Begin+":00") <= new TimeSpan(DateTime.Now.Hour, DateTime.Now.Minute, 0)){
if (task.RunStatus == TaskRunningStatus.Done || task.RunStatus == TaskRunningStatus.Cancelled || task.RunStatus == TaskRunningStatus.Error)
{
Logger.Append("HUBRN", Severity.INFO, "Task " + task.Id + ", Backupset " + task.BackupSet.ToString() + " ended with status " + task.RunStatus + ", removing from queue.");
task.EndDate = DateTime.Now;
new DAL.TaskDAO().Complete(task);
TasksQueue.RemoveAt(i); // TODO : remove this line, instead keep DEBUG one
if (TaskEvent != null)
{
TaskEvent(task, null);
}
#if DEBUG
#else
TasksQueue.RemoveAt(i);
#endif
}
else if (task.RunStatus == TaskRunningStatus.PendingStart)
{
task.Status = TaskStatus.Ok;
try{
StartTask(task);
task.RunStatus = TaskRunningStatus.Started;
// wait 0.1 second before starting another task
Thread.Sleep(100);
}
catch (OverQuotaException oqe) {
Logger.Append("HUBRN", Severity.ERROR, "Could not start task #" + task.Id + " on client node #" + task.BackupSet.NodeId
+ " : " + oqe.Message);
task.Status = TaskStatus.Error;
task.RunStatus = TaskRunningStatus.Error;
task.EndDate = DateTime.Now;
task.CurrentAction = oqe.Message;
task.AddLogEntry(new TaskLogEntry(task.Id)
{
Code = 830, Message1 = "", Message2 = ""
});
}
catch (UnreachableNodeException) {
Logger.Append("HUBRN", Severity.ERROR, "Could not send task #" + task.Id + " to node #" + task.BackupSet.NodeId + ": Node is offline");
TimeSpan lateness = DateTime.Now.Subtract(task.StartDate);
if (lateness >= new TimeSpan(0, maxLateness, 0))
{
Logger.Append("HUBRN", Severity.ERROR, "Could not start task " + task.Id + " for client #" + task.BackupSet.NodeId + ", retry time expired. (lateness :" + lateness.Minutes + " minutes)");
task.Status = TaskStatus.Error;
task.RunStatus = TaskRunningStatus.Error;
task.EndDate = DateTime.Now;
task.CurrentAction = "Retry time expired";
task.AddLogEntry(new TaskLogEntry(task.Id)
{
Code = 901, Message1 = "", Message2 = ""
});
}
else
{
int remainingRetry = new TimeSpan(0, maxLateness, 0).Subtract(lateness).Minutes;
Logger.Append("HUBRN", Severity.WARNING, "Could not start task " + task.Id + " for client #" + task.BackupSet.NodeId + ", will retry during " + remainingRetry + " mn");
task.Status = TaskStatus.Warning;
//TaskPublisher.Instance().Notify(task);
task.AddLogEntry(new TaskLogEntry(task.Id)
{
Code = 901, Message1 = remainingRetry.ToString(), Message2 = ""
});
task.CurrentAction = "Could not start operation, will retry during " + remainingRetry + " mn";
}
}
}
} // end for
Utils.SetProcInfo("Hub (" + TasksQueue.Count + " tasks)");
for (int i = 0; i < 60; i++)
{
if (!cancelToken.IsCancellationRequested)
{
Thread.Sleep(1000);
}
}
}
Logger.Append("HUBRN", Severity.INFO, "Scheduler stopped.");
}
private bool StartTask(Task task)
{
bool done = false;
if (task.Operation == TaskOperation.HouseKeeping)
{
return(StartHouseKeeping(task));
}
PeerNode taskTargetNode = GetHandlingNode(task);
// temp : for debugging udp wakeup
/*if(taskTargetNode == null){
* NodesMonitor.Instance.WakeUp(new DAL.NodeDAO().Get(task.NodeId));
* Thread.Sleep(5000);
* taskTargetNode = GetHandlingNode(task.Id);
* }*/
if (taskTargetNode == null)
{
throw new UnreachableNodeException("Node #" + taskTargetNode + " is offline or unreachable");
}
else if (taskTargetNode.Quota > 0 && taskTargetNode.UsedQuota >= taskTargetNode.Quota)
{
throw new OverQuotaException(taskTargetNode.UsedQuota, taskTargetNode.Quota);
}
else if (taskTargetNode.Status == NodeStatus.Idle)
{
Logger.Append("HUBRN", Severity.INFO, "Node #" + taskTargetNode.Id + " is idle, telling him to wakeup and prepare for task #" + task.Id);
NodeWakeUpNeeded(taskTargetNode);
return(false);
}
Logger.Append("HUBRN", Severity.INFO, "Starting Task " + task.Id + " : type " + task.Type + " ( level " + task.Level /* .BackupSet.ScheduleTimes[0].Level*/ + "), backup Set " + task.BackupSet.Id + " for client #" + task.BackupSet.NodeId + " (handled by node #" + task.BackupSet.HandledBy + ")");
//Console.WriteLine("TaskScheduler : handledby = "+task.BackupSet.HandledBy+", proxying info is null : "+(task.BackupSet.ProxyingInfo == null));
try{
BackupLevel referenceLevel = BackupLevel.Default;
if (task.Level == BackupLevel.Differential)
{
referenceLevel = BackupLevel.Full;
}
P2PBackup.Common.Task referenceTask = new DAL.TaskDAO().GetLastReferenceTask(task.BackupSet.Id, referenceLevel);
if (referenceTask != null)
{
task.StorageBudget = (int)((referenceTask.OriginalSize / task.BackupSet.MaxChunkSize) + 2);
Console.WriteLine(" ____ ref task=" + referenceTask.Id + ", oSize=" + referenceTask.OriginalSize / 1024 / 1024 + "MB, maxchunksize=" + task.BackupSet.MaxChunkSize / 1024 / 1024 + "MB, %%=" + referenceTask.OriginalSize / task.BackupSet.MaxChunkSize + ", calculated budget=" + task.StorageBudget);
task.ParentTask = referenceTask;
}
if (task.Level != BackupLevel.Full)
{
if (referenceTask == null || referenceTask.Id <= 0) // no ref backup found, doing full
{
Logger.Append("HUBRN", Severity.INFO, "No reference backup found for task " + task.Id + ", performing FULL backup.");
task.Level = BackupLevel.Full;
}
else
{
task.ParentTrackingId = referenceTask.Id;
Logger.Append("HUBRN", Severity.INFO, "Task " + task.Id + " is " + task.Level + "."
+ " Using reference task " + referenceTask.Id + " (" + referenceTask.StartDate + " - " + referenceTask.EndDate + ")");
}
}
taskTargetNode.ManageTask(task, TaskAction.Start);
task.RunStatus = TaskRunningStatus.Started;
//n.Status = NodeStatus.Backuping;
done = true;
}
catch (Exception e) {
done = false;
Logger.Append("HUBRN", Severity.ERROR, "Could not send task " + task.Id + " to node #" + taskTargetNode.Id + " : " + e.ToString() /*+"---Stacktrace:"+e.StackTrace+" inner msg:"+e.InnerException.Message*/);
//n.Status = NodeStatus.Error;
}
return(done);
}
