private bool StartHouseKeeping(Task task) { var cleanThread = System.Threading.Tasks.Task.Factory.StartNew(() => { List <P2PBackup.Common.Task> expiredBackups = new DAL.TaskDAO().GetExpiredBackups(); task.OriginalSize = expiredBackups.Sum(o => o.FinalSize); task.TotalItems = expiredBackups.Count; Logger.Append("HUBRN", Severity.INFO, "Started cleaning " + expiredBackups.Count + " expired backups"); //int done = 0; try{ foreach (P2PBackup.Common.Task nodeTask in expiredBackups) { if (nodeTask == null) { continue; } nodeTask.RunStatus = TaskRunningStatus.Expiring; new DAL.TaskDAO().Update(nodeTask); PeerNode node = Hub.NodesList.GetById(nodeTask.BackupSet.NodeId); if (node != null) { Logger.Append("HUBRN", Severity.INFO, "Asking node #" + node.Id + " (" + node.Name + ") to expire task " + nodeTask.Id); //node.SendMessage("EXP "+task.Id+" "+nodeTask.Id+" "+nodeTask.IndexName+" "+nodeTask.IndexSum); node.ManageTask(nodeTask, TaskAction.Expire); } else { Logger.Append("HUBRN", Severity.WARNING, "Can't expire task " + nodeTask.Id + " of node #" + nodeTask.BackupSet.NodeId + ", node is offline"); } //done++; } } catch (Exception e) { Console.WriteLine("StartHouseKeeping() : " + e.Message + " ---- " + e.StackTrace); } }, System.Threading.Tasks.TaskCreationOptions.LongRunning); /*cleanThread.ContinueWith(o=>{ * UpdateTask(task.Id, task.OriginalSize, task.FinalSize, "", "", new List<int>(), 100); * }, System.Threading.Tasks.TaskContinuationOptions.OnlyOnRanToCompletion);*/ return(true); }
private Task CreateTask(BackupSet bs, TaskStartupType taskType, User u, BackupLevel?overridenLevel) { if (TasksQueueExists(bs)) { return(null); } Task newBackup = new Task(bs, taskType); newBackup.Operation = bs.Operation; newBackup.NodeId = bs.NodeId; newBackup.CurrentAction = "Initializing"; newBackup.RunStatus = TaskRunningStatus.PendingStart; newBackup.StartDate = DateTime.Now; if (overridenLevel.HasValue) { newBackup.Level = overridenLevel.Value; } else if (bs.ScheduleTimes.Count == 1) { newBackup.Level = bs.ScheduleTimes[0].Level; } else { newBackup.Level = BackupLevel.Refresh; } if (u != null) { newBackup.UserId = u.Id; Logger.Append("HUBRN", Severity.DEBUG, "User " + u.Id + " (" + u.Name + ") started new task for backupset " + bs.Id + " with level " + newBackup.Level + " (client #" + bs.NodeId + ")"); } // set an encryption key newBackup.EncryptionKey = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); newBackup = new DAL.TaskDAO().Save(newBackup); TasksQueue.Add((Task)newBackup); Logger.Append("RUN", Severity.TRIVIA, "Created new task for scheduled backupset " + bs.ToString()); TaskPublisher.Instance().Notify(newBackup); return(newBackup); }
private void Schedule(CancellationToken cancelToken) { while (!cancelToken.IsCancellationRequested) { if (isFirstStart) { int interruptedTasks = new DAL.TaskDAO().UpdateInterrupted(); Logger.Append("HUBRN", Severity.INFO, "Updated " + interruptedTasks + " previously interrupted task(s)."); Logger.Append("HUBRN", Severity.INFO, "Waiting 20s before starting to schedule tasks..."); Thread.Sleep(20000); isFirstStart = false; } Logger.Append("HUBRN", Severity.DEBUG, "Polling database for next BackupSets to run."); var nextBS = new List <BackupSet> (); try{ nextBS = new DAL.BackupSetDAO().GetNextToSchedule(1); } catch (Exception e) { Logger.Append("HUBRN", Severity.CRITICAL, "Cannot retrieve next tasks to schedule from DB : " + e.ToString()); } foreach (BackupSet bsn in nextBS) { CreateTask(bsn, TaskStartupType.Scheduled, null, null); } Logger.Append("HUBRN", Severity.DEBUG, "Added " + nextBS.Count + " Backup Sets to queue. Total queue size : " + TasksQueue.Count); for (int i = TasksQueue.Count - 1; i >= 0; i--) { Task task = TasksQueue.GetByIndex(i); //REACTIVER!!! //if(TimeSpan.Parse(bs.BackupTimes[0].Begin+":00") <= new TimeSpan(DateTime.Now.Hour, DateTime.Now.Minute, 0)){ if (task.RunStatus == TaskRunningStatus.Done || task.RunStatus == TaskRunningStatus.Cancelled || task.RunStatus == TaskRunningStatus.Error) { Logger.Append("HUBRN", Severity.INFO, "Task " + task.Id + ", Backupset " + task.BackupSet.ToString() + " ended with status " + task.RunStatus + ", removing from queue."); task.EndDate = DateTime.Now; new DAL.TaskDAO().Complete(task); TasksQueue.RemoveAt(i); // TODO : remove this line, instead keep DEBUG one if (TaskEvent != null) { TaskEvent(task, null); } #if DEBUG #else TasksQueue.RemoveAt(i); #endif } else if (task.RunStatus == TaskRunningStatus.PendingStart) { task.Status = TaskStatus.Ok; try{ StartTask(task); task.RunStatus = TaskRunningStatus.Started; // wait 0.1 second before starting another task Thread.Sleep(100); } catch (OverQuotaException oqe) { Logger.Append("HUBRN", Severity.ERROR, "Could not start task #" + task.Id + " on client node #" + task.BackupSet.NodeId + " : " + oqe.Message); task.Status = TaskStatus.Error; task.RunStatus = TaskRunningStatus.Error; task.EndDate = DateTime.Now; task.CurrentAction = oqe.Message; task.AddLogEntry(new TaskLogEntry(task.Id) { Code = 830, Message1 = "", Message2 = "" }); } catch (UnreachableNodeException) { Logger.Append("HUBRN", Severity.ERROR, "Could not send task #" + task.Id + " to node #" + task.BackupSet.NodeId + ": Node is offline"); TimeSpan lateness = DateTime.Now.Subtract(task.StartDate); if (lateness >= new TimeSpan(0, maxLateness, 0)) { Logger.Append("HUBRN", Severity.ERROR, "Could not start task " + task.Id + " for client #" + task.BackupSet.NodeId + ", retry time expired. (lateness :" + lateness.Minutes + " minutes)"); task.Status = TaskStatus.Error; task.RunStatus = TaskRunningStatus.Error; task.EndDate = DateTime.Now; task.CurrentAction = "Retry time expired"; task.AddLogEntry(new TaskLogEntry(task.Id) { Code = 901, Message1 = "", Message2 = "" }); } else { int remainingRetry = new TimeSpan(0, maxLateness, 0).Subtract(lateness).Minutes; Logger.Append("HUBRN", Severity.WARNING, "Could not start task " + task.Id + " for client #" + task.BackupSet.NodeId + ", will retry during " + remainingRetry + " mn"); task.Status = TaskStatus.Warning; //TaskPublisher.Instance().Notify(task); task.AddLogEntry(new TaskLogEntry(task.Id) { Code = 901, Message1 = remainingRetry.ToString(), Message2 = "" }); task.CurrentAction = "Could not start operation, will retry during " + remainingRetry + " mn"; } } } } // end for Utils.SetProcInfo("Hub (" + TasksQueue.Count + " tasks)"); for (int i = 0; i < 60; i++) { if (!cancelToken.IsCancellationRequested) { Thread.Sleep(1000); } } } Logger.Append("HUBRN", Severity.INFO, "Scheduler stopped."); }
private bool StartTask(Task task) { bool done = false; if (task.Operation == TaskOperation.HouseKeeping) { return(StartHouseKeeping(task)); } PeerNode taskTargetNode = GetHandlingNode(task); // temp : for debugging udp wakeup /*if(taskTargetNode == null){ * NodesMonitor.Instance.WakeUp(new DAL.NodeDAO().Get(task.NodeId)); * Thread.Sleep(5000); * taskTargetNode = GetHandlingNode(task.Id); * }*/ if (taskTargetNode == null) { throw new UnreachableNodeException("Node #" + taskTargetNode + " is offline or unreachable"); } else if (taskTargetNode.Quota > 0 && taskTargetNode.UsedQuota >= taskTargetNode.Quota) { throw new OverQuotaException(taskTargetNode.UsedQuota, taskTargetNode.Quota); } else if (taskTargetNode.Status == NodeStatus.Idle) { Logger.Append("HUBRN", Severity.INFO, "Node #" + taskTargetNode.Id + " is idle, telling him to wakeup and prepare for task #" + task.Id); NodeWakeUpNeeded(taskTargetNode); return(false); } Logger.Append("HUBRN", Severity.INFO, "Starting Task " + task.Id + " : type " + task.Type + " ( level " + task.Level /* .BackupSet.ScheduleTimes[0].Level*/ + "), backup Set " + task.BackupSet.Id + " for client #" + task.BackupSet.NodeId + " (handled by node #" + task.BackupSet.HandledBy + ")"); //Console.WriteLine("TaskScheduler : handledby = "+task.BackupSet.HandledBy+", proxying info is null : "+(task.BackupSet.ProxyingInfo == null)); try{ BackupLevel referenceLevel = BackupLevel.Default; if (task.Level == BackupLevel.Differential) { referenceLevel = BackupLevel.Full; } P2PBackup.Common.Task referenceTask = new DAL.TaskDAO().GetLastReferenceTask(task.BackupSet.Id, referenceLevel); if (referenceTask != null) { task.StorageBudget = (int)((referenceTask.OriginalSize / task.BackupSet.MaxChunkSize) + 2); Console.WriteLine(" ____ ref task=" + referenceTask.Id + ", oSize=" + referenceTask.OriginalSize / 1024 / 1024 + "MB, maxchunksize=" + task.BackupSet.MaxChunkSize / 1024 / 1024 + "MB, %%=" + referenceTask.OriginalSize / task.BackupSet.MaxChunkSize + ", calculated budget=" + task.StorageBudget); task.ParentTask = referenceTask; } if (task.Level != BackupLevel.Full) { if (referenceTask == null || referenceTask.Id <= 0) // no ref backup found, doing full { Logger.Append("HUBRN", Severity.INFO, "No reference backup found for task " + task.Id + ", performing FULL backup."); task.Level = BackupLevel.Full; } else { task.ParentTrackingId = referenceTask.Id; Logger.Append("HUBRN", Severity.INFO, "Task " + task.Id + " is " + task.Level + "." + " Using reference task " + referenceTask.Id + " (" + referenceTask.StartDate + " - " + referenceTask.EndDate + ")"); } } taskTargetNode.ManageTask(task, TaskAction.Start); task.RunStatus = TaskRunningStatus.Started; //n.Status = NodeStatus.Backuping; done = true; } catch (Exception e) { done = false; Logger.Append("HUBRN", Severity.ERROR, "Could not send task " + task.Id + " to node #" + taskTargetNode.Id + " : " + e.ToString() /*+"---Stacktrace:"+e.StackTrace+" inner msg:"+e.InnerException.Message*/); //n.Status = NodeStatus.Error; } return(done); }