protected void RemvoeExpiredTasks()
{
while (CurrentTasks.Count > 0 && CurrentTasks.Peek() <= DateTime.Now)
{
CurrentTasks.Dequeue();
}
}
public NewTaskVM()
{
Create = new DelegateCommand(() =>
{
CurrentTasks.Instance().InsertNewTask(task);
TaskCreated?.Invoke();
});
}
public string Generate()
{
var sb = new StringBuilder();
sb.AppendLine("*Current*:");
foreach (var task in CurrentTasks)
{
sb.AppendLine(task.GetLine());
}
if (!CurrentTasks.Any())
{
sb.AppendLine("none");
}
sb.AppendLine();
sb.AppendLine("*Done*:");
foreach (var task in DoneTasks)
{
sb.AppendLine(task.GetLine());
}
if (!DoneTasks.Any())
{
sb.AppendLine("none");
}
sb.AppendLine();
sb.AppendLine("*Blocked*:");
foreach (var task in BlockedTasks)
{
sb.AppendLine(task.GetLine());
}
if (!BlockedTasks.Any())
{
sb.AppendLine("none");
}
sb.AppendLine();
sb.Append($"*{PreviousDayLabel}*: ");
sb.AppendLine(string.IsNullOrWhiteSpace(PreviousDaySummary) ? "none" : PreviousDaySummary);
sb.AppendLine();
sb.Append("*Today*: ");
sb.AppendLine(string.IsNullOrWhiteSpace(TodaySummary) ? "none" : TodaySummary);
sb.AppendLine();
sb.Append("*Impediments*: ");
sb.Append(string.IsNullOrWhiteSpace(BlockedSummary) ? "none" : BlockedSummary);
return(sb.ToString());
}
public NewTask(CurrentTasks currentTasks)
{
task = new ProductionTask();
Create = new DelegateCommand(() =>
{
task.CreationDate = DateTime.Now;
task.Status = "1";
currentTasks.InsertNewTask(task);
TaskCreated?.Invoke();
});
}
/// <summary>
/// Cleans up after a task.
/// </summary>
/// <param name="mgr">The task manager.</param>
private void CleanupAfterTask(TaskWorkerManager mgr)
{
mgr.TasksCancelled -= mgr_TasksCancelled;
mgr.TasksCompleted -= mgr_TasksCompleted;
mgr.TasksFailed -= mgr_TasksFailed;
TaskWorker dummy;
CurrentTasks.TryRemove(mgr.CurrentTask, out dummy);
completed.Add(mgr.CurrentTask);
managers.TryRemove(mgr, out mgr);
}
public NewTask(CurrentTasks currentTasks, ProductionTask task)
{
this.task = task;
Create = new DelegateCommand(() =>
{
this.task.Status = "f";
this.task.CreationDate = DateTime.Now;
this.task.FinishDate = DateTime.Now;
currentTasks.InsertNewTask(this.task);
TaskCreated?.Invoke();
});
}
void Awake()
{
//Singleton stuff
if (instance == null)
{
instance = this;
}
else
{
Destroy(gameObject);
return;
}
}
public void StartTask(BuildNode node)
{
_logger.LogDebug($"BuildProcess.StartTask: Node: \"{node.Name}\"");
var task = FindTask(node);
if (task != null)
{
_logger.LogDebug($"BuildProcess.StartTask: Task: {task.GetHashCode()}");
CurrentTasks.Add(task);
task.Start();
TaskStarted?.Invoke(task);
}
}
private void ProcessIndividualTask()
{
if (CurrentTasks.Count > 0)
{
TaskItem currenttask = CurrentTasks.Dequeue();
currenttask.CurrentTask.ContinueWith(x => ProcessIndividualTask());
currenttask.CurrentTask.Start();
}
else
{
TaskQueueRunning = false;
}
}
public void Stop()
{
if (!IsActive)
{
return;
}
IsActive = false;
cancelSource.Cancel();
CurrentTasks.Clear();
currentTaskNumber = 1;
StateChanged?.Invoke(this, false);
}
/// <summary>
/// Runs the tasks.
/// </summary>
private void RunTasks()
{
if (completed.Count == Tasks.Count)
{
TasksCompleted.RaiseEvent(this);
return;
}
lock (CurrentTasks)
{
IEnumerable <TaskWorker> available = Dependancies.GetAvailable(Tasks, Completed);
available = available.Except(CurrentTasks.Keys);
foreach (TaskWorker worker in available)
{
List <TaskWorker> ensureNotRunning = new List <TaskWorker>();
//Check if we have a key saying that this cannot be run at the same time as...
if (PreventSimultaneousRunning.Dependancies.ContainsKey(worker))
{
ensureNotRunning = PreventSimultaneousRunning.Dependancies[worker].DependsOn.ToList();
}
//Then see if anything says x is not allowed to be run at the same time as this
IEnumerable <Dependacy <TaskWorker> > dependancies = PreventSimultaneousRunning.Dependancies.Where(d => d.Value.DependsOn.Contains(worker)).Select(kvp => kvp.Value);
ensureNotRunning.AddRange(dependancies.Select(depend =>
{
List <TaskWorker> items = new List <TaskWorker>();
items.Add(depend.Item);
return(items);
}).SelectMany(i => i));
if (CurrentTasks.Any(t => ensureNotRunning.Contains(t.Value)))
{
continue;
}
CurrentTasks.TryAdd(worker, worker);
TaskWorkerManager mgr = new TaskWorkerManager(worker);
mgr.AutoTryRecover = AutoTryRecover;
mgr.TasksCancelled += mgr_TasksCancelled;
mgr.TasksCompleted += mgr_TasksCompleted;
mgr.TasksFailed += mgr_TasksFailed;
managers.TryAdd(mgr, mgr);
mgr.Start();
}
}
}
public void DoneTask(BuildNode node, DateTime time, CommandResult result)
{
_logger.LogDebug($"BuildProcess.DoneTask: {time}");
var task = FindTask(node);
if (task == null)
{
return;
}
_logger.LogDebug($"BuildProcess.DoneTask: Task: {task.GetHashCode()}");
task.Done(result.IsSuccess, result.Message, result.Result);
Silent = result.Silent;
TaskDone?.Invoke(task);
CurrentTasks.Remove(task);
if (IsDone || IsAborted)
{
DoneBuild(time);
}
}
public MainVM()
{
TaskList = new ObservableCollection <Task>(CurrentTasks.Instance().Tasks);
OpenNewTaskWindow = new DelegateCommand(openNewTaskWindow);
StartTask = new DelegateCommand(startTask);
((INotifyCollectionChanged)CurrentTasks.Instance().Tasks).CollectionChanged += (s, a) =>
{
if (a.NewItems?.Count == 1)
{
Application.Current.Dispatcher.BeginInvoke(new Action(() =>
TaskList.Add(a.NewItems[0] as Task)));
}
if (a.OldItems?.Count == 1)
{
Application.Current.Dispatcher.BeginInvoke(new Action(() =>
TaskList.Remove(a.OldItems[0] as Task)));
}
};
}
public override async void OnNavigatedTo(NavigatedToEventArgs e, Dictionary <string, object> viewModelState)
{
bool flag = false;
string ErrMess = "";
try {
UserEntity user = _accountService.get();
CurrentTasks = _dbTaskManager.getTaskList(user);
TasksFilter = CurrentTasks.Where(x => x.Status == 0 || x.Status == 1).ToList();
NumberTasks = "ЗАДАНИЯ (" + CurrentTasks.Count + ")";
} catch (Exception ex) {
flag = true;
ErrMess = ex.Message;
}
if (flag)
{
await dialog("Error", ErrMess);
}
base.OnNavigatedTo(e, viewModelState);
}
public void StartTask()
{
bool finished = false;
while (!finished)
{
while (NumberOfOutstandingTasks >= SimultaneiousTasksLimit)
{
System.Threading.Thread.Sleep(100);
}
lock (lockObject)
{
if (NumberOfOutstandingTasks >= SimultaneiousTasksLimit)
{
// Another thread added a task to the queue before we could get the lock.
continue;
}
RemvoeExpiredTasks();
if (CurrentTasks.Count == WindowLimit)
{
TimeSpan waitTime = CurrentTasks.Dequeue() - DateTime.Now;
if (waitTime.TotalMilliseconds > 0)
{
Throttled?.Invoke(this, new ThottledEventArgs(waitTime));
System.Threading.Thread.Sleep(waitTime);
}
}
System.Threading.Interlocked.Increment(ref _numberOfOutstandingTasks);
CurrentTasks.Enqueue(DateTime.Now + WindowSize);
finished = true;
}
}
}
public void AddTask(Task task)
{
CurrentTasks.Enqueue(task);
ProcessTasks();
}
public Main()
{
errorScroller = new ErrorScroller();
errorScroller.RaiseErrorChanged += () => RaisePropertyChanged(nameof(CurrentError));
// Работа с SQL
currentTasks = new CurrentTasks(errorScroller);
checkForNewTasks = new AsutpServer(errorScroller);
TaskList = new ObservableCollection <ProductionTask>(currentTasks.TaskList);
((INotifyCollectionChanged)currentTasks.TaskList).CollectionChanged += (s, a) =>
{
if (a.NewItems?.Count >= 1)
{
Application.Current.Dispatcher.BeginInvoke(new Action(() =>
{
foreach (ProductionTask task in a.NewItems)
{
TaskList.Add(task);
}
}));
}
if (a.OldItems?.Count >= 1)
{
Application.Current.Dispatcher.BeginInvoke(new Action(() =>
{
foreach (ProductionTask task in a.OldItems)
{
TaskList.Remove(task);
}
}));
}
};
FinishedTaskList = new ObservableCollection <ProductionTask>(currentTasks.FinishedTaskList);
((INotifyCollectionChanged)currentTasks.FinishedTaskList).CollectionChanged += (s, a) =>
{
if (a.NewItems?.Count >= 1)
{
Application.Current.Dispatcher.BeginInvoke(new Action(() =>
{
foreach (ProductionTask task in a.NewItems)
{
FinishedTaskList.Add(task);
}
}));
}
if (a.OldItems?.Count >= 1)
{
Application.Current.Dispatcher.BeginInvoke(new Action(() =>
{
foreach (ProductionTask task in a.OldItems)
{
FinishedTaskList.Remove(task);
}
}));
}
};
//Работа с ПЛК
plc = new Plc();
OpenNewTaskWindow = new DelegateCommand(() =>
{
NewTaskWindow newTaskWindow = new NewTaskWindow(new NewTask(currentTasks));
newTaskWindow.ShowDialog();
});
StartTask = new DelegateCommand(() =>
{
try
{
plc.SendTask(new ProductionTaskExtended(SelectedTask));
currentTasks.UpdateStartDate(SelectedTask.ID);
}
catch (Exception e)
{
Log.logThis(e.Message);
MessageBox.Show("Нет подключения к ПЛК");
}
//RaisePropertyChanged(nameof(SelectedTask));
});
FinishTask = new DelegateCommand(() =>
{
try
{
ProductionTask taskResult = plc.GetCurrentTaskResult();
taskResult.FinishDate = DateTime.Now;
try
{
currentTasks.LoadTaskResult(taskResult);
}
catch (TaskNotCreatedException)
{
MessageBox.Show("Задание не найдено. Введите параметры");
NewTaskWindow newTaskWindow = new NewTaskWindow(new NewTask(currentTasks, taskResult));
newTaskWindow.ShowDialog();
}
}
catch (Exception e)
{
MessageBox.Show(e.Message);
}
});
ShowCurrentTask = new DelegateCommand(() => { VisibleCurrentTask = true; VisibleFinishedTask = false; });
ShowFinishedTask = new DelegateCommand(() => { VisibleCurrentTask = false; VisibleFinishedTask = true; });
}
private async void Work()
{
var taskNumber = 0;
semaphore.Wait();
taskNumber = currentTaskNumber++;
CurrentTasks.TryAdd(taskNumber, null);
semaphore.Release();
while (!cancelSource.IsCancellationRequested)
{
Work w;
string url;
try
{
#region Get valid work
if (!Manager.IsWorkAvailable || Manager.GetWork(out w, crawlDelay: Config.CrawlDelaySeconds) == false)
{
// unable to get work, wait a bit and try again
CurrentTasks[taskNumber] = null;
await Task.Delay(20);
continue;
}
url = w.Url;
// check if url is whitelisted
if (Extensions.IsUrlWhitelisted(url, Config) == false)
{
Logger.Log($"Skipping URL '{url}' - Not whitelisted!", Logger.LogSeverity.Debug);
continue;
}
// check robots.txt and blacklisted tags
// (this also attempts to download robots.txt on first run)
if (robots.IsUrlExcluded(url, Config, true).Result)
{
continue;
}
// get crawl-delay as defined by 'robots.txt'
var wait = robots.GetWaitTime(url, Config);
// wait the difference between [ROBOTS.TXT CRAWL DELAY] - [GLOBAL CRAWL DELAY]
var difference = wait - Config.CrawlDelaySeconds;
if (difference > 0)
{
Task.Delay((int)TimeSpan.FromSeconds(difference).TotalMilliseconds).Wait();
}
#endregion
}
catch (Exception ex)
{
Logger.Log($"Error while trying to get valid work! " + ex.GetDetailedMessage(), Logger.LogSeverity.Warning);
continue;
}
DateTime?recrawlDate = null;
var lastCrawl = w.LastCrawled;
w.LastCrawled = DateTime.Now;
CurrentTasks[taskNumber] = url;
HttpStatusCode?statusCode = null;
try
{
// Get response headers - DO NOT READ CONTENT yet (performance reasons)
var response = await httpClient
.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancelSource.Token);
statusCode = response.StatusCode;
if (!response.IsSuccessStatusCode || cancelSource.IsCancellationRequested)
{
#region Failed to crawl
// TODO: treat differently based on status code (for ex. if page doesn't exist at all, or if 500, 404,...)
switch (response.StatusCode)
{
case HttpStatusCode.Redirect:
// Add the redirected location to backlog
var newurl = response.Headers.Location.AbsoluteUri;
if (string.IsNullOrEmpty(newurl) == false)
{
// check if URL is eligible for crawling
if (Manager.IsUrlEligibleForCrawl(newurl) == false)
{
continue;
}
if (Manager.IsUrlCrawled(newurl))
{
// ignore already-crawled urls
}
else
{
Manager.AddToBacklog(newurl);
}
}
break;
case HttpStatusCode.MethodNotAllowed:
case HttpStatusCode.Gone:
case HttpStatusCode.BadRequest:
case HttpStatusCode.NoContent:
case HttpStatusCode.Unauthorized:
case HttpStatusCode.NotFound:
case HttpStatusCode.Forbidden:
// ignore it - mark as failed
break;
case HttpStatusCode.BadGateway:
case HttpStatusCode.TooManyRequests:
case HttpStatusCode.InternalServerError:
// if no recrawl date set yet, set it into the future
if (w.RecrawlDate == null && w.LastCrawled != null)
{
recrawlDate = DateTime.Now.AddMinutes(5);
}
// if recrawl was already set, double it since last time
else
{
var duration = w.RecrawlDate.Value.Subtract(lastCrawl.Value);
recrawlDate = DateTime.Now.Add(duration.Multiply(2));
}
break;
default:
break;
}
// if recrawl date was set
if (recrawlDate != null)
{
w.RecrawlDate = recrawlDate;
}
w.Success = false;
continue;
#endregion
}
var mediaType = response.Content?.Headers?.ContentType?.MediaType;
if (mediaType == null)
{
continue;
}
// Check if media type is set as a scanning target, if yes, scan it for new URLs
if (Config.ScanTargetsMediaTypes.Count(x => x == mediaType) > 0)
{
// scan content for more urls
var content = await response.Content.ReadAsStringAsync();
UrlFinder.ScanContentAndAddToManager(url, content, Config, plugins, Manager, robots, cancelSource);
}
// Check if media type is set as an accepted file to download
#region Download resource if valid
// attempt to get filename
var filename = GetFilename(url, mediaType);
// check if URL matches defined URL patterns
if (Extensions.IsURLMatch(url, Config) == false)
{
continue;
}
// don't download file if not acceptable
if (IsAcceptable(filename, mediaType) == false ||
cancelSource.IsCancellationRequested)
{
continue;
}
// check file size limits
var size = response.Content.Headers.ContentLength;
// if content length is not provided, ignore file
if (size == null)
{
continue;
}
var sizekB = size / 1024;
if (Config.MinimumAllowedFileSizekB != -1 && sizekB < Config.MinimumAllowedFileSizekB)
{
continue;
}
if (Config.MaximumAllowedFileSizekB != -1 && sizekB > Config.MaximumAllowedFileSizekB)
{
continue;
}
// construct path
var directory = GetDirectoryPath(url, true);
var path = Path.Combine(directory, filename);
// check plugins
if (plugins?.Invoke(p => p.BeforeDownload(url, path), true) == false)
{
Logger.Log($"Plugin rejected download of '{filename}'", Logger.LogSeverity.Debug);
continue;
}
// get temporary file to download content to
var temp = Extensions.GetTempFile(ConfigManager.TemporaryFileTransferDirectory);
try
{
// download content to temporary file
using (var fstream = new FileStream(temp, FileMode.Create, FileAccess.Write, FileShare.None))
await response.Content.CopyToAsync(fstream);
// now compare temp file contents to destination file - check for duplicates using MD5 hash comparing
path = Extensions.CopyToAndGetPath(temp, path);
plugins?.Invoke(p => p.AfterDownload(url, path));
}
catch
{
throw;
}
finally
{
File.Delete(temp);
}
// log the download
RecentDownloads.Add(new DownloadedWork(path, response.Content.Headers.ContentLength.Value));
// Logger.Log($"Downloaded '{url}' to '{path}'");
w.DownloadLocation = Extensions.GetRelativeFilePath(path, Config);
w.IsDownloaded = true;
w.Success = true;
Logger.Log($"Downloaded ({response.StatusCode}) {url}");
#endregion
}
catch (OperationCanceledException) { }
catch (NullReferenceException nex)
{
Logger.Log($"NullReferenceException while crawling - {url} - {nex.Message} -- {nex.StackTrace}",
Logger.LogSeverity.Error);
}
catch (IOException iex)
{
// usually happens when trying to download file with same name
Logger.Log($"IOException while crawling - {iex.Message}",
Logger.LogSeverity.Debug);
}
catch (Exception ex)
{
Logger.Log($"Exception while crawling - {url} - ({ex.GetType().Name}) {ex.Message}",
Logger.LogSeverity.Debug);
}
finally
{
// also log crawled that weren't downloaded and had successful response
if (!w.Success && Config.LogEveryCrawl)
{
if (statusCode == null)
{
Logger.Log($"Canceled {url}");
// TODO: re-add it to backlog maybe?
}
else
{
Logger.Log($"Crawled ({statusCode}) {url}");
}
}
CurrentTasks[taskNumber] = null;
Manager.ReportWorkResult(w);
}
}
}
