public IList <Dog> Get()
{
IList <Dog> result = MemoryCache.Get <IList <Dog> >(cacheName);
if (result == null)
{
Task <IList <Dog> > dbFetchTask = new Task <IList <Dog> >(() => DogService.Get());
bool isRunning = false;
lock (ThreadWatcher)
{
isRunning = ThreadWatcher.IsRunning(cacheName);
}
if (!isRunning)
{
ThreadStartedEvent?.Invoke(this, new ThreadStartedEvent(cacheName)); // Fire:a event.
dbFetchTask.Start(); // Starta bara tråden, om det inte redan körs en sådan.
}
dbFetchTask.ContinueWith(x =>
{
MemoryCache.Set(cacheName, x.Result, options: MemoryCacheEntryOptions);
ThreadFinishedEvent?.Invoke(this, new ThreadFinishedEvent(cacheName)); // Tala om att tråden är klar
});
}
else
{
Console.WriteLine($"Used MemoryCache for {cacheName}.");
}
return(result);
}
private void OnThreadStartedEvent(object sender, ThreadStartedEvent e)
{
Console.WriteLine("Started thread!!!!!!");
lock (ThreadWatcher)
{
ThreadWatcher.StartThread(e);
}
}
private void OnThreadFinishedEvent(object sender, ThreadFinishedEvent e)
{
Console.WriteLine("Stopped thread!!!!!!");
lock (ThreadWatcher)
{
ThreadWatcher.FinishThread(e);
}
}
public frmThreadEdit(ThreadWatcher watcher, Dictionary<string, int> categories)
{
InitializeComponent();
GUI.SetFontAndScaling(this);
foreach (string key in categories.Keys) {
cboCategory.Items.Add(key);
}
cboCheckEvery.ValueMember = "Value";
cboCheckEvery.DisplayMember = "Text";
cboCheckEvery.DataSource = new[] {
new ListItemInt32(0, "1 or <"),
new ListItemInt32(2, "2"),
new ListItemInt32(3, "3"),
new ListItemInt32(5, "5"),
new ListItemInt32(10, "10"),
new ListItemInt32(60, "60")
};
txtDescription.Text = watcher.Description;
cboCategory.Text = watcher.Category;
chkPageAuth.Checked = !String.IsNullOrEmpty(watcher.PageAuth);
txtPageAuth.Text = watcher.PageAuth;
chkPageAuth.Checked = !String.IsNullOrEmpty(watcher.ImageAuth);
txtImageAuth.Text = watcher.ImageAuth;
chkOneTime.Checked = watcher.OneTimeDownload;
chkAutoFollow.Checked = watcher.AutoFollow;
switch (watcher.CheckIntervalSeconds / 60) {
case 0: cboCheckEvery.SelectedIndex = 0; break;
case 1: cboCheckEvery.SelectedIndex = 0; break;
case 2: cboCheckEvery.SelectedIndex = 1; break;
case 3: cboCheckEvery.SelectedIndex = 2; break;
case 5: cboCheckEvery.SelectedIndex = 3; break;
case 10: cboCheckEvery.SelectedIndex = 4; break;
case 60: cboCheckEvery.SelectedIndex = 5; break;
default:
txtCheckEvery.Text = (watcher.CheckIntervalSeconds / 60).ToString(CultureInfo.InvariantCulture);
cboCheckEvery.SelectedIndex = -1;
cboCheckEvery.Enabled = false;
break;
}
if (watcher.IsRunning) {
chkPageAuth.Enabled = false;
txtPageAuth.Enabled = false;
chkImageAuth.Enabled = false;
txtImageAuth.Enabled = false;
chkOneTime.Enabled = false;
chkAutoFollow.Enabled = false;
pnlCheckEvery.Enabled = false;
}
cboCheckEvery.SelectedIndexChanged += cboCheckEvery_SelectedIndexChanged;
txtCheckEvery.TextChanged += txtCheckEvery_TextChanged;
}
private async Task ScrapeThreadAsync(string destinationDirectory, string threadUrl)
{
Downloader downloader = new Downloader(_logger)
{
TargetDirectory = destinationDirectory,
};
_logger.Log($"Directory target set to {destinationDirectory}");
ImageQueuer imageQueuer = new ImageQueuer(_logger, downloader);
ThreadWatcher threadWatcher = new ThreadWatcher(_logger, downloader, imageQueuer)
{
RefreshRate = 5000,
ThreadURL = threadUrl
};
_logger.Log($"Thread target set to {threadUrl}");
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
_logger.Log($"Thread watcher started");
try
{
await threadWatcher.WatchThread(_cancellationToken);
}
catch (OperationCanceledException)
{
stopwatch.Stop();
TimeSpan ts = stopwatch.Elapsed;
_logger.Log("Watching stopped.");
_logger.Log($"Time elapsed: {ts.Hours}:{ts.Minutes}:{ts.Seconds}.{ts.Milliseconds}");
return;
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Initializes a new DebugDisplay class.
/// </summary>
/// <param name="entityComposer">The EntityComposer.</param>
public DebugDisplay(EntityComposer entityComposer)
{
_currentEntityComposer = entityComposer;
_gameLoop = SGL.QueryComponents <GameLoop>();
_font = new Font("Segoe UI", 12, TypefaceStyle.Regular);
_cpuWatcher = new CpuWatcher();
_memoryWatcher = new MemoryWatcher();
_threadWatcher = new ThreadWatcher();
_display = new Rectangle(0, 0, 800, 480);
_debugMessage = "Query information ...";
_pen = new Pen(Color.Green, 1);
_pen2 = new Pen(Color.Red, 1);
_pen3 = new Pen(Color.Blue, 1);
_cpuWatcher.Start();
_memoryWatcher.Start();
_threadWatcher.Start();
}
private void DisplayStatus(ThreadWatcher watcher, string status)
{
SetSubItemText(watcher, ColumnIndex.Status, status);
}
private void DisplayDescription(ThreadWatcher watcher)
{
SetSubItemText(watcher, ColumnIndex.Description, watcher.Description);
}
private void DisplayCategory(ThreadWatcher watcher)
{
SetSubItemText(watcher, ColumnIndex.Category, watcher.Category);
}
private void DisplayAddedFrom(ThreadWatcher watcher)
{
ThreadWatcher fromWatcher;
_watchers.TryGetValue(((WatcherExtraData)watcher.Tag).AddedFrom ?? String.Empty, out fromWatcher);
SetSubItemText(watcher, ColumnIndex.AddedFrom, fromWatcher != null ? fromWatcher.Description : String.Empty);
}
private void ThreadWatcher_WaitStatus(ThreadWatcher watcher, EventArgs args)
{
BeginInvoke(() => {
SetWaitStatus(watcher);
SetupWaitTimer();
});
}
private void ThreadWatcher_StopStatus(ThreadWatcher watcher, StopStatusEventArgs args)
{
BeginInvoke(() => {
SetStopStatus(watcher, args.StopReason);
SetupWaitTimer();
if (args.StopReason != StopReason.UserRequest && args.StopReason != StopReason.Exiting) {
_saveThreadList = true;
}
});
}
private void ThreadWatcher_DownloadStatus(ThreadWatcher watcher, DownloadStatusEventArgs args)
{
WatcherExtraData extraData = (WatcherExtraData)watcher.Tag;
bool isInitialPageDownload = false;
bool isFirstImageUpdate = false;
if (args.DownloadType == DownloadType.Page) {
if (!extraData.HasDownloadedPage) {
extraData.HasDownloadedPage = true;
isInitialPageDownload = true;
}
extraData.PreviousDownloadWasPage = true;
}
if (args.DownloadType == DownloadType.Image && extraData.PreviousDownloadWasPage) {
extraData.LastImageOn = DateTime.Now;
extraData.PreviousDownloadWasPage = false;
isFirstImageUpdate = true;
}
BeginInvoke(() => {
SetDownloadStatus(watcher, args.DownloadType, args.CompleteCount, args.TotalCount);
if (isInitialPageDownload) {
DisplayDescription(watcher);
_saveThreadList = true;
}
if (isFirstImageUpdate) {
DisplayLastImageOn(watcher);
_saveThreadList = true;
}
SetupWaitTimer();
});
}
private void SetSubItemText(ThreadWatcher watcher, ColumnIndex columnIndex, string text)
{
ListViewItem item = ((WatcherExtraData)watcher.Tag).ListViewItem;
var subItem = item.SubItems[(int)columnIndex];
if (subItem.Text != text) {
subItem.Text = text;
}
}
private void SetReparseStatus(ThreadWatcher watcher, ReparseType reparseType, int completeCount, int totalCount)
{
string type;
bool hideDetail = false;
switch (reparseType) {
case ReparseType.Page:
type = totalCount == 1 ? "page" : "pages";
hideDetail = totalCount == 1;
break;
case ReparseType.Image:
type = "images";
break;
default:
return;
}
string status = hideDetail ? "Reparsing " + type :
String.Format("Reparsing {0}: {1} of {2} completed", type, completeCount, totalCount);
DisplayStatus(watcher, status);
//TODO: Find alternative to Application.DoEvents() (throws StackOverflowException when reparsing a large number of images)
//Application.DoEvents();
}
private bool AddThread(string pageURL, string pageAuth, string imageAuth, int checkInterval, bool oneTime, string saveDir, string description, StopReason? stopReason, WatcherExtraData extraData)
{
ThreadWatcher watcher = null;
ListViewItem newListViewItem = null;
foreach (ThreadWatcher existingWatcher in ThreadWatchers) {
if (String.Equals(existingWatcher.PageURL, pageURL, StringComparison.OrdinalIgnoreCase)) {
if (existingWatcher.IsRunning) return false;
watcher = existingWatcher;
break;
}
}
if (watcher == null) {
watcher = new ThreadWatcher(pageURL);
watcher.ThreadDownloadDirectory = saveDir;
watcher.Description = description;
watcher.DownloadStatus += ThreadWatcher_DownloadStatus;
watcher.WaitStatus += ThreadWatcher_WaitStatus;
watcher.StopStatus += ThreadWatcher_StopStatus;
watcher.ThreadDownloadDirectoryRename += ThreadWatcher_ThreadDownloadDirectoryRename;
watcher.DownloadStart += ThreadWatcher_DownloadStart;
watcher.DownloadProgress += ThreadWatcher_DownloadProgress;
watcher.DownloadEnd += ThreadWatcher_DownloadEnd;
newListViewItem = new ListViewItem(String.Empty);
for (int i = 1; i < lvThreads.Columns.Count; i++) {
newListViewItem.SubItems.Add(String.Empty);
}
newListViewItem.Tag = watcher;
lvThreads.Items.Add(newListViewItem);
}
watcher.PageAuth = pageAuth;
watcher.ImageAuth = imageAuth;
watcher.CheckIntervalSeconds = checkInterval;
watcher.OneTimeDownload = oneTime;
if (extraData == null) {
extraData = watcher.Tag as WatcherExtraData;
if (extraData == null) {
extraData = new WatcherExtraData {
AddedOn = DateTime.Now
};
}
}
if (newListViewItem != null) {
extraData.ListViewItem = newListViewItem;
}
watcher.Tag = extraData;
DisplayDescription(watcher);
DisplayAddedOn(watcher);
DisplayLastImageOn(watcher);
if (stopReason == null) {
watcher.Start();
}
else {
watcher.Stop(stopReason.Value);
}
return true;
}
private void ThreadWatcher_DownloadStart(ThreadWatcher watcher, DownloadStartEventArgs args)
{
DownloadProgressInfo info = new DownloadProgressInfo();
info.DownloadID = args.DownloadID;
info.URL = args.URL;
info.TryNumber = args.TryNumber;
info.StartTicks = TickCount.Now;
info.TotalSize = args.TotalSize;
lock (_downloadProgresses) {
_downloadProgresses[args.DownloadID] = info;
ConcurrentDownloads += 1;
}
}
private void SetDownloadStatus(ThreadWatcher watcher, DownloadType downloadType, int completeCount, int totalCount)
{
string type;
bool hideDetail = false;
switch (downloadType) {
case DownloadType.Page:
type = totalCount == 1 ? "page" : "pages";
hideDetail = totalCount == 1;
break;
case DownloadType.Image:
type = "images";
break;
case DownloadType.Thumbnail:
type = "thumbnails";
break;
default:
return;
}
string status = hideDetail ? "Downloading " + type :
String.Format("Downloading {0}: {1} of {2} completed", type, completeCount, totalCount);
DisplayStatus(watcher, status);
}
private void ThreadWatcher_ReparseStatus(ThreadWatcher watcher, ReparseStatusEventArgs args)
{
BeginInvoke(() => {
SetReparseStatus(watcher, args.ReparseType, args.CompleteCount, args.TotalCount);
SetupWaitTimer();
});
}
private void SetReparseStatus(ThreadWatcher watcher, ReparseType reparseType, int completeCount, int totalCount)
{
string type;
bool hideDetail = false;
switch (reparseType) {
case ReparseType.Page:
type = totalCount == 1 ? "page" : "pages";
hideDetail = totalCount == 1;
break;
case ReparseType.Image:
type = "images";
break;
default:
return;
}
string status = hideDetail ? "Reparsing " + type :
String.Format("Reparsing {0}: {1} of {2} completed", type, completeCount, totalCount);
DisplayStatus(watcher, status);
}
private void ThreadWatcher_ThreadDownloadDirectoryRename(ThreadWatcher watcher, EventArgs args)
{
BeginInvoke(() => {
_saveThreadList = true;
});
}
private void SetStopStatus(ThreadWatcher watcher, StopReason stopReason)
{
string status = "Stopped: ";
switch (stopReason) {
case StopReason.UserRequest:
status += "User requested";
break;
case StopReason.Exiting:
status += "Exiting";
break;
case StopReason.PageNotFound:
status += "Page not found";
break;
case StopReason.DownloadComplete:
status += "Download complete";
break;
case StopReason.IOError:
status += "Error writing to disk";
break;
default:
status += "Unknown error";
break;
}
DisplayStatus(watcher, status);
}
private bool AddThread(ThreadInfo thread)
{
ThreadWatcher watcher = null;
ThreadWatcher parentThread = null;
ListViewItem newListViewItem = null;
SiteHelper siteHelper = SiteHelper.GetInstance((new Uri(thread.URL)).Host);
siteHelper.SetURL(thread.URL);
string pageID = siteHelper.GetPageID();
if (IsBlacklisted(pageID)) return false;
if (_watchers.ContainsKey(pageID)) {
watcher = _watchers[pageID];
if (watcher.IsRunning) return false;
}
if (watcher == null) {
watcher = new ThreadWatcher(thread.URL);
watcher.ThreadDownloadDirectory = thread.SaveDir;
watcher.Description = thread.Description;
if (_isLoadingThreadsFromFile) watcher.DoNotRename = true;
watcher.Category = thread.Category;
watcher.DoNotRename = false;
if (thread.ExtraData != null && !String.IsNullOrEmpty(thread.ExtraData.AddedFrom)) {
_watchers.TryGetValue(thread.ExtraData.AddedFrom, out parentThread);
watcher.ParentThread = parentThread;
}
watcher.DownloadStatus += ThreadWatcher_DownloadStatus;
watcher.WaitStatus += ThreadWatcher_WaitStatus;
watcher.StopStatus += ThreadWatcher_StopStatus;
watcher.ReparseStatus += ThreadWatcher_ReparseStatus;
watcher.ThreadDownloadDirectoryRename += ThreadWatcher_ThreadDownloadDirectoryRename;
watcher.DownloadStart += ThreadWatcher_DownloadStart;
watcher.DownloadProgress += ThreadWatcher_DownloadProgress;
watcher.DownloadEnd += ThreadWatcher_DownloadEnd;
watcher.AddThread += ThreadWatcher_AddThread;
newListViewItem = new ListViewItem(String.Empty);
for (int i = 1; i < lvThreads.Columns.Count; i++) {
newListViewItem.SubItems.Add(String.Empty);
}
newListViewItem.Tag = watcher;
lvThreads.Items.Add(newListViewItem);
lvThreads.Sort();
UpdateCategories(watcher.Category);
}
watcher.PageAuth = thread.PageAuth;
watcher.ImageAuth = thread.ImageAuth;
watcher.CheckIntervalSeconds = thread.CheckIntervalSeconds;
watcher.OneTimeDownload = thread.OneTimeDownload;
watcher.AutoFollow = thread.AutoFollow;
if (thread.ExtraData == null) {
thread.ExtraData = watcher.Tag as WatcherExtraData ?? new WatcherExtraData { AddedOn = DateTime.Now };
}
if (newListViewItem != null) {
thread.ExtraData.ListViewItem = newListViewItem;
}
watcher.Tag = thread.ExtraData;
if (parentThread != null) parentThread.ChildThreads.Add(watcher.PageID, watcher);
if (!_watchers.ContainsKey(watcher.PageID)) {
_watchers.Add(watcher.PageID, watcher);
}
else {
_watchers[watcher.PageID] = watcher;
}
DisplayData(watcher);
if (thread.StopReason == null && !_isLoadingThreadsFromFile) {
watcher.Start();
}
else if (thread.StopReason != null) {
watcher.Stop(thread.StopReason.Value);
}
return true;
}
private void SetSubItemText(ThreadWatcher watcher, ColumnIndex columnIndex, string text)
{
ListViewItem item = ((WatcherExtraData)watcher.Tag).ListViewItem;
var subItem = item.SubItems[(int)columnIndex];
if (subItem.Text != text) {
subItem.Text = text;
if (!_isLoadingThreadsFromFile) lvThreads.Sort();
}
}
private void DisplayAddedOn(ThreadWatcher watcher)
{
DateTime time = ((WatcherExtraData)watcher.Tag).AddedOn;
SetSubItemText(watcher, ColumnIndex.AddedOn, time.ToString("yyyy/MM/dd HH:mm:ss"));
}
private void SetWaitStatus(ThreadWatcher watcher)
{
int remainingSeconds = (watcher.MillisecondsUntilNextCheck + 999) / 1000;
DisplayStatus(watcher, String.Format("Waiting {0} seconds", remainingSeconds));
}
private void DisplayData(ThreadWatcher watcher)
{
DisplayDescription(watcher);
DisplayAddedOn(watcher);
DisplayLastImageOn(watcher);
if (!_isLoadingThreadsFromFile) DisplayAddedFrom(watcher);
DisplayCategory(watcher);
}
private void ThreadWatcher_AddThread(ThreadWatcher watcher, AddThreadEventArgs args)
{
BeginInvoke(() => {
ThreadInfo thread = new ThreadInfo {
URL = args.PageURL,
PageAuth = watcher.PageAuth,
ImageAuth = watcher.ImageAuth,
CheckIntervalSeconds = watcher.CheckIntervalSeconds,
OneTimeDownload = watcher.OneTimeDownload,
SaveDir = null,
Description = String.Empty,
StopReason = null,
ExtraData = new WatcherExtraData {
AddedOn = DateTime.Now,
AddedFrom = watcher.PageID
},
Category = watcher.Category,
AutoFollow = Settings.RecursiveAutoFollow != false
};
SiteHelper siteHelper = SiteHelper.GetInstance((new Uri(thread.URL)).Host);
siteHelper.SetURL(thread.URL);
if (_watchers.ContainsKey(siteHelper.GetPageID())) return;
if (AddThread(thread)) {
_saveThreadList = true;
}
});
}
private void DisplayLastImageOn(ThreadWatcher watcher)
{
DateTime? time = ((WatcherExtraData)watcher.Tag).LastImageOn;
SetSubItemText(watcher, ColumnIndex.LastImageOn, time != null ? time.Value.ToString("yyyy/MM/dd HH:mm:ss") : String.Empty);
}
private void ThreadWatcher_DownloadEnd(ThreadWatcher watcher, DownloadEndEventArgs args)
{
lock (_downloadProgresses) {
DownloadProgressInfo info;
if (!_downloadProgresses.TryGetValue(args.DownloadID, out info)) return;
info.EndTicks = TickCount.Now;
info.DownloadedSize = args.DownloadedSize;
info.TotalSize = args.DownloadedSize;
_downloadProgresses[args.DownloadID] = info;
ConcurrentDownloads -= 1;
}
}
private void FocusThread(ThreadWatcher watcher)
{
ListViewItem item = ((WatcherExtraData)watcher.Tag).ListViewItem;
lvThreads.SelectedItems.Clear();
lvThreads.Select();
item.Selected = true;
item.EnsureVisible();
}
private void ThreadWatcher_DownloadProgress(ThreadWatcher watcher, DownloadProgressEventArgs args)
{
lock (_downloadProgresses) {
DownloadProgressInfo info;
if (!_downloadProgresses.TryGetValue(args.DownloadID, out info)) return;
info.DownloadedSize = args.DownloadedSize;
_downloadProgresses[args.DownloadID] = info;
}
}