public void RequestDownload(string title, string[] urls, string[] paths, SemaphoreExtends se, string folder, IArticle article)
{
lock (count_lock)
{
if (!Settings.Instance.Model.DownloadWithRawFileName)
{
paths = PathFilenameSorter.Sort(paths);
}
for (int i = 0; i < urls.Length; i++)
{
Application.Current.Dispatcher.Invoke(new Action(
delegate
{
view_model.Items.Add(new DownloadDataGridItemViewModel
{
인덱스 = (++index_count).ToString(),
제목 = title,
경로 = paths[i]
});
Progress.Maximum += 1;
Status.Text = $"{Progress.Value} / {Progress.Maximum}";
}));
}
DownloadGroup.Instance.Add(urls, paths, Tuple.Create(folder, article), null, se);
}
}
private void Notify()
{
int i = mtx;
if (queue.Count > i)
{
string s1 = queue[i].Item1;
string s2 = queue[i].Item2;
object s3 = queue[i].Item3;
SemaphoreCallBack s4 = queue[i].Item4;
SemaphoreExtends s5 = queue[i].Item5;
lock (task_lock)
tasks.Add(Task.Run(() => DownloadRemoteImageFile(s1, s2, s3, s4, s5)).ContinueWith(
x => Task.Run(() => { lock (task_lock) tasks.RemoveAll(y => y.IsCompleted); })));
Interlocked.Increment(ref mtx);
}
}
private void remote_download_thread_handler(object i)
{
int index = (int)i;
//Monitor.Instance.Push($"[Emilia Queue] Starts download thread [{i}]");
while (true)
{
interrupt[index].WaitOne();
Tuple <string, string, object, SemaphoreCallBack, SemaphoreExtends> job;
lock (queue)
{
if (queue.Count > 0)
{
job = queue[0];
queue.RemoveAt(0);
}
else
{
//Monitor.Instance.Push($"[Emilia Queue] Suspends download thread [{i}]");
interrupt[index].Reset();
continue;
}
}
string uri = job.Item1;
string fileName = job.Item2;
object obj = job.Item3;
SemaphoreCallBack callback = job.Item4;
SemaphoreExtends se = job.Item5;
if (!Directory.Exists(Path.GetDirectoryName(fileName)))
{
Monitor.Instance.Push($"[Directory Not Found] {uri} is auto deleted in download queue.");
goto END;
}
int retry_count = 0;
bool lock_donwload_size = false;
RETRY:
if (retry_count > Settings.Instance.Net.RetryCount)
{
Monitor.Instance.Push($"[Many Retry] {uri} is auto deleted in download queue.");
lock (err_callback) err_callback(uri, "[Emilia Queue] Many retry. auto deleted in download queue.", obj);
lock (callback) callback(uri, fileName, obj);
return;
}
if (!uri.StartsWith("http"))
{
Monitor.Instance.Push($"[Url Error] {uri} is not corret url");
lock (err_callback) err_callback(uri, "[Emilia Queue] Url Error. not corret url.", obj);
lock (callback) callback(uri, fileName, obj);
return;
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
se.RunPass(ref request);
// Temporary Assignments
if (uri.Contains("hitomi.la"))
{
request.Referer = $"https://hitomi.la/galleries/{uri.Split('/')[5]}.html";
}
request.Timeout = timeout_infinite ? Timeout.Infinite : timeout_ms;
request.KeepAlive = true;
request.Proxy = proxy;
lock (requests) requests.Add(new Tuple <string, HttpWebRequest>(uri, request));
try
{
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
Monitor.Instance.Push($"404 Not Found {uri}");
}
else if (response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
{
using (Stream inputStream = response.GetResponseStream())
using (Stream outputStream = File.OpenWrite(fileName))
{
byte[] buffer = new byte[buffer_size];
int bytesRead;
if (!lock_donwload_size)
{
lock (download_callback)
download_callback(uri, response.ContentLength, obj);
}
lock_donwload_size = true;
do
{
bytesRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, bytesRead);
lock (status_callback) status_callback(uri, bytesRead, obj);
shutdown_lock_taken = false;
shutdown_lock.Enter(ref shutdown_lock_taken);
if (shutdown)
{
shutdown_lock.Exit(); break;
}
shutdown_lock.Exit();
if (preempt_take)
{
Monitor.Instance.Push($"[Preempt Queue] {uri}");
while (preempt_take)
{
Thread.Sleep(500);
}
Monitor.Instance.Push($"[Exit Preempt] {uri}");
}
} while (bytesRead != 0);
}
shutdown_lock_taken = false;
shutdown_lock.Enter(ref shutdown_lock_taken);
if (shutdown)
{
shutdown_lock.Exit();
File.Delete(fileName);
Monitor.Instance.Push($"[Shutdown] {uri}");
return;
}
shutdown_lock.Exit();
}
}
}
catch (Exception e)
{
if (e is WebException we)
{
HttpWebResponse webResponse = (HttpWebResponse)we.Response;
if (webResponse != null && webResponse.StatusCode == HttpStatusCode.NotFound)
{
Monitor.Instance.Push($"[Emilia Queue] 404 error {uri}");
lock (err_callback) err_callback(uri, "[Emilia Queue] 404 error. auto deleted in download queue.", obj);
goto END;
}
}
Monitor.Instance.Push($"[{retry_count}] {e.Message}");
lock (aborted)
if (!aborted.Contains(uri))
{
lock (retry_callback) retry_callback(uri, obj);
request.Abort();
Thread.Sleep(1000);
retry_count++;
goto RETRY;
}
else
{
File.Delete(fileName);
lock (callback) callback(uri, fileName, obj);
return;
}
}
END:
lock (callback) callback(uri, fileName, obj);
}
}
/// <summary>
/// 새로운 작업을 큐에 추가합니다.
/// </summary>
/// <param name="url"></param>
/// <param name="path"></param>
/// <param name="obj"></param>
/// <param name="callback"></param>
/// <param name="se"></param>
public void Add(string url, string path, object obj, SemaphoreCallBack callback, SemaphoreExtends se = null)
{
lock (queue) queue.Add(new Tuple <string, string, object, SemaphoreCallBack, SemaphoreExtends>(url, path, obj, callback, se));
lock (notify_lock) Notify();
}
private void DownloadRemoteImageFile(string uri, string fileName, object obj, SemaphoreCallBack callback, SemaphoreExtends se)
{
int retry_count = 0;
RETRY:
if (retry_count > Settings.Instance.Net.RetryCount)
{
Monitor.Instance.Push($"[Many Retry] {uri} is auto deleted in download queue.");
return;
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
se.RunPass(ref request);
request.Timeout = timeout_infinite ? Timeout.Infinite : timeout_ms;
request.KeepAlive = true;
request.Proxy = proxy;
lock (requests) requests.Add(new Tuple <string, HttpWebRequest>(uri, request));
try
{
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
Monitor.Instance.Push($"404 Not Found {uri}");
}
else if (response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
{
using (Stream inputStream = response.GetResponseStream())
using (Stream outputStream = File.OpenWrite(fileName))
{
byte[] buffer = new byte[buffer_size];
int bytesRead;
lock (download_callback) download_callback(uri, response.ContentLength, obj);
do
{
bytesRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, bytesRead);
lock (status_callback) status_callback(uri, bytesRead, obj);
lock (shutdown_lock) if (shutdown)
{
break;
}
if (preempt_take)
{
Monitor.Instance.Push($"[Preempt Queue] {uri}");
while (preempt_take)
{
Thread.Sleep(500);
}
Monitor.Instance.Push($"[Exit Preempt] {uri}");
}
} while (bytesRead != 0);
}
lock (shutdown_lock) if (shutdown)
{
File.Delete(fileName);
Monitor.Instance.Push($"[Shutdown] {uri}");
return;
}
}
}
}
catch (Exception e)
{
Monitor.Instance.Push($"[{retry_count}] {e.Message}");
lock (aborted)
if (!aborted.Contains(uri))
{
lock (retry_callback) retry_callback(uri, obj);
request.Abort();
Thread.Sleep(1000);
retry_count++;
goto RETRY;
}
else
{
File.Delete(fileName);
lock (callback) callback(uri, fileName, obj);
return;
}
}
lock (callback) callback(uri, fileName, obj);
lock (queue)
{
int at = 0;
for (; at < queue.Count; at++)
{
if (queue[at].Item1 == uri && queue[at].Item2 == fileName)
{
break;
}
}
if (at != queue.Count)
{
queue.RemoveAt(at);
}
}
Interlocked.Decrement(ref mtx);
lock (notify_lock) Notify();
}
/// <summary>
/// 새로운 작업을 큐에 추가합니다.
/// </summary>
/// <param name="url"></param>
/// <param name="path"></param>
/// <param name="obj"></param>
/// <param name="callback"></param>
/// <param name="se"></param>
public void Add(string url, string path, object obj, SemaphoreCallBack callback, SemaphoreExtends se = null)
{
queue.Add(new Tuple <string, string, object, SemaphoreCallBack, SemaphoreExtends>(url, path, obj, callback, se));
if (Wait())
{
Notify();
}
}
/// <summary>
/// 새 작업을 추가합니다.
/// </summary>
/// <param name="urls">다운로드할 파일의 URL입니다.</param>
/// <param name="paths">다운로드 경로를 지정합니다.</param>
/// <param name="obj">callback에서 전달될 객체입니다.</param>
/// <param name="callback">파일의 다운로드가 끝나면 이 함수가 호출됩니다.</param>
/// <param name="se">리퀘스트에 추가할 추가 옵션입니다.</param>
/// <param name="size_callback">리퀘스트 응답을 성공적으로 받을 시 파일의 크기가 전달됩니다.</param>
/// <param name="status_callback">파일의 바이트 블록(131,072 바이트)이나 맨 마지막 바이트 블록을 전달받으면 이 함수가 호출됩니다.</param>
/// <param name="retry_callback">리퀘스트 도중 응답이 끊기거나, 정의되지 않은 오류로인해 다운로드가 취소되어 파일을 재다운로드할 경우 이 함수가 호출됩니다.</param>
public void Add(string[] urls, string[] paths, object obj, SemaphoreCallBack callback, SemaphoreExtends se,
DownloadQueue.DownloadSizeCallBack size_callback = null, DownloadQueue.DownloadStatusCallBack status_callback = null, DownloadQueue.RetryCallBack retry_callback = null)
{
lock (job_lock)
{
jobs.Add(new Tuple <int, object, SemaphoreCallBack, DownloadQueue.DownloadSizeCallBack, DownloadQueue.DownloadStatusCallBack, DownloadQueue.RetryCallBack>(
index_count, obj, callback,
size_callback, status_callback, retry_callback));
download_file_count.Add(0);
file_count.Add(urls.Length);
}
lock (add_lock)
{
for (int i = 0; i < urls.Length; i++)
{
queue.Add(urls[i], paths[i], index_count, downloadCallback, se);
}
index_count++;
remain_contents += urls.Length;
}
}
private void StartFirstDownloads()
{
DispatchInformation dispatch_info = new DispatchInformation();
dispatch_info.DownloadSize = DownloadSize;
dispatch_info.DownloadStatus = DownloadStatus;
dispatch_info.DownloadRetry = DownloadRetry;
dispatch_info.CompleteFile = CompleteFile;
dispatch_info.CompleteArticle = CompleteArticle;
dispatch_info.CompleteSeries = CompleteSeries;
dispatch_info.ErrorOccured = ErrorOcurred;
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
CollectStatusPanel.Visibility = Visibility.Visible;
DownloadState.Text = $"수집 중";
}));
switch (manager.EngineType)
{
case ManagerEngineType.None:
{
//
// Collect 시작
//
int file_count = 0;
if (manager.Type == ManagerType.SingleArticleMultipleImages)
{
article.ImagesLink = manager.ParseImages(NetCommon.DownloadString(article.Archive), article);
}
else if (manager.Type == ManagerType.SingleSeriesMultipleArticles)
{
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
ProgressText.Text = $"가져오는 중... [0/{series.Articles.Count}]";
}));
var result = EmiliaJob.Instance.AddJob(series.Archive.ToList(),
(count) =>
{
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
ProgressText.Text = $"가져오는 중...[{count}/{series.Articles.Count}]";
}));
});
for (int i = 0; i < series.Articles.Count; i++)
{
series.Articles[i].ImagesLink = manager.ParseImages(result[i], series.Articles[i]);
file_count += series.Articles[i].ImagesLink.Count;
int k = i;
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
//ProgressText.Text = $"가져오는 중... [{i}/{series.Articles.Count}] (파일 {file_count}개)";
if (k == 0 && string.IsNullOrEmpty(series.Thumbnail))
{
LoadThumbnail(thumbnail = series.Articles[0].ImagesLink[0]);
}
}));
}
}
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
CollectStatusPanel.Visibility = Visibility.Collapsed;
DownloadStatusPanel.Visibility = Visibility.Visible;
Progress.Maximum = file_count;
ProgressStatus.Text = $"[0/{file_count}]";
}));
//
// 다운로드 시작
//
EmiliaSeriesSegment series_seg = new EmiliaSeriesSegment();
series_seg.Index = EmiliaDispatcher.Instance.GetSeriesIndex();
series_seg.Title = series.Title;
download_folder = series_seg.Path = Path.Combine(Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), manager.Name.Trim()), DeleteInvalid(series.Title));
List <EmiliaArticleSegment> article_segs = new List <EmiliaArticleSegment>();
HashSet <string> folder_names = new HashSet <string>();
int ov = 0;
for (int i = 0; i < series.Articles.Count; i++)
{
EmiliaArticleSegment article_seg = new EmiliaArticleSegment();
article_seg.Index = i;
article_seg.Name = series.Articles[i].Title;
string folder_name = DeleteInvalid(series.Articles[i].Title).Trim();
if (!folder_names.Contains(folder_name))
{
article_seg.FolderName = DeleteInvalid(series.Articles[i].Title).Trim();
}
else
{
article_seg.FolderName = DeleteInvalid(series.Articles[i].Title).Trim() + $" [OV{++ov}]";
}
folder_names.Add(article_seg.FolderName);
article_seg.SereisIndex = series_seg.Index;
Directory.CreateDirectory(Path.Combine(series_seg.Path, article_seg.FolderName));
List <EmiliaFileSegment> file_segs = new List <EmiliaFileSegment>();
List <string> file_names = manager.GetDownloadFileNames(series.Articles[i]);
if (!Settings.Instance.Model.DownloadWithRawFileName)
{
file_names = PathFilenameSorter.SortOnlyFilename(file_names.ToArray()).ToList();
}
for (int j = 0; j < series.Articles[i].ImagesLink.Count; j++)
{
EmiliaFileSegment file_seg = new EmiliaFileSegment();
file_seg.Index = j;
file_seg.ArticleIndex = i;
file_seg.SeriesIndex = series_seg.Index;
file_seg.FileName = file_names[j];
file_seg.Url = series.Articles[i].ImagesLink[j];
SemaphoreExtends se = SemaphoreExtends.MakeDefault();
se.Referer = url;
file_seg.Extends = se;
file_segs.Add(file_seg);
}
article_seg.Files = file_segs;
article_segs.Add(article_seg);
}
series_seg.Articles = article_segs;
EmiliaDispatcher.Instance.Add(series_seg, dispatch_info);
}
break;
case ManagerEngineType.UsingDriver:
{
int file_count = 0;
if (manager.Type == ManagerType.SingleArticleMultipleImages)
{
wrapper.Navigate(article.Archive);
try { wrapper.ClickXPath("//a[@class='maia-button maia-button-primary']"); } catch { }
article.ImagesLink = manager.ParseImages(wrapper.GetHtml(), article);
}
else if (manager.Type == ManagerType.SingleSeriesMultipleArticles)
{
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
ProgressText.Text = $"가져오는 중... [0/{series.Articles.Count}]";
}));
for (int i = 0; i < series.Articles.Count; i++)
{
wrapper.Navigate(series.Archive[i]);
try { wrapper.ClickXPath("//a[@class='maia-button maia-button-primary']"); } catch { }
series.Articles[i].ImagesLink = manager.ParseImages(wrapper.GetHtml(), series.Articles[i]);
file_count += series.Articles[i].ImagesLink.Count;
int k = i;
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
ProgressText.Text = $"가져오는 중... [{i}/{series.Articles.Count}] (파일 {file_count}개)";
if (k == 0 && string.IsNullOrEmpty(series.Thumbnail))
{
LoadThumbnail(thumbnail = series.Articles[0].ImagesLink[0]);
}
}));
}
}
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
CollectStatusPanel.Visibility = Visibility.Collapsed;
DownloadStatusPanel.Visibility = Visibility.Visible;
Progress.Maximum = file_count;
ProgressStatus.Text = $"[0/{file_count}]";
}));
//
// 다운로드 시작
//
EmiliaSeriesSegment series_seg = new EmiliaSeriesSegment();
series_seg.Index = EmiliaDispatcher.Instance.GetSeriesIndex();
series_seg.Title = series.Title;
download_folder = series_seg.Path = Path.Combine(Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), manager.Name.Trim()), DeleteInvalid(series.Title));
List <EmiliaArticleSegment> article_segs = new List <EmiliaArticleSegment>();
for (int i = 0; i < series.Articles.Count; i++)
{
EmiliaArticleSegment article_seg = new EmiliaArticleSegment();
article_seg.Index = i;
article_seg.Name = series.Articles[i].Title;
article_seg.FolderName = DeleteInvalid(series.Articles[i].Title).Trim();
article_seg.SereisIndex = series_seg.Index;
Directory.CreateDirectory(Path.Combine(series_seg.Path, article_seg.FolderName));
List <EmiliaFileSegment> file_segs = new List <EmiliaFileSegment>();
List <string> file_names = manager.GetDownloadFileNames(series.Articles[i]);
if (!Settings.Instance.Model.DownloadWithRawFileName)
{
file_names = PathFilenameSorter.SortOnlyFilename(file_names.ToArray()).ToList();
}
for (int j = 0; j < series.Articles[i].ImagesLink.Count; j++)
{
EmiliaFileSegment file_seg = new EmiliaFileSegment();
file_seg.Index = j;
file_seg.ArticleIndex = i;
file_seg.SeriesIndex = series_seg.Index;
file_seg.FileName = file_names[j];
file_seg.Url = series.Articles[i].ImagesLink[j];
SemaphoreExtends se = SemaphoreExtends.MakeDefault();
se.Referer = url;
file_seg.Extends = se;
file_segs.Add(file_seg);
}
article_seg.Files = file_segs;
article_segs.Add(article_seg);
}
series_seg.Articles = article_segs;
EmiliaDispatcher.Instance.Add(series_seg, dispatch_info);
}
break;
}
Application.Current.Dispatcher.BeginInvoke(new Action(
delegate
{
DownloadState.Text = $"대기 중";
}));
}
public void Add(string url, string path, object obj, SemaphoreCallBack callback, SemaphoreExtends se = null)
{
}
