/// <summary>
/// 새로운 작업을 큐에 추가합니다.
/// </summary>
/// <param name="url"></param>
/// <param name="path"></param>
/// <param name="obj"></param>
/// <param name="callback"></param>
/// <param name="se"></param>
public void Add(string url, string path, object obj, SemaphoreCallBack callback, SemaphoreExtends se = null)
{
queue.Add(new Tuple <string, string, object, SemaphoreCallBack, SemaphoreExtends>(url, path, obj, callback, se));
if (Wait())
{
Notify();
}
}
private void Notify()
{
int i = mtx;
if (queue.Count > i)
{
string s1 = queue[i].Item1;
string s2 = queue[i].Item2;
object s3 = queue[i].Item3;
SemaphoreCallBack s4 = queue[i].Item4;
SemaphoreExtends s5 = queue[i].Item5;
lock (task_lock)
tasks.Add(Task.Run(() => DownloadRemoteImageFile(s1, s2, s3, s4, s5)).ContinueWith(
x => Task.Run(() => { lock (task_lock) tasks.RemoveAll(y => y.IsCompleted); })));
Interlocked.Increment(ref mtx);
}
}
private void remote_download_thread_handler(object i)
{
int index = (int)i;
while (true)
{
interrupt[index].WaitOne();
Tuple <string, int, object, SemaphoreCallBack> job;
lock (queue)
{
if (queue.Count > 0)
{
job = queue[0];
queue.RemoveAt(0);
}
else
{
interrupt[index].Reset();
continue;
}
}
string uri = job.Item1;
int job_count = job.Item2;
object obj = job.Item3;
SemaphoreCallBack callback = job.Item4;
try
{
lock (callback) callback(uri, NetCommon.DownloadExHentaiString(uri), new Tuple <int, object> (job_count, obj));
}
catch (Exception e)
{
Monitor.Instance.Push($"[Emilia Job] {uri} {e.Message}");
lock (callback) callback(uri, "", obj);
}
}
}
/// <summary>
/// 새 작업을 추가합니다.
/// </summary>
/// <param name="urls">다운로드할 파일의 URL입니다.</param>
/// <param name="paths">다운로드 경로를 지정합니다.</param>
/// <param name="obj">callback에서 전달될 객체입니다.</param>
/// <param name="callback">파일의 다운로드가 끝나면 이 함수가 호출됩니다.</param>
/// <param name="se">리퀘스트에 추가할 추가 옵션입니다.</param>
/// <param name="size_callback">리퀘스트 응답을 성공적으로 받을 시 파일의 크기가 전달됩니다.</param>
/// <param name="status_callback">파일의 바이트 블록(131,072 바이트)이나 맨 마지막 바이트 블록을 전달받으면 이 함수가 호출됩니다.</param>
/// <param name="retry_callback">리퀘스트 도중 응답이 끊기거나, 정의되지 않은 오류로인해 다운로드가 취소되어 파일을 재다운로드할 경우 이 함수가 호출됩니다.</param>
public void Add(string[] urls, string[] paths, object obj, SemaphoreCallBack callback, SemaphoreExtends se,
DownloadQueue.DownloadSizeCallBack size_callback = null, DownloadQueue.DownloadStatusCallBack status_callback = null, DownloadQueue.RetryCallBack retry_callback = null)
{
lock (job_lock)
{
jobs.Add(new Tuple <int, object, SemaphoreCallBack, DownloadQueue.DownloadSizeCallBack, DownloadQueue.DownloadStatusCallBack, DownloadQueue.RetryCallBack>(
index_count, obj, callback,
size_callback, status_callback, retry_callback));
download_file_count.Add(0);
file_count.Add(urls.Length);
}
lock (add_lock)
{
for (int i = 0; i < urls.Length; i++)
{
queue.Add(urls[i], paths[i], index_count, downloadCallback, se);
}
index_count++;
remain_contents += urls.Length;
}
}
private void remote_download_thread_handler(object i)
{
int index = (int)i;
//Monitor.Instance.Push($"[Emilia Queue] Starts download thread [{i}]");
while (true)
{
interrupt[index].WaitOne();
Tuple <string, string, object, SemaphoreCallBack, SemaphoreExtends> job;
lock (queue)
{
if (queue.Count > 0)
{
job = queue[0];
queue.RemoveAt(0);
}
else
{
//Monitor.Instance.Push($"[Emilia Queue] Suspends download thread [{i}]");
interrupt[index].Reset();
continue;
}
}
string uri = job.Item1;
string fileName = job.Item2;
object obj = job.Item3;
SemaphoreCallBack callback = job.Item4;
SemaphoreExtends se = job.Item5;
if (!Directory.Exists(Path.GetDirectoryName(fileName)))
{
Monitor.Instance.Push($"[Directory Not Found] {uri} is auto deleted in download queue.");
goto END;
}
int retry_count = 0;
bool lock_donwload_size = false;
RETRY:
if (retry_count > Settings.Instance.Net.RetryCount)
{
Monitor.Instance.Push($"[Many Retry] {uri} is auto deleted in download queue.");
lock (err_callback) err_callback(uri, "[Emilia Queue] Many retry. auto deleted in download queue.", obj);
lock (callback) callback(uri, fileName, obj);
return;
}
if (!uri.StartsWith("http"))
{
Monitor.Instance.Push($"[Url Error] {uri} is not corret url");
lock (err_callback) err_callback(uri, "[Emilia Queue] Url Error. not corret url.", obj);
lock (callback) callback(uri, fileName, obj);
return;
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
se.RunPass(ref request);
// Temporary Assignments
if (uri.Contains("hitomi.la"))
{
request.Referer = $"https://hitomi.la/galleries/{uri.Split('/')[5]}.html";
}
request.Timeout = timeout_infinite ? Timeout.Infinite : timeout_ms;
request.KeepAlive = true;
request.Proxy = proxy;
lock (requests) requests.Add(new Tuple <string, HttpWebRequest>(uri, request));
try
{
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
Monitor.Instance.Push($"404 Not Found {uri}");
}
else if (response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
{
using (Stream inputStream = response.GetResponseStream())
using (Stream outputStream = File.OpenWrite(fileName))
{
byte[] buffer = new byte[buffer_size];
int bytesRead;
if (!lock_donwload_size)
{
lock (download_callback)
download_callback(uri, response.ContentLength, obj);
}
lock_donwload_size = true;
do
{
bytesRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, bytesRead);
lock (status_callback) status_callback(uri, bytesRead, obj);
shutdown_lock_taken = false;
shutdown_lock.Enter(ref shutdown_lock_taken);
if (shutdown)
{
shutdown_lock.Exit(); break;
}
shutdown_lock.Exit();
if (preempt_take)
{
Monitor.Instance.Push($"[Preempt Queue] {uri}");
while (preempt_take)
{
Thread.Sleep(500);
}
Monitor.Instance.Push($"[Exit Preempt] {uri}");
}
} while (bytesRead != 0);
}
shutdown_lock_taken = false;
shutdown_lock.Enter(ref shutdown_lock_taken);
if (shutdown)
{
shutdown_lock.Exit();
File.Delete(fileName);
Monitor.Instance.Push($"[Shutdown] {uri}");
return;
}
shutdown_lock.Exit();
}
}
}
catch (Exception e)
{
if (e is WebException we)
{
HttpWebResponse webResponse = (HttpWebResponse)we.Response;
if (webResponse != null && webResponse.StatusCode == HttpStatusCode.NotFound)
{
Monitor.Instance.Push($"[Emilia Queue] 404 error {uri}");
lock (err_callback) err_callback(uri, "[Emilia Queue] 404 error. auto deleted in download queue.", obj);
goto END;
}
}
Monitor.Instance.Push($"[{retry_count}] {e.Message}");
lock (aborted)
if (!aborted.Contains(uri))
{
lock (retry_callback) retry_callback(uri, obj);
request.Abort();
Thread.Sleep(1000);
retry_count++;
goto RETRY;
}
else
{
File.Delete(fileName);
lock (callback) callback(uri, fileName, obj);
return;
}
}
END:
lock (callback) callback(uri, fileName, obj);
}
}
/// <summary>
/// 새로운 작업을 큐에 추가합니다.
/// </summary>
/// <param name="url"></param>
/// <param name="path"></param>
/// <param name="obj"></param>
/// <param name="callback"></param>
/// <param name="se"></param>
public void Add(string url, string path, object obj, SemaphoreCallBack callback, SemaphoreExtends se = null)
{
lock (queue) queue.Add(new Tuple <string, string, object, SemaphoreCallBack, SemaphoreExtends>(url, path, obj, callback, se));
lock (notify_lock) Notify();
}
private void DownloadRemoteImageFile(string uri, string fileName, object obj, SemaphoreCallBack callback, SemaphoreExtends se)
{
int retry_count = 0;
RETRY:
if (retry_count > Settings.Instance.Net.RetryCount)
{
Monitor.Instance.Push($"[Many Retry] {uri} is auto deleted in download queue.");
return;
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
se.RunPass(ref request);
request.Timeout = timeout_infinite ? Timeout.Infinite : timeout_ms;
request.KeepAlive = true;
request.Proxy = proxy;
lock (requests) requests.Add(new Tuple <string, HttpWebRequest>(uri, request));
try
{
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
Monitor.Instance.Push($"404 Not Found {uri}");
}
else if (response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
{
using (Stream inputStream = response.GetResponseStream())
using (Stream outputStream = File.OpenWrite(fileName))
{
byte[] buffer = new byte[buffer_size];
int bytesRead;
lock (download_callback) download_callback(uri, response.ContentLength, obj);
do
{
bytesRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, bytesRead);
lock (status_callback) status_callback(uri, bytesRead, obj);
lock (shutdown_lock) if (shutdown)
{
break;
}
if (preempt_take)
{
Monitor.Instance.Push($"[Preempt Queue] {uri}");
while (preempt_take)
{
Thread.Sleep(500);
}
Monitor.Instance.Push($"[Exit Preempt] {uri}");
}
} while (bytesRead != 0);
}
lock (shutdown_lock) if (shutdown)
{
File.Delete(fileName);
Monitor.Instance.Push($"[Shutdown] {uri}");
return;
}
}
}
}
catch (Exception e)
{
Monitor.Instance.Push($"[{retry_count}] {e.Message}");
lock (aborted)
if (!aborted.Contains(uri))
{
lock (retry_callback) retry_callback(uri, obj);
request.Abort();
Thread.Sleep(1000);
retry_count++;
goto RETRY;
}
else
{
File.Delete(fileName);
lock (callback) callback(uri, fileName, obj);
return;
}
}
lock (callback) callback(uri, fileName, obj);
lock (queue)
{
int at = 0;
for (; at < queue.Count; at++)
{
if (queue[at].Item1 == uri && queue[at].Item2 == fileName)
{
break;
}
}
if (at != queue.Count)
{
queue.RemoveAt(at);
}
}
Interlocked.Decrement(ref mtx);
lock (notify_lock) Notify();
}
public void Add(string url, string path, object obj, SemaphoreCallBack callback, SemaphoreExtends se = null)
{
}
