//刷新url的回调函数
private void _main_url_refresh_callback(bool suc, string[] urls, object state)
{
if (suc && urls != null && urls.Length > 0)
{
lock (_url_lock)
{
//成功，并且url数大于0时，覆盖原url，将url的有效时长设为半个小时
_urls = urls;
_url_expire_time = DateTime.Now.AddHours(0.5);
}
}
else
{
//失败，增加url的时长为30s（避免多次触发url刷新），然后在3s后重试，增加重试的失败次数
_url_expire_time = DateTime.Now.AddSeconds(30);
Thread.Sleep(3000);
_url_fail_to_fetch_count++;
if (_url_fail_to_fetch_count < _MAX_URL_FAIL_COUNT && (_download_thread_flag & _DOWNLOAD_THREAD_FLAG_STARTED) != 0)
{
_api.GetAccount(_data.AccountID).GetLocateDownloadLinkAsync(_data.Path, _main_url_refresh_callback);
}
else
{
//重试失败次数多于最大阈值时，增加error的标志，并且暂停该任务（跨线程暂停）
if (_url_fail_to_fetch_count >= _MAX_URL_FAIL_COUNT)
{
lock (_thread_flag_lock)
_download_thread_flag |= _DOWNLOAD_THREAD_FLAG_ERROR;
Pause();
try { TaskError?.Invoke(this, new EventArgs()); }
catch { }
}
}
}
}
/// <summary>
/// Handles the tasks in queue.
/// </summary>
private void HandleTasksInQueue()
{
bool isDebugEnabled = Log.IsDebugEnabled;
Log.Debug(
"HandleTasksInQueue: Instance {0} thread {1} starting with {2}",
_id,
Thread.CurrentThread.Name,
_taskQueue.GetType().Name);
using (ScopedInstance <IBlockingQueue <Runnable> > .Set(_taskQueue)) // introduces the queue into scope
{
while (_liveMode != LiveMode.STOPPED)
{
Runnable task;
Interlocked.Increment(ref _tasksRunning);
try {
if (_taskQueue.Pop(500, out task))
{
try {
task.Invoke();
}
catch (Exception e) {
Log.Warn("HandleTasksInQueue: Instance {0} finished with abnormal termination", _id, e);
TaskError?.Invoke(this, new ThreadExceptionEventArgs(e));
}
finally {
Interlocked.Increment(ref _numExecuted);
}
}
else if (_liveMode == LiveMode.STOPPING)
{
if (isDebugEnabled)
{
Log.Debug(
"HandleTasksInQueue: Instance {0} no items detected in queue, terminating",
_id);
}
break;
}
else if (isDebugEnabled)
{
Log.Debug(
"HandleTasksInQueue: Instance {0} no items detected in queue, start loop again",
_id);
}
}
finally {
Interlocked.Decrement(ref _tasksRunning);
}
}
}
Log.Debug("HandleTasksInQueue: Instance {0} thread ending", _id);
}
public virtual StatusCode OnError()
{
TaskError?.Invoke(this, new EventArgs());
Deactivate();
return(StatusCode.SUCCEED_STATUS);
}
protected void OnTaskError(StringValueEventArgs e)
{
TaskError?.Invoke(this, e);
}
private void _on_task_error(object sender, EventArgs e)
{
try { TaskError?.Invoke(sender, e); }
catch { }
}
protected virtual void OnTaskError(TaskQueueErrorEventArgs eventargs)
{
TaskError?.Invoke(eventargs);
}
private void _monitor_thread_callback(object _ = null)
{
_monitor_thread_created.Set();
_start_time = DateTime.Now;
try
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_STARTED) & ~(_UPLOAD_THREAD_FLAG_START_REQUESTED | _UPLOAD_THREAD_FLAG_PAUSED);
//local io
#region file io
if (string.IsNullOrEmpty(_local_data.Path))
{
_local_cacher.FileIORequest(_local_path);
_upload_thread_flag |= _UPLOAD_THREAD_FLAG_DIGEST_REQUESTED;
try { FileDigestStarted?.Invoke(this, new EventArgs()); } catch { }
_file_io_response.Wait();
_file_io_response.Reset();
try { FileDigestFinished?.Invoke(this, new EventArgs()); } catch { }
}
_uploaded_size = 0;
#endregion
#region status check
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_CANCEL_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_CANCELLED) & ~(_UPLOAD_THREAD_FLAG_CANCEL_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_PAUSE_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_PAUSED) & ~(_UPLOAD_THREAD_FLAG_PAUSE_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_ERROR) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_PAUSED) & ~_UPLOAD_THREAD_FLAG_STARTED;
_end_time = DateTime.Now;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
return;
}
#endregion
#region encryption
if (_enable_encryption)
{
_upload_thread_flag |= _UPLOAD_THREAD_FLAG_FILE_ENCRYPTING;
try { EncryptStarted?.Invoke(this, new EventArgs()); } catch { }
_file_encrypt();
_upload_thread_flag = _upload_thread_flag & ~_UPLOAD_THREAD_FLAG_FILE_ENCRYPTING;
try { EncryptFinished?.Invoke(this, new EventArgs()); } catch { }
_uploaded_size = 0;
//handling IO
_local_cacher.FileIORequest(_local_path);
_upload_thread_flag |= _UPLOAD_THREAD_FLAG_DIGEST_REQUESTED;
try { EncryptFileDigestStarted?.Invoke(this, new EventArgs()); } catch { }
_file_io_response.Wait();
_file_io_response.Reset();
try { EncryptFileDigestFinished?.Invoke(this, new EventArgs()); } catch { }
//handling file slice data
_uploaded_size = 0;
_file_size = _local_data.Size;
_slice_count = (int)Math.Ceiling(_file_size * 1.0 / BaiduPCS.UPLOAD_SLICE_SIZE);
}
#endregion
//status check
#region status check
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_CANCEL_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_CANCELLED) & ~(_UPLOAD_THREAD_FLAG_CANCEL_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_PAUSE_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_PAUSED) & ~(_UPLOAD_THREAD_FLAG_PAUSE_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_ERROR) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_PAUSED) & ~_UPLOAD_THREAD_FLAG_STARTED;
_end_time = DateTime.Now;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
return;
}
#endregion
//rapid upload test
var sync_lock = new ManualResetEventSlim();
var rapid_param = _local_data;
bool rapid_upload_suc = false;
if (_enable_rapid_upload)
{
_remote_cacher.RapidUploadAsync(_remote_path, (ulong)rapid_param.Size, rapid_param.MD5, rapid_param.CRC32.ToString("X2").ToLower(), rapid_param.Slice_MD5, (suc, data, e) =>
{
rapid_upload_suc = suc;
_remote_data = data;
sync_lock.Set();
}, _overwrite ? BaiduPCS.ondup.overwrite : BaiduPCS.ondup.newcopy, _selected_account_id);
sync_lock.Wait(120000);
sync_lock.Reset();
}
if (rapid_upload_suc == false)
{
//deleting existed file if overwrite is true
if (_overwrite)
{
var temp_struct = new _temp_struct {
lck = sync_lock, suc = false
};
for (int i = 0; i < 10; i++)
{
_remote_cacher.DeletePathAsync(_remote_path, _delete_callback, _selected_account_id, temp_struct);
sync_lock.Wait(60000);
sync_lock.Reset();
if (temp_struct.suc)
{
break;
}
}
if (temp_struct.suc == false)
{
Tracer.GlobalTracer.TraceWarning("delete file " + _remote_path + " failed, using newcopy instead (reached max. retry times)");
}
}
//pre create file request
if (string.IsNullOrEmpty(_upload_id))
{
var temp_struct = new _temp_struct {
lck = sync_lock, suc = false
};
for (int i = 0; i < 3; i++)
{
_remote_cacher.PreCreateFileAsync(_remote_path, _slice_count, _pre_create_request_callback, _selected_account_id, temp_struct);
sync_lock.Wait(120000);
sync_lock.Reset();
if (temp_struct.suc)
{
break;
}
}
if (temp_struct.suc == false)
{
//precreate failed
_upload_thread_flag |= _UPLOAD_THREAD_FLAG_ERROR;
_end_time = DateTime.Now;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
return;
}
#region status check
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_CANCEL_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_CANCELLED) & ~(_UPLOAD_THREAD_FLAG_CANCEL_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_PAUSE_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_PAUSED) & ~(_UPLOAD_THREAD_FLAG_PAUSE_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
_end_time = DateTime.Now;
return;
}
#endregion
}
//initializing multi thread data
var max_thread = _max_thread;
_task_id = new Guid[max_thread];
_task_seq = new int[max_thread];
_last_sent = new DateTime[max_thread];
//adding slice sequence
_slice_seq = new ConcurrentQueue <int>();
for (int i = 0; i < _slice_count; i++)
{
if (_slice_result.ContainsKey(i) == false)
{
_slice_seq.Enqueue(i);
}
}
//upload start, multi thread
var next_time = DateTime.Now.AddSeconds(1);
#region loop
while (true)
{
//handling finish state
if (_slice_result.Count == _slice_count)
{
break;
}
//handling other state
#region status check
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_CANCEL_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_CANCELLED) & ~(_UPLOAD_THREAD_FLAG_CANCEL_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
for (int i = 0; i < _task_id.Length; i++)
{
if (_task_id[i] != Guid.Empty)
{
_remote_cacher.UploadSliceCancelAsync(_task_id[i]);
}
}
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & _UPLOAD_THREAD_FLAG_PAUSE_REQUESTED) != 0)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_PAUSED) & ~(_UPLOAD_THREAD_FLAG_PAUSE_REQUESTED | _UPLOAD_THREAD_FLAG_STARTED);
for (int i = 0; i < _task_id.Length; i++)
{
if (_task_id[i] != Guid.Empty)
{
_remote_cacher.UploadSliceCancelAsync(_task_id[i]);
}
}
_end_time = DateTime.Now;
return;
}
if ((_upload_thread_flag & (_UPLOAD_THREAD_FLAG_ERROR | _UPLOAD_THREAD_FLAG_FILE_MODIFIED)) != 0)
{
_upload_thread_flag = _upload_thread_flag & ~_UPLOAD_THREAD_FLAG_STARTED;
_end_time = DateTime.Now;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
return;
}
#endregion
lock (_thread_data_lock)
{
for (int i = 0; i < max_thread; i++)
{
if ((DateTime.Now - _last_sent[i]).TotalSeconds > 120)
{
if (_task_id[i] != Guid.Empty)
{
_last_sent[i] = DateTime.Now;
_remote_cacher.UploadSliceCancelAsync(_task_id[i], _selected_account_id);
_task_id[i] = Guid.Empty;
continue;
}
if (_slice_seq.Count == 0)
{
continue;
}
//error handling for dequeue failure
if (_slice_seq.TryDequeue(out _task_seq[i]) == false)
{
_upload_thread_flag |= _UPLOAD_THREAD_FLAG_ERROR;
_end_time = DateTime.Now;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
return;
}
try
{
_remote_cacher.UploadSliceBeginAsync((ulong)Math.Min(_file_size - BaiduPCS.UPLOAD_SLICE_SIZE * (long)_task_seq[i], BaiduPCS.UPLOAD_SLICE_SIZE), _remote_path, _upload_id, _task_seq[i], _on_slice_upload_request_callback, _selected_account_id, i);
}
catch { _slice_seq.Enqueue(_task_seq[i]); }
_last_sent[i] = DateTime.Now;
}
}
}
//speed calculation
long size = Interlocked.Read(ref _uploaded_size);
_upload_size_5s.RemoveFirst();
_upload_size_5s.AddLast(size);
_average_speed_5s = (_upload_size_5s.Last.Value - _upload_size_5s.First.Value) / 5.0;
_average_speed_total = size / (DateTime.Now - _start_time).TotalSeconds;
_current_bytes = 0;
Tracer.GlobalTracer.TraceInfo("Uploaded " + _uploaded_size + "/" + _file_size + " [" + (_average_speed_5s / 1024.0).ToString("0.00") + "KB/s]");
//monitor loop
var ts = next_time - DateTime.Now;
next_time = next_time.AddSeconds(1);
if (ts.TotalMilliseconds > 1)
{
Thread.Sleep((int)ts.TotalMilliseconds);
}
}
#endregion
//merging slice request
var temp_struct1 = new _temp_struct {
lck = sync_lock, suc = false
};
for (int i = 0; i < 3; i++)
{
_remote_cacher.CreteSuperFileAsync(_remote_path, _upload_id, from item in _slice_result orderby item.Key ascending select item.Value, (ulong)_file_size, _create_superfile_request_callback, _selected_account_id, temp_struct1);
sync_lock.Wait(120000);
sync_lock.Reset();
if (temp_struct1.suc)
{
break;
}
}
if (temp_struct1.suc)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_FINISHED) & ~_UPLOAD_THREAD_FLAG_STARTED;
try { TaskFinished?.Invoke(this, new EventArgs()); } catch { }
}
else
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_ERROR | _UPLOAD_THREAD_FLAG_PAUSED) & ~_UPLOAD_THREAD_FLAG_STARTED;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
}
_end_time = DateTime.Now;
}
else
{
//rapid upload succeeded
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_FINISHED) & ~_UPLOAD_THREAD_FLAG_STARTED;
_end_time = DateTime.Now;
try { TaskFinished?.Invoke(this, new EventArgs()); } catch { }
}
}
catch (Exception)
{
_upload_thread_flag = (_upload_thread_flag | _UPLOAD_THREAD_FLAG_ERROR | _UPLOAD_THREAD_FLAG_PAUSED) & ~_UPLOAD_THREAD_FLAG_STARTED;
_end_time = DateTime.Now;
try { TaskError?.Invoke(this, new EventArgs()); } catch { }
}
finally
{
_file_watcher.EnableRaisingEvents = false;
_file_watcher.Dispose();
_file_watcher = null;
//deleting temporary encrypted file
if (_enable_encryption && File.Exists(_local_path) && _local_path.EndsWith(".encrypted"))
{
File.Delete(_local_path);
}
_monitor_thread_exited.Set();
}
}
protected virtual void OnTaskError(AutomaticCancellationTaskResult <T> e, Exception innerException)
{
TaskError?.Invoke(this, new TaskErrorEventArgs <T>(e, innerException));
}