private void WebClientOnDownloadFileCompleted(object sender, AsyncCompletedEventArgs asyncCompletedEventArgs)
{
if (asyncCompletedEventArgs.Cancelled)
{
if (Exists())
{
Delete();
}
Cancelled.Handle(h => h(this));
return;
}
if (asyncCompletedEventArgs.Error != null)
{
DownloadFailed.Handle(h => h(this, asyncCompletedEventArgs.Error));
Trace.Write(asyncCompletedEventArgs.Error);
return;
}
if (Exists())
{
Downloaded.Handle(h => h(this));
}
}
private void WebClientOnDownloadFileCompleted(object sender, AsyncCompletedEventArgs asyncCompletedEventArgs)
{
if (asyncCompletedEventArgs.Cancelled)
{
if (Exists())
{
Delete();
}
Cancelled?.Invoke(this, new EventArgs());
return;
}
if (asyncCompletedEventArgs.Error != null)
{
DownloadFailed?.Invoke(this, new DownloadFailEvent(asyncCompletedEventArgs.Error));
AppLogger.Log.Error("Problem downloading file ", asyncCompletedEventArgs.Error);
return;
}
if (Exists())
{
Downloaded?.Invoke(this, new EventArgs());
}
}
public async void Add(DownloadDataModel downloadDataModel, Task <byte[]> callback)
{
downloadDataModel.Id = counter++;
DownloadedDataList.Add(downloadDataModel);
Added?.Invoke(downloadDataModel);
downloadDataModel.Data = await callback;
Downloaded?.Invoke(downloadDataModel);
}
protected void OnDownloaded(UnityWebRequest request)
{
if (IsDownloaded)
{
return;
}
IsDownloaded = true;
OnHeadFetched(request);
Downloaded?.Invoke(this);
}
private static void OnDownloaded(long downloadedbytes, long totalbytes)
{
try
{
Downloaded?.Invoke(downloadedbytes, totalbytes);
}
catch
{
// ignore all
}
}
public async void StartDownload(CancellationToken cancellationToken)
{
List <Task> tasks = new List <Task>();
while (queue.TryDequeue(out Image img))
{
CheckOrCreateDirectory(img.FolderToDownload);
if (!File.Exists(img.FullPath))
{
tasks.Add(DownloadImage(img, cancellationToken));
}
}
await Task.WhenAll(tasks);
Downloaded?.Invoke();
}
private async void MenuItem_Click(object sender, RoutedEventArgs e)
{
var selectedFiles = fileListView.SelectedItems.Cast <FileInfo>();
foreach (var selectedFile in selectedFiles)
{
var filename = selectedFile.UUID + selectedFile.Format;
filename = await HttpService.GetFile(IPAddress, Port, filename, selectedFile.Name);
var tags = mappers
.Where(item => item.FileId == selectedFile.Id)
.Join(this.tags, x => x.TagId, y => y.Id, (x, y) => y)
.ToArray();
Downloaded?.Invoke(this, new DownloadedEventArgs(filename, tags));
}
}
/**
* ダウンロードが失敗したときの処理
*/
private void onDownloadError(Exception error)
{
Error = error;
lock (mLock)
{
mFile = null;
try
{
Downloaded?.Invoke(this, null);
Downloaded = null;
}
catch (Exception e)
{
// コールバック中のエラーは無視する
CmLog.error(e, "WvvCache.onDownloadError (error occurred in downloaded-callback(s)... ignored.)");
}
}
}
public void Download()
{
if (!CanDownload)
{
return;
}
IsDownloaded = false;
_downloadCompletionSource = new TaskCompletionSource <bool>();
ContentProvider.DownloadStream(Id)
.ContinueWith(t =>
{
Content = GetAssetFromStream(t.Result);
_downloadCompletionSource.SetResult(true);
IsDownloaded = true;
Downloaded?.Invoke();
}, TaskContinuationOptions.None);
}
/**
* ダウンロードが成功したときの処理
*/
private void onDownloadCompleted(StorageFile file)
{
Error = null;
lock (mLock)
{
mFile = file;
mInvalidFile = null;
try
{
Downloaded?.Invoke(this, file);
Downloaded = null;
}
catch (Exception e)
{
// コールバック中のエラーは無視する
CmLog.error(e, "WvvCache.onDownloadCompleted (error occurred in downloaded-callback(s)... ignored.)");
}
}
}
private void OnLinkedSourceFetched(string uri, TextSource sourceCode)
{
if (sourceCode == null)
{
return;
}
try
{
var source = sourceCode.Text;
FilterString(ref source);
_source = source;
Downloaded.Fire(uri);
}
catch (Exception e)
{
var msg = $"{GetFullName()}.LinkedSource - error reading data from \"{uri}.\"";
Logger.Warn(msg);
_source = $"An exception occured when reading data from {uri}. \n{e.Message}.\n{e.Source}";
}
}
private async Task <string> DownloadContentAsync(HttpClient client, string url)
{
var e = new ScraperDonwloadEventArgs(client, url);
Downloading?.Invoke(this, e);
if (e.Html != null || e.StatusCode != default(HttpStatusCode))
{
return(e.Html);
}
var res = await client.GetAsync(url);
e.StatusCode = res.StatusCode;
if (res.IsSuccessStatusCode)
{
e.Html = await res.Content.ReadAsStringAsync();
}
Downloaded?.Invoke(this, e);
return(e.Html);
}
private void Downloader(Uri url)
{
var downloadedStream = new SimultaneousStream(SimultaneousStream.PositionMode.FromRead);
using (WebResponse response = ((HttpWebRequest)WebRequest.Create(url)).GetResponse())
using (Stream stream = ((HttpWebResponse)response).GetResponseStream())
{
bool preDownloaded = false;
var buffer = new byte[16 * 1024]; // 16Kb chunks
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
{
if (NeedsToBeStopped)
{
NeedsToBeStopped = false;
return;
}
downloadedStream.Write(buffer, 0, read);
if (downloadedStream.Length > 64 * 1024 && !preDownloaded)
{
preDownloaded = true;
new Thread(() => PreDownloaded?.Invoke(downloadedStream)).Start();
}
}
if (!preDownloaded)
{
preDownloaded = true;
new Thread(() => PreDownloaded?.Invoke(downloadedStream)).Start();
}
Downloaded?.Invoke(downloadedStream);
}
}
private void MediaPlayer_BufferingChanged(object sender, Plugin.MediaManager.Abstractions.EventArguments.BufferingChangedEventArgs e)
{
BufferPosition = Downloaded.ToString();
}
/*
* 交给线程执行的核心爬取控制函数
*/
private void Process()
{
int checkNullCount = 0; //检查发现url队列为空的次数，达到一定次数时结束线程
while (!shutDownFlag) //增加一个强制关闭Flag
{
if (urlsUndisposed.Count != 0)
{
Url task;
lock (this) //从队列中取出任务
{
task = urlsUndisposed.Dequeue();
urlsDisposed.Enqueue(task);
Console.WriteLine("处理URL: " + task.url + "\t深度: " + task.depth);
}
try
{
WebClient webClient = new WebClient(); //获取网页
webClient.Encoding = Encoding.UTF8;
string html = webClient.DownloadString(task.url);
Downloaded?.Invoke(this, new DownloadedEventArgs(html)); //调用下载完成的事件
List <string> newUrls = GetLinks(html); //获取网页中的链接
if (task.depth < maxDepth) //查重后放入队列
{
lock (this)
{
foreach (var url in newUrls)
{
if (!ExistUrl(url))
{
urlsUndisposed.Enqueue(new Url(url, task.depth + 1));
}
}
}
}
Dictionary <string, int> newWordFrequency = GetWordFrequency(html); //获取单词
lock (this) //计算词频
{
foreach (var wf in newWordFrequency)
{
if (wordFrequency.ContainsKey(wf.Key))
{
wordFrequency[wf.Key] += wf.Value;
}
else
{
wordFrequency[wf.Key] = wf.Value;
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("URL处理失败: " + ex.Message);
}
checkNullCount = 0;
}
else
{
checkNullCount += 1;
if (checkNullCount >= maxCheckNullCount)
{
break;
}
else
{
Thread.Sleep(waitTime);
}
}
}
threadCount -= 1;
if (threadCount == 0)
{
Console.WriteLine("所有线程均已结束");
Save();
TaskTerminated?.Invoke(this, new TaskTerminatedEvent()); //当所有进程都结束时，调用任务结束事件
}
}
public override int GetHashCode() => Downloaded.GetHashCode() ^ Total.GetHashCode();
public void GetTrayHistoryDownload()
{
Downloaded.Clear();
string FileDir = Path.Combine(path, AccountInfo.Phone);
string FilePath = Path.Combine(FileDir, "trayHistoryDownload.xml");
if (!Directory.Exists(FileDir))
{
Directory.CreateDirectory(FileDir);
}
if (!File.Exists(FilePath))
{
XmlHelper.CreateXml(FileDir, "trayHistoryDownload.xml");
}
XElement xele = XElement.Load(FilePath);
var item = (from ele in xele.DescendantsAndSelf("File")
select ele).ToList();
EmptyIsShow = item.Count < 1;
if (!EmptyIsShow)
{
XmlDocument document = new XmlDocument();
document.LoadXml(xele.ToString());
string str = JsonConvert.SerializeXmlNode(document);
JObject json = (JObject)JsonConvert.DeserializeObject(str);
Regex reg = new Regex(@"\[[^\[\]]+\]");
MatchCollection Files = reg.Matches(str);
bool FileCount = Files.Count > 0;
int index = 0;
if (FileCount)
{
for (int i = 0; i < json["Root"]["File"].Count(); i++)
{
var cc = json["Root"]["File"][i];
Downloaded.Add(new Model.TrayHistory()
{
Index = index + 1,
CreateDate = Convert.ToDateTime(cc["CreateTime"].ToString()),
FileName = cc["FileName"].ToString(),
FileUrl = cc["FileUrl"].ToString(),
DownloadPath = cc["DownLoadPath"].ToString(),
Id = cc["Id"].ToString()
});
index++;
}
}
else
{
var cc = json["Root"]["File"];
Downloaded.Add(new Model.TrayHistory()
{
Index = index + 1,
CreateDate = Convert.ToDateTime(cc["CreateTime"].ToString()),
FileName = cc["FileName"].ToString(),
FileUrl = cc["FileUrl"].ToString(),
DownloadPath = cc["DownLoadPath"].ToString(),
Id = cc["Id"].ToString()
});
}
//string str = "<Root>";
//for (int i = 0; i < item.Count; i++)
//{
// str += item[i].ToString();
//}
//str += "</Root>";
//string json = XmlHelper.ParseXmlToJson(str);
//JObject jo = ((JObject)JsonConvert.DeserializeObject(json));
//int index = 0;
//MessageBox.Show(jo["Root"]["Item"].ToString());
}
RunState = false;
}
private void MyDownloadButton_Click(object sender, Windows.UI.Xaml.RoutedEventArgs e)
{
Downloaded?.Invoke(this, EventArgs.Empty);
}
public async Task Crawl()
{
List <Task> downloadTask = new List <Task>();
pendingUrls.Enqueue(InitialUrl);
while (pendingUrls.Count > 0)
{
if (!pendingUrls.TryDequeue(out var current))
{
continue;
}
;
if (CompletedUrls.Count + DownloadingUrls.Count > MaxCount)
{
break;
}
//限制并发
if (MaxParallel > 0 && DownloadingUrls.Count >= MaxParallel)
{
await Task.Delay(100);
continue;
}
//下载开始事件
var downEvent = new BeforeDownloadEventArgs()
{
Url = current,
Cancelled = false
};
BeforeDownload?.Invoke(this, downEvent);
if (!downEvent.Cancelled)
{
lock (urlLock)
{
DownloadingUrls.Add(current);
}
var down = DownLoad(current, nameIndex++.ToString())
.ContinueWith(taks =>
{
if (taks.IsFaulted)
{
FailedUrls.Add(current);
lock (urlLock)
{
DownloadingUrls.Remove(current);
}
return;
}
CompletedUrls.Add(current);
lock (urlLock)
{
DownloadingUrls.Remove(current);
}
var ev = new DownloadedEventArgs()
{
Html = taks.Result,
OverrideUrlParse = false,
Url = current
};
Downloaded?.Invoke(this, ev);
//覆盖默认的url解析
if (ev.OverrideUrlParse && ev.NextUrls != null)
{
foreach (var u in ev.NextUrls)
{
AddPending(u);
}
}
else
{
Parse(taks.Result, current);//解析,并加入新的链接
}
});
downloadTask.Add(down);
Downloading?.Invoke(this, new DownloadingEventArgs()
{
Url = current
});
}
if (pendingUrls.Count <= 0)
{
await Task.WhenAny(downloadTask);
downloadTask.RemoveAll(t => t.IsCompleted);
}
}
await Task.WhenAll(downloadTask);
this.CrawlerCompleted?.Invoke(this, new CrawlerCompletedEventArgs()
{
FailedCount = FailedUrls.Count,
SuccessCount = CompletedUrls.Count
});
}
/// <summary>
/// 下载一个视频
/// </summary>
/// <param name="downloading"></param>
/// <returns></returns>
private async Task SingleDownload(DownloadingItem downloading)
{
// 路径
downloading.DownloadBase.FilePath = downloading.DownloadBase.FilePath.Replace("\\", "/");
string[] temp = downloading.DownloadBase.FilePath.Split('/');
//string path = downloading.DownloadBase.FilePath.Replace(temp[temp.Length - 1], "");
string path = downloading.DownloadBase.FilePath.TrimEnd(temp[temp.Length - 1].ToCharArray());
// 路径不存在则创建
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
try
{
await Task.Run(new Action(() =>
{
// 初始化
downloading.DownloadStatusTitle = string.Empty;
downloading.DownloadContent = string.Empty;
//downloading.Downloading.DownloadFiles.Clear();
// 解析并依次下载音频、视频、弹幕、字幕、封面等内容
Parse(downloading);
// 暂停
Pause(downloading);
string audioUid = null;
// 如果需要下载音频
if (downloading.DownloadBase.NeedDownloadContent["downloadAudio"])
{
//audioUid = DownloadAudio(downloading);
for (int i = 0; i < retry; i++)
{
audioUid = DownloadAudio(downloading);
if (audioUid != null && audioUid != nullMark)
{
break;
}
}
}
if (audioUid == nullMark)
{
DownloadFailed(downloading);
return;
}
// 暂停
Pause(downloading);
string videoUid = null;
// 如果需要下载视频
if (downloading.DownloadBase.NeedDownloadContent["downloadVideo"])
{
//videoUid = DownloadVideo(downloading);
for (int i = 0; i < retry; i++)
{
videoUid = DownloadVideo(downloading);
if (videoUid != null && videoUid != nullMark)
{
break;
}
}
}
if (videoUid == nullMark)
{
DownloadFailed(downloading);
return;
}
// 暂停
Pause(downloading);
string outputDanmaku = null;
// 如果需要下载弹幕
if (downloading.DownloadBase.NeedDownloadContent["downloadDanmaku"])
{
outputDanmaku = DownloadDanmaku(downloading);
}
// 暂停
Pause(downloading);
List <string> outputSubtitles = null;
// 如果需要下载字幕
if (downloading.DownloadBase.NeedDownloadContent["downloadSubtitle"])
{
outputSubtitles = DownloadSubtitle(downloading);
}
// 暂停
Pause(downloading);
string outputCover = null;
string outputPageCover = null;
// 如果需要下载封面
if (downloading.DownloadBase.NeedDownloadContent["downloadCover"])
{
// page的封面
string pageCoverFileName = $"{downloading.DownloadBase.FilePath}.{GetImageExtension(downloading.DownloadBase.PageCoverUrl)}";
outputPageCover = DownloadCover(downloading, downloading.DownloadBase.PageCoverUrl, pageCoverFileName);
string coverFileName = $"{downloading.DownloadBase.FilePath}.Cover.{GetImageExtension(downloading.DownloadBase.CoverUrl)}";
// 封面
//outputCover = DownloadCover(downloading, downloading.DownloadBase.CoverUrl, $"{path}/Cover.{GetImageExtension(downloading.DownloadBase.CoverUrl)}");
outputCover = DownloadCover(downloading, downloading.DownloadBase.CoverUrl, coverFileName);
}
// 暂停
Pause(downloading);
// 混流
string outputMedia = string.Empty;
if (downloading.DownloadBase.NeedDownloadContent["downloadAudio"] || downloading.DownloadBase.NeedDownloadContent["downloadVideo"])
{
outputMedia = MixedFlow(downloading, audioUid, videoUid);
}
// 这里本来只有IsExist，没有pause，不知道怎么处理
// 是否存在
//isExist = IsExist(downloading);
//if (!isExist.Result)
//{
// return;
//}
// 检测音频、视频是否下载成功
bool isMediaSuccess = true;
if (downloading.DownloadBase.NeedDownloadContent["downloadAudio"] || downloading.DownloadBase.NeedDownloadContent["downloadVideo"])
{
// 只有下载音频不下载视频时才输出aac
// 只要下载视频就输出mp4
if (File.Exists(outputMedia))
{
// 成功
isMediaSuccess = true;
}
else
{
isMediaSuccess = false;
}
}
// 检测弹幕是否下载成功
bool isDanmakuSuccess = true;
if (downloading.DownloadBase.NeedDownloadContent["downloadDanmaku"])
{
if (File.Exists(outputDanmaku))
{
// 成功
isDanmakuSuccess = true;
}
else
{
isDanmakuSuccess = false;
}
}
// 检测字幕是否下载成功
bool isSubtitleSuccess = true;
if (downloading.DownloadBase.NeedDownloadContent["downloadSubtitle"])
{
if (outputSubtitles == null)
{
// 为null时表示不存在字幕
}
else
{
foreach (string subtitle in outputSubtitles)
{
if (!File.Exists(subtitle))
{
// 如果有一个不存在则失败
isSubtitleSuccess = false;
}
}
}
}
// 检测封面是否下载成功
bool isCover = true;
if (downloading.DownloadBase.NeedDownloadContent["downloadCover"])
{
if (File.Exists(outputCover) || File.Exists(outputPageCover))
{
// 成功
isCover = true;
}
else
{
isCover = false;
}
}
if (!isMediaSuccess || !isDanmakuSuccess || !isSubtitleSuccess || !isCover)
{
DownloadFailed(downloading);
return;
}
// 下载完成后处理
Downloaded downloaded = new Downloaded
{
MaxSpeedDisplay = Format.FormatSpeed(downloading.Downloading.MaxSpeed),
};
// 设置完成时间
downloaded.SetFinishedTimestamp(new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds());
DownloadedItem downloadedItem = new DownloadedItem
{
DownloadBase = downloading.DownloadBase,
Downloaded = downloaded
};
App.PropertyChangeAsync(new Action(() =>
{
// 加入到下载完成list中，并从下载中list去除
downloadedList.Add(downloadedItem);
downloadingList.Remove(downloading);
// 下载完成列表排序
DownloadFinishedSort finishedSort = SettingsManager.GetInstance().GetDownloadFinishedSort();
App.SortDownloadedList(finishedSort);
}));
}));
}
catch (OperationCanceledException e)
{
Core.Utils.Debugging.Console.PrintLine(Tag, e.ToString());
LogManager.Debug(Tag, e.Message);
}
}
