public bool IsExits(ExtractedInfo data)
{
lock (_sych)
{
return(_files.ContainsKey(data.Id));
}
}
public ExtractedInfo Select(int id)
{
ExtractedInfo xxx = null;
_files.TryGetValue(id, out xxx);
return(xxx);
}
public void Update(ExtractedInfo data)
{
lock (_sych)
{
var f = _data.FirstOrDefault(i => i.Id == data.Id);
if (f != null)
{
f.Description = data.Description;
f.Owner = data.Owner;
f.IsComplete = data.IsComplete;
f.IsError = data.IsError;
f.MapingResult = data.MapingResult;
_files[data.Id] = f;
}
}
SaveToDisk(data);
}
void SaveToDisk(ExtractedInfo doc)
{
new ThreadSafe(() =>
{
lock (_lockDisk)
{
var saveTo = _indexedDir + doc.Id.ToString() + ".ddat";
using (var sw = new StreamWriter(saveTo, false))
{
var serialize = new JavaScriptSerializer().Serialize(doc);
sw.Write(serialize);
sw.Flush();
sw.Close();
}
}
}).Start();
}
public void Add(ExtractedInfo data)
{
var id = data.Owner.Uri.UrlToHashCode();
data.Id = id;
if (!_files.ContainsKey(id))
{
lock (_sych)
{
if (!_files.ContainsKey(id))
{
_data.Add(data);
_files.Add(id, data);
}
}
SaveToDisk(data);
}
}
private ExtractedInfo BasicExtractString(LineInfo line)
{
ExtractedInfo res;
if (mIsLast)
{
res = new ExtractedInfo(line);
}
else
{
int sepPos;
sepPos = line.IndexOf(mSeparator);
if (sepPos == -1)
{
if (mNextIsOptional == false)
{
string msg = null;
if (mIsFirst && line.EmptyFromPos())
{
msg = "The line " + line.mReader.LineNumber.ToString() + " is empty. Maybe you need to use the attribute [IgnoreEmptyLines] in your record class.";
}
else
{
msg = "The delimiter '" + mSeparator + "' can´t be found after the field '" + mFieldInfo.Name + "' at line " + line.mReader.LineNumber.ToString() + " (the record has less fields, the delimiter is wrong or the next field must be marked as optional).";
}
throw new FileHelpersException(msg);
}
else
{
sepPos = line.mLine.Length - 1;
}
}
res = new ExtractedInfo(line, sepPos);
}
return(res);
}
private void OnDataMapingComplete(IDataMaping sender, Link owner, Document doc, List <File> files, List <Maping> mapeds)
{
if (!string.IsNullOrEmpty(doc.HtmlSource))
{
var extractedInfo = new ExtractedInfo();
extractedInfo.Owner = owner;
extractedInfo.MapingResult = mapeds;
RepositoryContainer.DataCollectRepository.AddOrUpdate(extractedInfo);
if (!string.IsNullOrEmpty(_config.PostAndSaveDataToUrl))
{
new ThreadSafe(() =>
{
string serialize = new JavaScriptSerializer().Serialize(extractedInfo);
//var postBuffer = Encoding.UTF8.GetBytes(serialize);
// var str64 = Convert.ToBase64String(postBuffer);
var test = new HttpRequest(_config.PostAndSaveDataToUrl).HtmlByPostJson(
serialize);
}).Start();
}
}
}
protected override ExtractedInfo ExtractFieldString(LineInfo line)
{
if (line.CurrentLength == 0)
{
if (mIsOptional)
{
return(ExtractedInfo.Empty);
}
else
{
throw new BadUsageException("End Of Line found processing the field: " + mFieldInfo.Name + " at line " + line.mReader.LineNumber.ToString() + ". (You need to mark it as [FieldOptional] if you want to avoid this exception)");
}
}
ExtractedInfo res;
if (line.CurrentLength < mFieldLength)
{
if (mFixedMode == FixedMode.AllowLessChars || mFixedMode == FixedMode.AllowVariableLength)
{
res = new ExtractedInfo(line);
}
else
{
throw new BadUsageException("The string '" + line.CurrentString + "' (length " + line.CurrentLength.ToString() + ") at line " + line.mReader.LineNumber.ToString() + " has less chars than the defined for " + mFieldInfo.Name + " (" + mFieldLength.ToString() + "). You can use the [FixedLengthRecord(FixedMode.AllowLessChars)] to avoid this problem.");
}
}
else if (mIsLast && line.CurrentLength > mFieldLength && mFixedMode != FixedMode.AllowMoreChars && mFixedMode != FixedMode.AllowVariableLength)
{
throw new BadUsageException("The string '" + line.CurrentString + "' (length " + line.CurrentLength.ToString() + ") at line " + line.mReader.LineNumber.ToString() + " has more chars than the defined for the last field " + mFieldInfo.Name + " (" + mFieldLength.ToString() + ").You can use the [FixedLengthRecord(FixedMode.AllowMoreChars)] to avoid this problem.");
}
else
{
res = new ExtractedInfo(line, line.mCurrentPos + mFieldLength);
}
return(res);
}
static DataCollectRepository()
{
_indexedDir = _root.Trim('/') + "/" + _indexedDir.Trim('/') + "/";
if (!Directory.Exists(_indexedDir))
{
Directory.CreateDirectory(_indexedDir);
}
new ThreadSafe(() =>
{
var files = Directory.GetFiles(_indexedDir);
foreach (var f in files)
{
var item = new ExtractedInfo();
lock (_sych)
{
item.Id = Files.GetIdFromFileIndexed(f);
item.LazyLoad();
_data.Add(item);
_files.Add(item.Id, item);
}
}
}).Start();
}
internal object AssignFromString(ExtractedInfo fieldString, LineInfo line)
{
object val;
switch (mTrimMode)
{
case TrimMode.None:
break;
case TrimMode.Both:
fieldString.TrimBoth(mTrimChars);
break;
case TrimMode.Left:
fieldString.TrimStart(mTrimChars);
break;
case TrimMode.Right:
fieldString.TrimEnd(mTrimChars);
break;
}
try
{
if (mConvertProvider == null)
{
if (mIsStringField)
{
val = fieldString.ExtractedString();
}
else
{
// Trim it to use Convert.ChangeType
fieldString.TrimBoth(WhitespaceChars);
if (fieldString.Length == 0)
{
// Empty stand for null
val = GetNullValue();
}
else
{
val = Convert.ChangeType(fieldString.ExtractedString(), mFieldType, null);
}
}
}
else
{
if (mConvertProvider.CustomNullHandling == false &&
fieldString.HasOnlyThisChars(WhitespaceChars))
{
val = GetNullValue();
}
else
{
var from = fieldString.ExtractedString();
val = mConvertProvider.StringToField(from);
if (val == null)
{
val = GetNullValue();
}
}
}
return(val);
}
catch (ConvertException ex)
{
var e = ConvertException.ReThrowException(ex, mFieldInfo.Name, line.mReader.LineNumber, fieldString.ExtractedFrom + 1);
throw e;
}
}
public DownloadElement(string url)
{
InitializeComponent();
DownloadInfo.DownloadStarts = DateTime.Now;
Info.Text = url;
int hitomi_id = 0;
if (int.TryParse(url, out hitomi_id))
{
url = "https://hitomi.la/galleries/" + url + ".html";
}
var dbm = new DownloadDBModel();
dbm.Url = url;
dbm.StartsTime = DownloadInfo.DownloadStarts;
dbm.State = DownloadDBState.Downloading;
DownloadDBManager.Instance.Add(dbm);
Commands.SetTap(Body, new Command(async() =>
{
await(Application.Current.MainPage as MainPage).NaviInstance.PushAsync(new DownloadInfoPage(dbm));
}));
if (!url.StartsWith("http://") && !url.StartsWith("https://"))
{
Status.Text = "옳바른 URL이 아닙니다!";
Status.TextColor = Color.Red;
Spinner.IsVisible = false;
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Add(dbm);
return;
}
SetupFavicon(url);
Task.Run(() =>
{
var extractor = ExtractorManager.Instance.GetExtractor(url);
if (extractor == null)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "적절한 다운로드작업을 찾을 수 없습니다!";
Status.TextColor = Color.Red;
Spinner.IsVisible = false;
});
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Update(dbm);
return;
}
if (extractor.IsForbidden)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "정책상 금지된 작업입니다.";
Status.TextColor = Color.Red;
Spinner.IsVisible = false;
});
dbm.State = DownloadDBState.Forbidden;
DownloadDBManager.Instance.Update(dbm);
return;
}
WAIT_ANOTHER_TASKS:
if (DownloadAvailable == 4)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = $"다른 작업이 끝나길 기다리는 중 입니다...";
});
while (DownloadAvailable >= 4)
{
Thread.Sleep(1000);
}
}
if (Interlocked.Increment(ref DownloadAvailable) > 4)
{
goto WAIT_ANOTHER_TASKS;
}
Device.BeginInvokeOnMainThread(() =>
{
Info.Text = extractor.GetType().Name.Replace("Extractor", "") + " (" + "/" + string.Join("/", url.Split('/').Skip(3)) + ")";
dbm.ShortInfo = Info.Text;
DownloadDBManager.Instance.Update(dbm);
Status.Text = "다운로드 정보를 추출 중 입니다...";
});
var option = extractor.RecommendOption(url);
long extracting_progress_max = 0;
long extracting_cumulative_count = 0;
option.ProgressMax = (count) =>
{
extracting_progress_max = count;
Device.BeginInvokeOnMainThread(() =>
{
ProgressProgressText.IsVisible = false;
Progress.IsVisible = true;
});
};
option.PostStatus = (count) =>
{
var val = Interlocked.Add(ref extracting_cumulative_count, count);
Device.BeginInvokeOnMainThread(() =>
{
if (extracting_progress_max != 0)
{
Progress.Progress = val / (double)extracting_progress_max;
Status.Text = $"추출중...[{val}/{extracting_progress_max}]";
}
else
{
Status.Text = $"추출중...[{val}개 항목 추출됨]";
}
});
};
option.SimpleInfoCallback = (info) =>
{
Device.BeginInvokeOnMainThread(() =>
{
Info.Text = $"{info}";
dbm.ShortInfo = info;
DownloadDBManager.Instance.Update(dbm);
});
};
option.ThumbnailCallback = (thumbnail) =>
{
Task.Run(async() =>
{
var ttask = NetTask.MakeDefault(thumbnail.Url);
ttask.Priority = new NetPriority {
Type = NetPriorityType.Trivial
};
ttask.Filename = Path.Combine(AppProvider.ApplicationPath, (url + "*thumbnail" + dbm.Id).GetHashMD5() + Path.GetExtension(thumbnail.Filename));
ttask.Headers = thumbnail.Headers;
ttask.Referer = thumbnail.Referer;
ttask.Cookie = thumbnail.Cookie;
ttask.Accept = thumbnail.Accept;
ttask.UserAgent = thumbnail.UserAgent;
dbm.ThumbnailCahce = ttask.Filename;
await NetTools.DownloadFileAsync(ttask);
DownloadDBManager.Instance.Update(dbm);
Device.BeginInvokeOnMainThread(() =>
{
Thumbnail.HeightRequest = Height - 8;
Thumbnail.IsVisible = true;
Thumbnail.Source = ttask.Filename;
});
});
};
(List <NetTask>, ExtractedInfo)tasks;
try
{
tasks = extractor.Extract(url, option);
}
catch (Exception e)
{
Logs.Instance.PushError(e.Message);
Logs.Instance.PushError(e.StackTrace);
Device.BeginInvokeOnMainThread(() =>
{
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Spinner.IsVisible = false;
Status.Text = "추출 작업 중 오류가 발생했습니다 :(\n" + e.Message;
Status.TextColor = Color.Red;
});
Interlocked.Decrement(ref DownloadAvailable);
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Update(dbm);
return;
}
if (tasks.Item1 == null)
{
Device.BeginInvokeOnMainThread(() =>
{
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Spinner.IsVisible = false;
Status.Text = "다운로드할 내용이 없습니다 :(";
});
Interlocked.Decrement(ref DownloadAvailable);
dbm.State = DownloadDBState.ErrorOccured;
DownloadDBManager.Instance.Update(dbm);
return;
}
if (tasks.Item2 != null)
{
ExtractedInfo = tasks.Item2;
CacheManager.Instance.Append(url + "*info" + dbm.Id, ExtractedInfo);
dbm.InfoCache = url + "*info" + dbm.Id;
DownloadDBManager.Instance.Update(dbm);
}
var format = extractor.RecommendFormat(option);
Device.BeginInvokeOnMainThread(() =>
{
Spinner.IsVisible = false;
Status.Text = "다운로드 중...";
ProgressProgressText.IsVisible = false;
Progress.IsVisible = true;
});
int download_count = 0;
long download_bytes = 0;
long download_1s = 0;
int task_count = AppProvider.Scheduler.LatestPriority != null ?
AppProvider.Scheduler.LatestPriority.TaskPriority : 0;
int post_process_count = 0;
int post_process_progress = 0;
bool canceled = false;
if (tasks.Item1.Count > 0)
{
var hash_set = new HashSet <string>();
tasks.Item1.ForEach(x => hash_set.Add(Path.GetDirectoryName(Path.Combine("/", x.Format.Formatting(format)))));
if (hash_set.Count == 1)
{
dbm.Directory = Path.GetDirectoryName(Path.Combine(Settings.Instance.Model.SuperPath, tasks.Item1[0].Format.Formatting(format)));
}
else
{
dbm.Directory = Path.GetDirectoryName(Path.GetDirectoryName(Path.Combine(Settings.Instance.Model.SuperPath, tasks.Item1[0].Format.Formatting(format))));
}
dbm.CountOfFiles = tasks.Item1.Count;
DownloadDBManager.Instance.Update(dbm);
}
tasks.Item1.ForEach(task => {
task.Priority.TaskPriority = task_count++;
task.Filename = Path.Combine(Settings.Instance.Model.SuperPath, task.Format.Formatting(format));
if (!Directory.Exists(Path.GetDirectoryName(task.Filename)))
{
Directory.CreateDirectory(Path.GetDirectoryName(task.Filename));
}
task.DownloadCallback = (sz) =>
{
Interlocked.Add(ref download_1s, sz);
Interlocked.Add(ref download_bytes, sz);
};
task.CompleteCallback = () =>
{
var cur = Interlocked.Increment(ref download_count);
Device.BeginInvokeOnMainThread(() =>
{
Progress.Progress = cur / (double)tasks.Item1.Count;
});
};
task.CancleCallback = () =>
{
if (!canceled)
{
dbm.State = DownloadDBState.Aborted;
DownloadDBManager.Instance.Update(dbm);
}
canceled = true;
};
task.Cancel = CancelSource.Token;
if (task.PostProcess != null)
{
task.StartPostprocessorCallback = () =>
{
Interlocked.Increment(ref post_process_count);
};
task.PostProcess.CompletePostprocessor = (index) =>
{
Interlocked.Increment(ref post_process_progress);
};
}
AppProvider.Scheduler.Add(task);
});
while (tasks.Item1.Count != download_count && !canceled)
{
Thread.Sleep(1000);
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = $"[{download_count}/{tasks.Item1.Count}] ({convert_bytes2string(download_1s)}/S {convert_bytes2string(download_bytes)})";
Interlocked.Exchange(ref download_1s, 0);
});
}
Interlocked.Decrement(ref DownloadAvailable);
dbm.State = DownloadDBState.Downloaded;
dbm.EndsTime = DateTime.Now;
dbm.SizeOfContents = download_bytes;
DownloadDBManager.Instance.Update(dbm);
while (post_process_progress != post_process_count && !canceled)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = $"후처리 작업 중...[{post_process_progress}/{post_process_count}]";
Progress.Progress = post_process_progress / (double)post_process_count;
});
Thread.Sleep(1000);
}
if (!canceled)
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "다운로드 완료";
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Plugin.XSnack.CrossXSnack.Current.ShowMessage(Info.Text + " 항목의 다운로드가 완료되었습니다.");
});
}
else
{
Device.BeginInvokeOnMainThread(() =>
{
Status.Text = "다운로드 취소됨";
ProgressProgressText.IsVisible = true;
ProgressProgressText.Text = "";
ProgressText.Text = "";
Progress.IsVisible = false;
Plugin.XSnack.CrossXSnack.Current.ShowMessage(Info.Text + " 항목의 다운로드가 취소되었습니다.");
});
}
DownloadInfo.DownloadEnds = DateTime.Now;
});
}
