public bool IsExits(ExtractedInfo data) { lock (_sych) { return(_files.ContainsKey(data.Id)); } }
public ExtractedInfo Select(int id) { ExtractedInfo xxx = null; _files.TryGetValue(id, out xxx); return(xxx); }
public void Update(ExtractedInfo data) { lock (_sych) { var f = _data.FirstOrDefault(i => i.Id == data.Id); if (f != null) { f.Description = data.Description; f.Owner = data.Owner; f.IsComplete = data.IsComplete; f.IsError = data.IsError; f.MapingResult = data.MapingResult; _files[data.Id] = f; } } SaveToDisk(data); }
void SaveToDisk(ExtractedInfo doc) { new ThreadSafe(() => { lock (_lockDisk) { var saveTo = _indexedDir + doc.Id.ToString() + ".ddat"; using (var sw = new StreamWriter(saveTo, false)) { var serialize = new JavaScriptSerializer().Serialize(doc); sw.Write(serialize); sw.Flush(); sw.Close(); } } }).Start(); }
public void Add(ExtractedInfo data) { var id = data.Owner.Uri.UrlToHashCode(); data.Id = id; if (!_files.ContainsKey(id)) { lock (_sych) { if (!_files.ContainsKey(id)) { _data.Add(data); _files.Add(id, data); } } SaveToDisk(data); } }
private ExtractedInfo BasicExtractString(LineInfo line) { ExtractedInfo res; if (mIsLast) { res = new ExtractedInfo(line); } else { int sepPos; sepPos = line.IndexOf(mSeparator); if (sepPos == -1) { if (mNextIsOptional == false) { string msg = null; if (mIsFirst && line.EmptyFromPos()) { msg = "The line " + line.mReader.LineNumber.ToString() + " is empty. Maybe you need to use the attribute [IgnoreEmptyLines] in your record class."; } else { msg = "The delimiter '" + mSeparator + "' can´t be found after the field '" + mFieldInfo.Name + "' at line " + line.mReader.LineNumber.ToString() + " (the record has less fields, the delimiter is wrong or the next field must be marked as optional)."; } throw new FileHelpersException(msg); } else { sepPos = line.mLine.Length - 1; } } res = new ExtractedInfo(line, sepPos); } return(res); }
private void OnDataMapingComplete(IDataMaping sender, Link owner, Document doc, List <File> files, List <Maping> mapeds) { if (!string.IsNullOrEmpty(doc.HtmlSource)) { var extractedInfo = new ExtractedInfo(); extractedInfo.Owner = owner; extractedInfo.MapingResult = mapeds; RepositoryContainer.DataCollectRepository.AddOrUpdate(extractedInfo); if (!string.IsNullOrEmpty(_config.PostAndSaveDataToUrl)) { new ThreadSafe(() => { string serialize = new JavaScriptSerializer().Serialize(extractedInfo); //var postBuffer = Encoding.UTF8.GetBytes(serialize); // var str64 = Convert.ToBase64String(postBuffer); var test = new HttpRequest(_config.PostAndSaveDataToUrl).HtmlByPostJson( serialize); }).Start(); } } }
protected override ExtractedInfo ExtractFieldString(LineInfo line) { if (line.CurrentLength == 0) { if (mIsOptional) { return(ExtractedInfo.Empty); } else { throw new BadUsageException("End Of Line found processing the field: " + mFieldInfo.Name + " at line " + line.mReader.LineNumber.ToString() + ". (You need to mark it as [FieldOptional] if you want to avoid this exception)"); } } ExtractedInfo res; if (line.CurrentLength < mFieldLength) { if (mFixedMode == FixedMode.AllowLessChars || mFixedMode == FixedMode.AllowVariableLength) { res = new ExtractedInfo(line); } else { throw new BadUsageException("The string '" + line.CurrentString + "' (length " + line.CurrentLength.ToString() + ") at line " + line.mReader.LineNumber.ToString() + " has less chars than the defined for " + mFieldInfo.Name + " (" + mFieldLength.ToString() + "). You can use the [FixedLengthRecord(FixedMode.AllowLessChars)] to avoid this problem."); } } else if (mIsLast && line.CurrentLength > mFieldLength && mFixedMode != FixedMode.AllowMoreChars && mFixedMode != FixedMode.AllowVariableLength) { throw new BadUsageException("The string '" + line.CurrentString + "' (length " + line.CurrentLength.ToString() + ") at line " + line.mReader.LineNumber.ToString() + " has more chars than the defined for the last field " + mFieldInfo.Name + " (" + mFieldLength.ToString() + ").You can use the [FixedLengthRecord(FixedMode.AllowMoreChars)] to avoid this problem."); } else { res = new ExtractedInfo(line, line.mCurrentPos + mFieldLength); } return(res); }
static DataCollectRepository() { _indexedDir = _root.Trim('/') + "/" + _indexedDir.Trim('/') + "/"; if (!Directory.Exists(_indexedDir)) { Directory.CreateDirectory(_indexedDir); } new ThreadSafe(() => { var files = Directory.GetFiles(_indexedDir); foreach (var f in files) { var item = new ExtractedInfo(); lock (_sych) { item.Id = Files.GetIdFromFileIndexed(f); item.LazyLoad(); _data.Add(item); _files.Add(item.Id, item); } } }).Start(); }
internal object AssignFromString(ExtractedInfo fieldString, LineInfo line) { object val; switch (mTrimMode) { case TrimMode.None: break; case TrimMode.Both: fieldString.TrimBoth(mTrimChars); break; case TrimMode.Left: fieldString.TrimStart(mTrimChars); break; case TrimMode.Right: fieldString.TrimEnd(mTrimChars); break; } try { if (mConvertProvider == null) { if (mIsStringField) { val = fieldString.ExtractedString(); } else { // Trim it to use Convert.ChangeType fieldString.TrimBoth(WhitespaceChars); if (fieldString.Length == 0) { // Empty stand for null val = GetNullValue(); } else { val = Convert.ChangeType(fieldString.ExtractedString(), mFieldType, null); } } } else { if (mConvertProvider.CustomNullHandling == false && fieldString.HasOnlyThisChars(WhitespaceChars)) { val = GetNullValue(); } else { var from = fieldString.ExtractedString(); val = mConvertProvider.StringToField(from); if (val == null) { val = GetNullValue(); } } } return(val); } catch (ConvertException ex) { var e = ConvertException.ReThrowException(ex, mFieldInfo.Name, line.mReader.LineNumber, fieldString.ExtractedFrom + 1); throw e; } }
public DownloadElement(string url) { InitializeComponent(); DownloadInfo.DownloadStarts = DateTime.Now; Info.Text = url; int hitomi_id = 0; if (int.TryParse(url, out hitomi_id)) { url = "https://hitomi.la/galleries/" + url + ".html"; } var dbm = new DownloadDBModel(); dbm.Url = url; dbm.StartsTime = DownloadInfo.DownloadStarts; dbm.State = DownloadDBState.Downloading; DownloadDBManager.Instance.Add(dbm); Commands.SetTap(Body, new Command(async() => { await(Application.Current.MainPage as MainPage).NaviInstance.PushAsync(new DownloadInfoPage(dbm)); })); if (!url.StartsWith("http://") && !url.StartsWith("https://")) { Status.Text = "옳바른 URL이 아닙니다!"; Status.TextColor = Color.Red; Spinner.IsVisible = false; dbm.State = DownloadDBState.ErrorOccured; DownloadDBManager.Instance.Add(dbm); return; } SetupFavicon(url); Task.Run(() => { var extractor = ExtractorManager.Instance.GetExtractor(url); if (extractor == null) { Device.BeginInvokeOnMainThread(() => { Status.Text = "적절한 다운로드작업을 찾을 수 없습니다!"; Status.TextColor = Color.Red; Spinner.IsVisible = false; }); dbm.State = DownloadDBState.ErrorOccured; DownloadDBManager.Instance.Update(dbm); return; } if (extractor.IsForbidden) { Device.BeginInvokeOnMainThread(() => { Status.Text = "정책상 금지된 작업입니다."; Status.TextColor = Color.Red; Spinner.IsVisible = false; }); dbm.State = DownloadDBState.Forbidden; DownloadDBManager.Instance.Update(dbm); return; } WAIT_ANOTHER_TASKS: if (DownloadAvailable == 4) { Device.BeginInvokeOnMainThread(() => { Status.Text = $"다른 작업이 끝나길 기다리는 중 입니다..."; }); while (DownloadAvailable >= 4) { Thread.Sleep(1000); } } if (Interlocked.Increment(ref DownloadAvailable) > 4) { goto WAIT_ANOTHER_TASKS; } Device.BeginInvokeOnMainThread(() => { Info.Text = extractor.GetType().Name.Replace("Extractor", "") + " (" + "/" + string.Join("/", url.Split('/').Skip(3)) + ")"; dbm.ShortInfo = Info.Text; DownloadDBManager.Instance.Update(dbm); Status.Text = "다운로드 정보를 추출 중 입니다..."; }); var option = extractor.RecommendOption(url); long extracting_progress_max = 0; long extracting_cumulative_count = 0; option.ProgressMax = (count) => { extracting_progress_max = count; Device.BeginInvokeOnMainThread(() => { ProgressProgressText.IsVisible = false; Progress.IsVisible = true; }); }; option.PostStatus = (count) => { var val = Interlocked.Add(ref extracting_cumulative_count, count); Device.BeginInvokeOnMainThread(() => { if (extracting_progress_max != 0) { Progress.Progress = val / (double)extracting_progress_max; Status.Text = $"추출중...[{val}/{extracting_progress_max}]"; } else { Status.Text = $"추출중...[{val}개 항목 추출됨]"; } }); }; option.SimpleInfoCallback = (info) => { Device.BeginInvokeOnMainThread(() => { Info.Text = $"{info}"; dbm.ShortInfo = info; DownloadDBManager.Instance.Update(dbm); }); }; option.ThumbnailCallback = (thumbnail) => { Task.Run(async() => { var ttask = NetTask.MakeDefault(thumbnail.Url); ttask.Priority = new NetPriority { Type = NetPriorityType.Trivial }; ttask.Filename = Path.Combine(AppProvider.ApplicationPath, (url + "*thumbnail" + dbm.Id).GetHashMD5() + Path.GetExtension(thumbnail.Filename)); ttask.Headers = thumbnail.Headers; ttask.Referer = thumbnail.Referer; ttask.Cookie = thumbnail.Cookie; ttask.Accept = thumbnail.Accept; ttask.UserAgent = thumbnail.UserAgent; dbm.ThumbnailCahce = ttask.Filename; await NetTools.DownloadFileAsync(ttask); DownloadDBManager.Instance.Update(dbm); Device.BeginInvokeOnMainThread(() => { Thumbnail.HeightRequest = Height - 8; Thumbnail.IsVisible = true; Thumbnail.Source = ttask.Filename; }); }); }; (List <NetTask>, ExtractedInfo)tasks; try { tasks = extractor.Extract(url, option); } catch (Exception e) { Logs.Instance.PushError(e.Message); Logs.Instance.PushError(e.StackTrace); Device.BeginInvokeOnMainThread(() => { ProgressProgressText.IsVisible = true; ProgressProgressText.Text = ""; ProgressText.Text = ""; Progress.IsVisible = false; Spinner.IsVisible = false; Status.Text = "추출 작업 중 오류가 발생했습니다 :(\n" + e.Message; Status.TextColor = Color.Red; }); Interlocked.Decrement(ref DownloadAvailable); dbm.State = DownloadDBState.ErrorOccured; DownloadDBManager.Instance.Update(dbm); return; } if (tasks.Item1 == null) { Device.BeginInvokeOnMainThread(() => { ProgressProgressText.IsVisible = true; ProgressProgressText.Text = ""; ProgressText.Text = ""; Progress.IsVisible = false; Spinner.IsVisible = false; Status.Text = "다운로드할 내용이 없습니다 :("; }); Interlocked.Decrement(ref DownloadAvailable); dbm.State = DownloadDBState.ErrorOccured; DownloadDBManager.Instance.Update(dbm); return; } if (tasks.Item2 != null) { ExtractedInfo = tasks.Item2; CacheManager.Instance.Append(url + "*info" + dbm.Id, ExtractedInfo); dbm.InfoCache = url + "*info" + dbm.Id; DownloadDBManager.Instance.Update(dbm); } var format = extractor.RecommendFormat(option); Device.BeginInvokeOnMainThread(() => { Spinner.IsVisible = false; Status.Text = "다운로드 중..."; ProgressProgressText.IsVisible = false; Progress.IsVisible = true; }); int download_count = 0; long download_bytes = 0; long download_1s = 0; int task_count = AppProvider.Scheduler.LatestPriority != null ? AppProvider.Scheduler.LatestPriority.TaskPriority : 0; int post_process_count = 0; int post_process_progress = 0; bool canceled = false; if (tasks.Item1.Count > 0) { var hash_set = new HashSet <string>(); tasks.Item1.ForEach(x => hash_set.Add(Path.GetDirectoryName(Path.Combine("/", x.Format.Formatting(format))))); if (hash_set.Count == 1) { dbm.Directory = Path.GetDirectoryName(Path.Combine(Settings.Instance.Model.SuperPath, tasks.Item1[0].Format.Formatting(format))); } else { dbm.Directory = Path.GetDirectoryName(Path.GetDirectoryName(Path.Combine(Settings.Instance.Model.SuperPath, tasks.Item1[0].Format.Formatting(format)))); } dbm.CountOfFiles = tasks.Item1.Count; DownloadDBManager.Instance.Update(dbm); } tasks.Item1.ForEach(task => { task.Priority.TaskPriority = task_count++; task.Filename = Path.Combine(Settings.Instance.Model.SuperPath, task.Format.Formatting(format)); if (!Directory.Exists(Path.GetDirectoryName(task.Filename))) { Directory.CreateDirectory(Path.GetDirectoryName(task.Filename)); } task.DownloadCallback = (sz) => { Interlocked.Add(ref download_1s, sz); Interlocked.Add(ref download_bytes, sz); }; task.CompleteCallback = () => { var cur = Interlocked.Increment(ref download_count); Device.BeginInvokeOnMainThread(() => { Progress.Progress = cur / (double)tasks.Item1.Count; }); }; task.CancleCallback = () => { if (!canceled) { dbm.State = DownloadDBState.Aborted; DownloadDBManager.Instance.Update(dbm); } canceled = true; }; task.Cancel = CancelSource.Token; if (task.PostProcess != null) { task.StartPostprocessorCallback = () => { Interlocked.Increment(ref post_process_count); }; task.PostProcess.CompletePostprocessor = (index) => { Interlocked.Increment(ref post_process_progress); }; } AppProvider.Scheduler.Add(task); }); while (tasks.Item1.Count != download_count && !canceled) { Thread.Sleep(1000); Device.BeginInvokeOnMainThread(() => { Status.Text = $"[{download_count}/{tasks.Item1.Count}] ({convert_bytes2string(download_1s)}/S {convert_bytes2string(download_bytes)})"; Interlocked.Exchange(ref download_1s, 0); }); } Interlocked.Decrement(ref DownloadAvailable); dbm.State = DownloadDBState.Downloaded; dbm.EndsTime = DateTime.Now; dbm.SizeOfContents = download_bytes; DownloadDBManager.Instance.Update(dbm); while (post_process_progress != post_process_count && !canceled) { Device.BeginInvokeOnMainThread(() => { Status.Text = $"후처리 작업 중...[{post_process_progress}/{post_process_count}]"; Progress.Progress = post_process_progress / (double)post_process_count; }); Thread.Sleep(1000); } if (!canceled) { Device.BeginInvokeOnMainThread(() => { Status.Text = "다운로드 완료"; ProgressProgressText.IsVisible = true; ProgressProgressText.Text = ""; ProgressText.Text = ""; Progress.IsVisible = false; Plugin.XSnack.CrossXSnack.Current.ShowMessage(Info.Text + " 항목의 다운로드가 완료되었습니다."); }); } else { Device.BeginInvokeOnMainThread(() => { Status.Text = "다운로드 취소됨"; ProgressProgressText.IsVisible = true; ProgressProgressText.Text = ""; ProgressText.Text = ""; Progress.IsVisible = false; Plugin.XSnack.CrossXSnack.Current.ShowMessage(Info.Text + " 항목의 다운로드가 취소되었습니다."); }); } DownloadInfo.DownloadEnds = DateTime.Now; }); }