/// <summary> /// LocalThreadReaderクラスのインスタンスを初期化 /// </summary> /// <param name="dataParser"></param> public LocalThreadReader(ThreadParser dataParser) : base(dataParser) { // // TODO: コンストラクタ ロジックをここに追加してください。 // }
public async Task TestGuestUserPost(string url) { var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo()); var thread = await parser.ParseThreadAsync(new Uri(url), 30, 30); Assert.Contains(thread.Posts, v => v.IsGuestPost && string.IsNullOrEmpty(v.GuestUsername) == false); }
/// <summary> /// LocalThreadStorageクラスのインスタンスを初期化 /// </summary> /// <param name="cache">基になるキャッシュ情報</param> /// <param name="parser">データを解析するときに使用するパーサ</param> /// <param name="formatter">書き込み時に使用するフォーマッタ</param> /// <param name="enc">書き込み時に使用するエンコーダ</param> public LocalThreadStorage(Cache cache, ThreadParser parser, ThreadFormatter formatter, Encoding enc) { if (cache == null) { throw new ArgumentNullException("cache"); } if (parser == null) { throw new ArgumentNullException("parser"); } if (formatter == null) { throw new ArgumentNullException("formatter"); } if (enc == null) { throw new ArgumentNullException("enc"); } this.cache = cache; this.dataParser = parser; this.formatter = formatter; this.encoding = enc; this.bufferSize = 4096; this.isOpen = false; }
public async Task ReadThreadInfoTest() { var parser = new ThreadParser(Guid.Parse("18273ef6-92f8-4f11-818d-7e3b8470307a")); var info = await parser.ReadThreadInfoAsync(); // HAPPY_PASS Assert.IsNotNull(info); }
/// <summary> /// ThreadReaderBaseクラスのインスタンスを初期化 /// </summary> /// <param name="parser">読み込み時に解析を行うためのパーサー</param> protected ThreadReaderBase(ThreadParser parser) : this() { if (parser == null) { throw new ArgumentNullException("parser"); } dataParser = parser; }
private static void Crawl(string url, string path) { var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo()); var task = Task.Run(async() => { var thread = await parser.ParseThreadAsync(new Uri(url)); thread.Serialize(new FileInfo(path)); }); Task.WaitAll(task); }
public async Task TestPostParser(string url, uint page, uint postuid, bool haslikes, bool hassiganture, bool hasuser) { var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo()); var thread = await parser.ParseThreadAsync(new Uri(url.Substring(0, url.IndexOf('#'))), page, page); var post = thread.Posts.Where(v => v.Uid == postuid).Single(); Assert.Equal <uint>(postuid, post.Uid); Assert.Equal(url, post.Url); Assert.True(haslikes ? (post.LikeCount > 0) : true); Assert.True(post.DislikeCount < 1); Assert.Equal <bool>(hasuser, post.User != null); Assert.Equal <bool>(hassiganture, !string.IsNullOrEmpty(post.User?.SignatureHtml)); }
public async Task TestParseThread(string teststr) { var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo()); var thread = await parser.ParseThreadAsync(new Uri(teststr), 1, 3); var postUsers = thread.Posts.Where(v => v.User != null).Select(v => v.User).Distinct(); var userIntersects = thread.Users.Users.Intersect(postUsers); Assert.Equal("https://forum.netcup.de/sonstiges/smalltalk/1051-das-l%C3%A4ngste-thema/", thread.StartpageUrl); Assert.True(thread.PageCount >= 383); Assert.True(thread.CurrentPageNo <= thread.PageCount && thread.CurrentPageNo >= 1); Assert.True(thread.Uid > 0); Assert.True(thread.Posts.Count > 0); Assert.True(thread.Users.Users.Count > 0); Assert.Equal <int>(postUsers.Count(), userIntersects.Count()); }
private async Task <bool> RegisterNewThreadAsync(string identifier) { // register a new thread item try { var parser = new ThreadParser(Guid.Parse(identifier)); var info = await parser.ReadThreadInfoAsync(); if (info == null) { return(false); } // query the database by the identifer / create a new item if not exist var model = new ThreadModel { Id = info.Id, Title = info.Title, AuthorId = info.AuthorId, CreateOn = info.CreateOn, ForumId = info.ForumId }; var tags = Utils.DetectTagsFromTitle(info.Title); foreach (var name in tags) { var tag = _tagRepository.CreateTagIfNotExists(name); model.Tags.Add(tag); } _threadRepository.Create(model); return(true); }catch (Exception e) { return(false); } }
private static int Main(string[] args) { void Configuration(ParserSettings with) { with.EnableDashDash = true; with.CaseInsensitiveEnumValues = true; with.AutoHelp = true; with.HelpWriter = null; } var parser = new Parser(Configuration); var parsed = parser.ParseArguments <HtmlDumpOptions, JsonDumpOptions, MediaDumpOptions>(args); var result = 0; try { var resolveMedia = false; BaseOptions baseOpts = null; var processors = new List <IProcessor>(); parsed .WithParsed <HtmlDumpOptions>(opts => { processors.Add(new HtmlGeneratorProcessor( opts.OutputDirectory, !opts.DontIncludeCustomPosterNames, !opts.DontUseColoredNames, !opts.DontIncludeSameFagCount, opts.Style, opts.Prettify)); baseOpts = opts; resolveMedia = !opts.DontResolveMedia; }) .WithParsed <JsonDumpOptions>(opts => { processors.Add(new JsonSerializerProcessor(opts.OutputDirectory, opts.Formatted)); baseOpts = opts; resolveMedia = !opts.DontResolveMedia; }) .WithParsed <MediaDumpOptions>(opts => { baseOpts = opts; resolveMedia = true; }) .WithNotParsed(errs => { var helpText = HelpText.AutoBuild(parsed, h => { h = HelpText.DefaultParsingErrorsHandler(parsed, h); h.Copyright = ""; h.Heading = ""; h.AutoVersion = false; h.AddDashesToOption = true; return(h); }, e => e, true, 190); Console.WriteAscii("Mitsuba Archivizer", Color.DodgerBlue); Console.WriteFormatted("commit {0} @ {1}", new Formatter(ThisAssembly.Git.Commit, Color.Crimson), new Formatter(ThisAssembly.Git.Branch, Color.Chartreuse), Color.White); Console.Write(helpText); result = 1; }); if (result != 1) { var postCount = 0; var cursorBackupTop1 = -1; var cursorBackupLeft1 = -1; ThreadParser.OnPostCount += count => postCount = count; ThreadParser.OnPostParsing += idx => { if (cursorBackupTop1 == -1 || cursorBackupLeft1 == -1) { cursorBackupTop1 = Console.CursorTop; cursorBackupLeft1 = Console.CursorLeft; } Console.SetCursorPosition(cursorBackupLeft1, cursorBackupTop1); Console.Write(new string(' ', Console.WindowWidth)); Console.SetCursorPosition(cursorBackupLeft1, cursorBackupTop1); Console.WriteLine("Parsing post: ({0}/{1})", idx + 1, postCount); }; var resolver = new MediaResolver( resolveMedia, baseOpts.OutputDirectory, !baseOpts.DontSeperateMediaByExtension, !baseOpts.DontDownloadThumbnails, baseOpts.AllowedMediaExtensions, baseOpts.AllowedThumbnailExtensions); var mediaCount = 0; var cursorBackupTop2 = -1; var cursorBackupLeft2 = -1; resolver.OnPostWithMediaCount += count => mediaCount = count; resolver.OnProcessingPostMedia += (post, idx) => { if (cursorBackupTop2 == -1 || cursorBackupLeft2 == -1) { cursorBackupTop2 = Console.CursorTop; cursorBackupLeft2 = Console.CursorLeft; } Console.SetCursorPosition(cursorBackupLeft2, cursorBackupTop2); Console.Write(new string(' ', Console.WindowWidth)); Console.SetCursorPosition(cursorBackupLeft2, cursorBackupTop2); Console.WriteLine("Downloading media: {0} ({1}/{2})", post.File.FileName, idx + 1, mediaCount); }; var pipeline = new ProcessorPipeline(processors, resolver); pipeline.OnProcessorInvoked += name => Console.WriteLine("{0} is working...", name); foreach (var input in baseOpts.InputSeq) { var thread = ThreadParser.TryParse(input).Result; pipeline.Process(thread).Wait(); } Console.WriteLine("Done!"); } } catch (Exception ex) { Console.Error.WriteLine(ex); result = 1; } return(result); }
public async Task <ContentResult> Thread(int threadId) { var chattyThread = await _chattyProvider.GetThread(threadId); var list = new List <V1RootCommentModel>(); var maxDepth = chattyThread.Posts.Max(x => x.Depth); var lastCommentAtDepth = new V1CommentModel[maxDepth + 1]; var op = chattyThread.Posts[0]; var v1RootCommentModel = new V1RootCommentModel { Comments = new List <V1CommentModel>(), ReplyCount = chattyThread.Posts.Count, Body = op.Body, Date = op.Date, Participants = GetParticipants(chattyThread), Category = op.Category, LastReplyId = $"{chattyThread.Posts.Max(x => x.Id)}", Author = op.Author, Preview = ThreadParser.PreviewFromBody(op.Body), Id = $"{op.Id}" }; foreach (var post in chattyThread.Posts.Skip(1)) { var v1CommentModel = new V1CommentModel { Comments = new List <V1CommentModel>(), Body = post.Body, Date = post.Date, Category = post.Category, Author = post.Author, Preview = ThreadParser.PreviewFromBody(post.Body), Id = $"{post.Id}" }; if (post.Depth == 1) { v1RootCommentModel.Comments.Add(v1CommentModel); } else { lastCommentAtDepth[post.Depth - 1].Comments.Add(v1CommentModel); } lastCommentAtDepth[post.Depth] = v1CommentModel; } var response = new V1ThreadModel { Comments = new List <V1RootCommentModel> { v1RootCommentModel } }; // the serializer has a maximum depth that is too low, even if you reconfigure it to the hard max. var sb = new StringBuilder(); var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; sb.Append("{"); AppendProperty(sb, "id", v1RootCommentModel.Id, options); AppendProperty(sb, "reply_count", v1RootCommentModel.ReplyCount, options); AppendProperty(sb, "body", v1RootCommentModel.Body, options); AppendProperty(sb, "date", V1DateTimeOffsetConverter.ToJsonString(v1RootCommentModel.Date), options); AppendProperty(sb, "participants", v1RootCommentModel.Participants, options); AppendProperty(sb, "category", V1ModerationFlagConverter.ToJsonString(v1RootCommentModel.Category), options); AppendProperty(sb, "last_reply_id", v1RootCommentModel.LastReplyId, options); AppendProperty(sb, "author", v1RootCommentModel.Author, options); AppendProperty(sb, "preview", v1RootCommentModel.Preview, options); sb.Append("\"comments\":[{"); AppendProperty(sb, "id", v1RootCommentModel.Id, options); AppendProperty(sb, "body", v1RootCommentModel.Body, options); AppendProperty(sb, "date", V1DateTimeOffsetConverter.ToJsonString(v1RootCommentModel.Date), options); AppendProperty(sb, "category", V1ModerationFlagConverter.ToJsonString(v1RootCommentModel.Category), options); AppendProperty(sb, "author", v1RootCommentModel.Author, options); AppendProperty(sb, "preview", v1RootCommentModel.Preview, options); sb.Append("\"comments\":"); AppendCommentList(sb, v1RootCommentModel.Comments, options); sb.Append("]}"); return(Content(sb.ToString(), "application/json"));
/// <summary> /// 指定したパーサを使用してX2chThreadReaderクラスのインスタンスを初期化。 /// </summary> /// <param name="dataParser"></param> public X2chThreadReader(ThreadParser dataParser) : base(dataParser) { getGzip = true; }
/// <summary> /// パーサを指定してMachiThreadReaderクラスのインスタンスを初期化 /// </summary> public MachiThreadReader(ThreadParser dataParser) : base(dataParser) { }
private async void archivizeButton_Click(object sender, EventArgs e) { if (!threadListTextBox.Lines.Any()) { return; } threadListTextBox.Enabled = false; mediaGroupBox.Enabled = false; jsonGroupBox.Enabled = false; htmlGroupBox.Enabled = false; archivizeButton.Enabled = false; outputDirectoryButton.Enabled = false; var processors = new List <IProcessor>(); if (jsonEnabledCheckbox.Checked) { processors.Add(new JsonSerializerProcessor(outputDirTextbox.Text, formattedCheckbox.Checked)); } if (htmlEnabledCheckbox.Checked) { processors.Add(new HtmlGeneratorProcessor(outputDirTextbox.Text, namesCheckbox.Checked, coloredNamesCheckbox.Checked, samefagCheckbox.Checked, (string)styleComboBox.SelectedItem, prettifyCheckbox.Checked)); } MediaResolver resolver; if (mediaEnabledCheckbox.Checked) { var exts = mediaExtTextbox.Text.Split(','); var thumbExts = thumbnailExtTextBox.Text.Split(','); resolver = new MediaResolver(true, outputDirTextbox.Text, groupByExtensionsCheckbox.Checked, thumbnailsEnabledCheckbox.Checked, exts, thumbExts); resolver.OnPostWithMediaCount += Resolver_OnPostWithMediaCount; resolver.OnProcessingPostMedia += Resolver_OnProcessingPostMedia; } else { resolver = new MediaResolver(); } var pipeline = new ProcessorPipeline(processors, resolver); pipeline.OnProcessorInvoked += Pipeline_OnProcessorInvoked; foreach (var line in threadListTextBox.Lines) { try { statusTextLabel.Text = "Parsing posts..."; var thread = await ThreadParser.TryParse(line); if (mediaEnabledCheckbox.Checked) { statusTextLabel.Text = "Resolving media..."; foreach (var post in thread.Posts) { if (post.File != null) { _total++; if (resolver.DownloadThumbnails && !string.IsNullOrEmpty(post.File.FileThumbUrl)) { _total++; } } } } await pipeline.Process(thread); } catch (Exception ex) { MessageBox.Show(ex.ToString(), $"Exception thrown while processing {line}", MessageBoxButtons.OK, MessageBoxIcon.Error); } } Reset(); }
public IActionResult GetPlayers(string id) { var result = ThreadParser.GetPlayers(id).ToList(); return(this.Ok(result)); }