/// <summary>
/// LocalThreadReaderクラスのインスタンスを初期化
/// </summary>
/// <param name="dataParser"></param>
public LocalThreadReader(ThreadParser dataParser)
: base(dataParser)
{
//
// TODO: コンストラクタ ロジックをここに追加してください。
//
}
public async Task TestGuestUserPost(string url)
{
var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo());
var thread = await parser.ParseThreadAsync(new Uri(url), 30, 30);
Assert.Contains(thread.Posts, v => v.IsGuestPost && string.IsNullOrEmpty(v.GuestUsername) == false);
}
/// <summary>
/// LocalThreadStorageクラスのインスタンスを初期化
/// </summary>
/// <param name="cache">基になるキャッシュ情報</param>
/// <param name="parser">データを解析するときに使用するパーサ</param>
/// <param name="formatter">書き込み時に使用するフォーマッタ</param>
/// <param name="enc">書き込み時に使用するエンコーダ</param>
public LocalThreadStorage(Cache cache, ThreadParser parser, ThreadFormatter formatter, Encoding enc)
{
if (cache == null)
{
throw new ArgumentNullException("cache");
}
if (parser == null)
{
throw new ArgumentNullException("parser");
}
if (formatter == null)
{
throw new ArgumentNullException("formatter");
}
if (enc == null)
{
throw new ArgumentNullException("enc");
}
this.cache = cache;
this.dataParser = parser;
this.formatter = formatter;
this.encoding = enc;
this.bufferSize = 4096;
this.isOpen = false;
}
public async Task ReadThreadInfoTest()
{
var parser = new ThreadParser(Guid.Parse("18273ef6-92f8-4f11-818d-7e3b8470307a"));
var info = await parser.ReadThreadInfoAsync();
// HAPPY_PASS
Assert.IsNotNull(info);
}
/// <summary>
/// ThreadReaderBaseクラスのインスタンスを初期化
/// </summary>
/// <param name="parser">読み込み時に解析を行うためのパーサー</param>
protected ThreadReaderBase(ThreadParser parser)
: this()
{
if (parser == null)
{
throw new ArgumentNullException("parser");
}
dataParser = parser;
}
private static void Crawl(string url, string path)
{
var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo());
var task = Task.Run(async() =>
{
var thread = await parser.ParseThreadAsync(new Uri(url));
thread.Serialize(new FileInfo(path));
});
Task.WaitAll(task);
}
public async Task TestPostParser(string url, uint page, uint postuid, bool haslikes, bool hassiganture, bool hasuser)
{
var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo());
var thread = await parser.ParseThreadAsync(new Uri(url.Substring(0, url.IndexOf('#'))), page, page);
var post = thread.Posts.Where(v => v.Uid == postuid).Single();
Assert.Equal <uint>(postuid, post.Uid);
Assert.Equal(url, post.Url);
Assert.True(haslikes ? (post.LikeCount > 0) : true);
Assert.True(post.DislikeCount < 1);
Assert.Equal <bool>(hasuser, post.User != null);
Assert.Equal <bool>(hassiganture, !string.IsNullOrEmpty(post.User?.SignatureHtml));
}
public async Task TestParseThread(string teststr)
{
var parser = new ThreadParser(new NetcupThreadInfo(), new NetcupPostInfo(), new NetcupUserInfo(), new NetcupSearchInfo());
var thread = await parser.ParseThreadAsync(new Uri(teststr), 1, 3);
var postUsers = thread.Posts.Where(v => v.User != null).Select(v => v.User).Distinct();
var userIntersects = thread.Users.Users.Intersect(postUsers);
Assert.Equal("https://forum.netcup.de/sonstiges/smalltalk/1051-das-l%C3%A4ngste-thema/", thread.StartpageUrl);
Assert.True(thread.PageCount >= 383);
Assert.True(thread.CurrentPageNo <= thread.PageCount && thread.CurrentPageNo >= 1);
Assert.True(thread.Uid > 0);
Assert.True(thread.Posts.Count > 0);
Assert.True(thread.Users.Users.Count > 0);
Assert.Equal <int>(postUsers.Count(), userIntersects.Count());
}
private async Task <bool> RegisterNewThreadAsync(string identifier)
{
// register a new thread item
try
{
var parser = new ThreadParser(Guid.Parse(identifier));
var info = await parser.ReadThreadInfoAsync();
if (info == null)
{
return(false);
}
// query the database by the identifer / create a new item if not exist
var model = new ThreadModel
{
Id = info.Id,
Title = info.Title,
AuthorId = info.AuthorId,
CreateOn = info.CreateOn,
ForumId = info.ForumId
};
var tags = Utils.DetectTagsFromTitle(info.Title);
foreach (var name in tags)
{
var tag = _tagRepository.CreateTagIfNotExists(name);
model.Tags.Add(tag);
}
_threadRepository.Create(model);
return(true);
}catch (Exception e)
{
return(false);
}
}
private static int Main(string[] args)
{
void Configuration(ParserSettings with)
{
with.EnableDashDash = true;
with.CaseInsensitiveEnumValues = true;
with.AutoHelp = true;
with.HelpWriter = null;
}
var parser = new Parser(Configuration);
var parsed = parser.ParseArguments <HtmlDumpOptions, JsonDumpOptions, MediaDumpOptions>(args);
var result = 0;
try
{
var resolveMedia = false;
BaseOptions baseOpts = null;
var processors = new List <IProcessor>();
parsed
.WithParsed <HtmlDumpOptions>(opts =>
{
processors.Add(new HtmlGeneratorProcessor(
opts.OutputDirectory,
!opts.DontIncludeCustomPosterNames,
!opts.DontUseColoredNames,
!opts.DontIncludeSameFagCount,
opts.Style,
opts.Prettify));
baseOpts = opts;
resolveMedia = !opts.DontResolveMedia;
})
.WithParsed <JsonDumpOptions>(opts =>
{
processors.Add(new JsonSerializerProcessor(opts.OutputDirectory, opts.Formatted));
baseOpts = opts;
resolveMedia = !opts.DontResolveMedia;
})
.WithParsed <MediaDumpOptions>(opts =>
{
baseOpts = opts;
resolveMedia = true;
})
.WithNotParsed(errs =>
{
var helpText = HelpText.AutoBuild(parsed, h =>
{
h = HelpText.DefaultParsingErrorsHandler(parsed, h);
h.Copyright = "";
h.Heading = "";
h.AutoVersion = false;
h.AddDashesToOption = true;
return(h);
}, e => e, true, 190);
Console.WriteAscii("Mitsuba Archivizer", Color.DodgerBlue);
Console.WriteFormatted("commit {0} @ {1}",
new Formatter(ThisAssembly.Git.Commit, Color.Crimson),
new Formatter(ThisAssembly.Git.Branch, Color.Chartreuse), Color.White);
Console.Write(helpText);
result = 1;
});
if (result != 1)
{
var postCount = 0;
var cursorBackupTop1 = -1;
var cursorBackupLeft1 = -1;
ThreadParser.OnPostCount += count => postCount = count;
ThreadParser.OnPostParsing += idx =>
{
if (cursorBackupTop1 == -1 || cursorBackupLeft1 == -1)
{
cursorBackupTop1 = Console.CursorTop;
cursorBackupLeft1 = Console.CursorLeft;
}
Console.SetCursorPosition(cursorBackupLeft1, cursorBackupTop1);
Console.Write(new string(' ', Console.WindowWidth));
Console.SetCursorPosition(cursorBackupLeft1, cursorBackupTop1);
Console.WriteLine("Parsing post: ({0}/{1})", idx + 1, postCount);
};
var resolver = new MediaResolver(
resolveMedia,
baseOpts.OutputDirectory,
!baseOpts.DontSeperateMediaByExtension,
!baseOpts.DontDownloadThumbnails,
baseOpts.AllowedMediaExtensions,
baseOpts.AllowedThumbnailExtensions);
var mediaCount = 0;
var cursorBackupTop2 = -1;
var cursorBackupLeft2 = -1;
resolver.OnPostWithMediaCount += count => mediaCount = count;
resolver.OnProcessingPostMedia += (post, idx) =>
{
if (cursorBackupTop2 == -1 || cursorBackupLeft2 == -1)
{
cursorBackupTop2 = Console.CursorTop;
cursorBackupLeft2 = Console.CursorLeft;
}
Console.SetCursorPosition(cursorBackupLeft2, cursorBackupTop2);
Console.Write(new string(' ', Console.WindowWidth));
Console.SetCursorPosition(cursorBackupLeft2, cursorBackupTop2);
Console.WriteLine("Downloading media: {0} ({1}/{2})", post.File.FileName, idx + 1, mediaCount);
};
var pipeline = new ProcessorPipeline(processors, resolver);
pipeline.OnProcessorInvoked += name => Console.WriteLine("{0} is working...", name);
foreach (var input in baseOpts.InputSeq)
{
var thread = ThreadParser.TryParse(input).Result;
pipeline.Process(thread).Wait();
}
Console.WriteLine("Done!");
}
}
catch (Exception ex)
{
Console.Error.WriteLine(ex);
result = 1;
}
return(result);
}
public async Task <ContentResult> Thread(int threadId)
{
var chattyThread = await _chattyProvider.GetThread(threadId);
var list = new List <V1RootCommentModel>();
var maxDepth = chattyThread.Posts.Max(x => x.Depth);
var lastCommentAtDepth = new V1CommentModel[maxDepth + 1];
var op = chattyThread.Posts[0];
var v1RootCommentModel =
new V1RootCommentModel
{
Comments = new List <V1CommentModel>(),
ReplyCount = chattyThread.Posts.Count,
Body = op.Body,
Date = op.Date,
Participants = GetParticipants(chattyThread),
Category = op.Category,
LastReplyId = $"{chattyThread.Posts.Max(x => x.Id)}",
Author = op.Author,
Preview = ThreadParser.PreviewFromBody(op.Body),
Id = $"{op.Id}"
};
foreach (var post in chattyThread.Posts.Skip(1))
{
var v1CommentModel =
new V1CommentModel
{
Comments = new List <V1CommentModel>(),
Body = post.Body,
Date = post.Date,
Category = post.Category,
Author = post.Author,
Preview = ThreadParser.PreviewFromBody(post.Body),
Id = $"{post.Id}"
};
if (post.Depth == 1)
{
v1RootCommentModel.Comments.Add(v1CommentModel);
}
else
{
lastCommentAtDepth[post.Depth - 1].Comments.Add(v1CommentModel);
}
lastCommentAtDepth[post.Depth] = v1CommentModel;
}
var response = new V1ThreadModel
{
Comments = new List <V1RootCommentModel> {
v1RootCommentModel
}
};
// the serializer has a maximum depth that is too low, even if you reconfigure it to the hard max.
var sb = new StringBuilder();
var options = new JsonSerializerOptions {
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
sb.Append("{");
AppendProperty(sb, "id", v1RootCommentModel.Id, options);
AppendProperty(sb, "reply_count", v1RootCommentModel.ReplyCount, options);
AppendProperty(sb, "body", v1RootCommentModel.Body, options);
AppendProperty(sb, "date", V1DateTimeOffsetConverter.ToJsonString(v1RootCommentModel.Date), options);
AppendProperty(sb, "participants", v1RootCommentModel.Participants, options);
AppendProperty(sb, "category", V1ModerationFlagConverter.ToJsonString(v1RootCommentModel.Category), options);
AppendProperty(sb, "last_reply_id", v1RootCommentModel.LastReplyId, options);
AppendProperty(sb, "author", v1RootCommentModel.Author, options);
AppendProperty(sb, "preview", v1RootCommentModel.Preview, options);
sb.Append("\"comments\":[{");
AppendProperty(sb, "id", v1RootCommentModel.Id, options);
AppendProperty(sb, "body", v1RootCommentModel.Body, options);
AppendProperty(sb, "date", V1DateTimeOffsetConverter.ToJsonString(v1RootCommentModel.Date), options);
AppendProperty(sb, "category", V1ModerationFlagConverter.ToJsonString(v1RootCommentModel.Category), options);
AppendProperty(sb, "author", v1RootCommentModel.Author, options);
AppendProperty(sb, "preview", v1RootCommentModel.Preview, options);
sb.Append("\"comments\":");
AppendCommentList(sb, v1RootCommentModel.Comments, options);
sb.Append("]}");
return(Content(sb.ToString(), "application/json"));
/// <summary>
/// 指定したパーサを使用してX2chThreadReaderクラスのインスタンスを初期化。
/// </summary>
/// <param name="dataParser"></param>
public X2chThreadReader(ThreadParser dataParser)
: base(dataParser)
{
getGzip = true;
}
/// <summary>
/// パーサを指定してMachiThreadReaderクラスのインスタンスを初期化
/// </summary>
public MachiThreadReader(ThreadParser dataParser)
: base(dataParser)
{
}
private async void archivizeButton_Click(object sender, EventArgs e)
{
if (!threadListTextBox.Lines.Any())
{
return;
}
threadListTextBox.Enabled = false;
mediaGroupBox.Enabled = false;
jsonGroupBox.Enabled = false;
htmlGroupBox.Enabled = false;
archivizeButton.Enabled = false;
outputDirectoryButton.Enabled = false;
var processors = new List <IProcessor>();
if (jsonEnabledCheckbox.Checked)
{
processors.Add(new JsonSerializerProcessor(outputDirTextbox.Text, formattedCheckbox.Checked));
}
if (htmlEnabledCheckbox.Checked)
{
processors.Add(new HtmlGeneratorProcessor(outputDirTextbox.Text, namesCheckbox.Checked, coloredNamesCheckbox.Checked, samefagCheckbox.Checked, (string)styleComboBox.SelectedItem, prettifyCheckbox.Checked));
}
MediaResolver resolver;
if (mediaEnabledCheckbox.Checked)
{
var exts = mediaExtTextbox.Text.Split(',');
var thumbExts = thumbnailExtTextBox.Text.Split(',');
resolver = new MediaResolver(true, outputDirTextbox.Text, groupByExtensionsCheckbox.Checked, thumbnailsEnabledCheckbox.Checked, exts, thumbExts);
resolver.OnPostWithMediaCount += Resolver_OnPostWithMediaCount;
resolver.OnProcessingPostMedia += Resolver_OnProcessingPostMedia;
}
else
{
resolver = new MediaResolver();
}
var pipeline = new ProcessorPipeline(processors, resolver);
pipeline.OnProcessorInvoked += Pipeline_OnProcessorInvoked;
foreach (var line in threadListTextBox.Lines)
{
try
{
statusTextLabel.Text = "Parsing posts...";
var thread = await ThreadParser.TryParse(line);
if (mediaEnabledCheckbox.Checked)
{
statusTextLabel.Text = "Resolving media...";
foreach (var post in thread.Posts)
{
if (post.File != null)
{
_total++;
if (resolver.DownloadThumbnails && !string.IsNullOrEmpty(post.File.FileThumbUrl))
{
_total++;
}
}
}
}
await pipeline.Process(thread);
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), $"Exception thrown while processing {line}", MessageBoxButtons.OK,
MessageBoxIcon.Error);
}
}
Reset();
}
public IActionResult GetPlayers(string id)
{
var result = ThreadParser.GetPlayers(id).ToList();
return(this.Ok(result));
}