public static void ParseRankingNicoChartOnePage(string html, List <Video> video_list)
{
int index = 0;
while ((index = html.IndexOf("<li id=\"rank", index)) >= 0)
{
Video video = new Video();
string view_str = IJStringUtil.GetStringBetweenTag(ref index, "em", html);
string res_str = IJStringUtil.GetStringBetweenTag(ref index, "em", html);
string mylist_str = IJStringUtil.GetStringBetweenTag(ref index, "em", html);
video.point.view = IJStringUtil.ToIntFromCommaValue(view_str);
video.point.res = IJStringUtil.ToIntFromCommaValue(res_str);
video.point.mylist = IJStringUtil.ToIntFromCommaValue(mylist_str);
int ps = html.IndexOf("watch/", index) + 6;
int pe = html.IndexOf('"', ps);
video.video_id = html.Substring(ps, pe - ps);
video.title = IJStringUtil.GetStringBetweenTag(ref index, "a", html);
index = html.IndexOf("<li class=\"release", index) - 1;
string date_str = IJStringUtil.GetStringBetweenTag(ref index, "li", html);
video.submit_date = NicoUtil.StringToDate(date_str);
if (RankFile.SearchVideo(video_list, video.video_id) < 0)
{
video_list.Add(video);
}
}
}
public void MakeDup(string rank_file_diff1_path, string rank_file_diff2_path, InputOutputOption iooption, RankingMethod ranking_method)
{
if (!File.Exists(rank_file_diff1_path))
{
throw new Exception("ランクファイル(1)が存在しません。");
}
if (!File.Exists(rank_file_diff2_path))
{
throw new Exception("ランクファイル(2)が存在しません。");
}
msgout_.Write("重複チェック中…\r\n");
RankFile rank_file_diff1 = new RankFile(rank_file_diff1_path, iooption.GetRankFileCustomFormat());
RankFile rank_file_diff2 = new RankFile(rank_file_diff2_path, iooption.GetRankFileCustomFormat());
List <Video> video_list_diff2 = rank_file_diff2.GetVideoList();
List <Video> video_list = new List <Video>();
for (int i = 0; i < rank_file_diff1.Count; ++i)
{
Video video = rank_file_diff1.GetVideo(i);
int index = RankFile.SearchVideo(video_list_diff2, video.video_id);
if (index >= 0)
{
video_list.Add(video);
}
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
rank_file.Sort(ranking_method);
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("重複チェックが終了しました。\r\n");
}
public void SortRankFile(InputOutputOption iooption, RankingMethod ranking_method)
{
RankFile rank_file = iooption.GetRankFile();
rank_file.Sort(ranking_method);
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("再計算しました。\r\n");
}
public void Merge(RankFile rank_file)
{
for (int i = 0; i < rank_file.video_list_.Count; ++i)
{
if (RankFile.SearchVideo(video_list_, rank_file.video_list_[i].video_id) < 0)
{
video_list_.Add(rank_file.video_list_[i]);
}
}
}
public void MergeRankFileB(string rank_file_diff1_path, string rank_file_diff2_path, InputOutputOption iooption, RankingMethod ranking_method)
{
bool exists_rank_file1 = File.Exists(rank_file_diff1_path);
bool exists_rank_file2 = File.Exists(rank_file_diff2_path);
if (!exists_rank_file1 && !exists_rank_file2)
{
throw new Exception("ランクファイル(1),(2)が存在しません。");
}
msgout_.Write("マージ中…\r\n");
if (!exists_rank_file1 && rank_file_diff1_path != "")
{
msgout_.WriteLine("ランクファイル(1)は存在しません。");
}
if (!exists_rank_file2 && rank_file_diff2_path != "")
{
msgout_.WriteLine("ランクファイル(2)は存在しません。");
}
RankFile rank_file_diff1 = (exists_rank_file1 ? new RankFile(rank_file_diff1_path, iooption.GetRankFileCustomFormat()) : null);
RankFile rank_file_diff2 = (exists_rank_file2 ? new RankFile(rank_file_diff2_path, iooption.GetRankFileCustomFormat()) : null);
List <Video> video_list = new List <Video>();
if (exists_rank_file1)
{
for (int i = 0; i < rank_file_diff1.Count; ++i)
{
Video video = rank_file_diff1.GetVideo(i);
if (RankFile.SearchVideo(video_list, video.video_id) < 0)
{
video_list.Add(video);
}
}
}
if (exists_rank_file2)
{
for (int i = 0; i < rank_file_diff2.Count; ++i)
{
Video video = rank_file_diff2.GetVideo(i);
if (RankFile.SearchVideo(video_list, video.video_id) < 0)
{
video_list.Add(video);
}
}
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
rank_file.Sort(ranking_method);
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("マージが終了しました。\r\n");
}
// ランキングまたはマイリストのRSSを解析する
// is_mylist が true ならマイリスト、false ならランキング
public static void ParsePointRss(string html, DateTime getting_dt, List <Video> video_list, bool is_total, bool is_mylist)
{
int index = -1;
while ((index = html.IndexOf("<item>", index + 1)) >= 0)
{
Video video = new Video();
video.point.getting_date = getting_dt;
video.title = IJStringUtil.GetStringBetweenTag(ref index, "title", html);
if (!is_mylist)
{
video.title = video.title.Substring(video.title.IndexOf('：') + 1);
}
string link = IJStringUtil.GetStringBetweenTag(ref index, "link", html);
video.video_id = link.Substring(link.LastIndexOf('/') + 1);
IJStringUtil.GetStringBetweenTag(ref index, "p", html);
video.description = IJStringUtil.GetStringBetweenTag(ref index, "p", html);
if (!is_mylist) // 読み飛ばし
{
IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
}
video.length = IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
string date_str = IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
video.submit_date = NicoUtil.StringToDate(date_str);
if (!is_mylist)
{
IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
if (!is_total)
{
for (int i = 0; i < 4; ++i) // 4回読み飛ばす
{
IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
}
}
}
string view_str = IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
string res_str = IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
string mylist_str = IJStringUtil.GetStringBetweenTag(ref index, "strong", html);
video.point.view = IJStringUtil.ToIntFromCommaValue(view_str);
video.point.res = IJStringUtil.ToIntFromCommaValue(res_str);
video.point.mylist = IJStringUtil.ToIntFromCommaValue(mylist_str);
if (RankFile.SearchVideo(video_list, video.video_id) < 0)
{
video_list.Add(video);
}
}
}
public void OutputRankFile(RankFile rank_file, RankingMethod ranking_method)
{
if (is_output_from_file_)
{
rank_file.Save(output_path_, ranking_method);
}
else
{
if (OnOutputRankFile != null)
{
OnOutputRankFile(rank_file.ToString(ranking_method));
}
}
}
public void AnalyzeRankingNicoChart(InputOutputOption iooption, RankingMethod ranking_method, string ranking_dir, DateTime start_date, DateTime end_date)
{
msgout_.Write("ランキング解析中…\r\n");
List <Video> video_list = ParseRankingNicoChart(ranking_dir, start_date.Date, end_date.Date);
if (ranking_method.sort_kind != SortKind.Nothing)
{
video_list.Sort(ranking_method.GetComparer());
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("ランキングの解析が終了しました。\r\n");
}
public void UpdatePoint(string rank_file_diff1_path, string rank_file_diff2_path, InputOutputOption iooption, RankingMethod ranking_method)
{
bool exists_rank_file1 = File.Exists(rank_file_diff1_path);
if (!exists_rank_file1)
{
throw new Exception("ランクファイル(1)が存在しません。");
}
bool exists_rank_file2 = File.Exists(rank_file_diff2_path);
if (!exists_rank_file2)
{
throw new Exception("ランクファイル(2)が存在しません。");
}
msgout_.Write("ポイント更新中…\r\n");
RankFile rank_file_diff1 = new RankFile(rank_file_diff1_path, iooption.GetRankFileCustomFormat());
RankFile rank_file_diff2 = new RankFile(rank_file_diff2_path, iooption.GetRankFileCustomFormat());
List <Video> video_list_diff2 = rank_file_diff2.GetVideoList();
List <Video> video_list = new List <Video>();
for (int i = 0; i < rank_file_diff1.Count; ++i)
{
Video video = rank_file_diff1.GetVideo(i);
int index = RankFile.SearchVideo(rank_file_diff2.GetVideoList(), video.video_id);
if (index >= 0)
{
video.title = video_list_diff2[index].title;
video.tag_set = video_list_diff2[index].tag_set;
video.point = video_list_diff2[index].point;
video.description = video_list_diff2[index].description;
video_list.Add(video);
}
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
rank_file.Sort(ranking_method);
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("ポイント更新が終了しました。\r\n");
}
public virtual RankFile GetRankFile()
{
if (is_input_from_file_)
{
if (!File.Exists(input_path_))
{
throw new Exception("入力ランクファイルが存在しません。");
}
return(new RankFile(input_path_, custom_format_));
}
else
{
if (is_input_from_stdin_)
{
StreamReader reader = null;
try
{
reader = new StreamReader(System.Console.OpenStandardInput());
input_text_ = reader.ReadToEnd();
}
finally
{
if (reader != null)
{
reader.Close();
}
}
is_input_from_stdin_ = false;
}
RankFile rank_file = new RankFile(custom_format_);
string separator = custom_format_.GetInputSeparator();
if (input_text_.IndexOf(separator) >= 0)
{
rank_file.Parse(input_text_);
}
else
{
rank_file.ParseFromIdList(input_text_);
}
return(rank_file);
}
}
public void CalculateSum(InputOutputOption iooption, RankingMethod ranking_method)
{
RankFile rank_file = iooption.GetRankFile();
msgout_.Write("------------------------------------------\r\n");
msgout_.Write("内訳: 再生数, コメント数, マイリスト数, ポイント\r\n");
RankPoint point = new RankPoint();
point.view = point.res = point.mylist = 0;
List <Video> video_list = rank_file.GetVideoList();
List <int> view_list = new List <int>();
List <int> res_list = new List <int>();
List <int> mylist_list = new List <int>();
List <int> point_list = new List <int>();
for (int i = 0; i < video_list.Count; ++i)
{
point += video_list[i].point;
view_list.Add(video_list[i].point.view);
res_list.Add(video_list[i].point.res);
mylist_list.Add(video_list[i].point.mylist);
point_list.Add(video_list[i].point.CalcScore(ranking_method));
}
msgout_.Write("合計: " + point.view + ", " + point.res + ", " + point.mylist + ", " + point.CalcScore(ranking_method) + "\r\n");
if (video_list.Count > 0)
{
msgout_.Write("平均: " + ((double)point.view / video_list.Count).ToString("0.00") + ", " +
((double)point.res / video_list.Count).ToString("0.00") + ", " +
((double)point.mylist / video_list.Count).ToString("0.00") + ", " +
((double)point.CalcScore(ranking_method) / video_list.Count).ToString("0.00") + "\r\n");
msgout_.Write("中央値: " + CalculateMedian(view_list) + ", " + CalculateMedian(res_list) + ", "
+ CalculateMedian(mylist_list) + ", " + CalculateMedian(point_list) + "\r\n");
msgout_.Write("最大値: " + view_list[view_list.Count - 1] + ", " + res_list[res_list.Count - 1] + ", "
+ mylist_list[mylist_list.Count - 1] + ", " + point_list[point_list.Count - 1] + "\r\n");
msgout_.Write("最小値: " + view_list[0] + ", " + res_list[0] + ", "
+ mylist_list[0] + ", " + point_list[0] + "\r\n");
}
}
public void MakeDiff(string rank_file_diff1_path, string rank_file_diff2_path, InputOutputOption iooption, RankingMethod ranking_method, DateTime exclusion_date)
{
if (!File.Exists(rank_file_diff1_path))
{
throw new Exception("ランクファイル(1)が存在しません。");
}
if (!File.Exists(rank_file_diff2_path))
{
throw new Exception("ランクファイル(2)が存在しません。");
}
msgout_.Write("差分を計算中…\r\n");
RankFile rank_file_diff1 = new RankFile(rank_file_diff1_path, iooption.GetRankFileCustomFormat());
RankFile rank_file_diff2 = new RankFile(rank_file_diff2_path, iooption.GetRankFileCustomFormat());
List <Video> video_list_diff2 = rank_file_diff2.GetVideoList();
List <Video> video_list = new List <Video>();
for (int i = 0; i < rank_file_diff1.Count; ++i)
{
Video video = rank_file_diff1.GetVideo(i);
int index = RankFile.SearchVideo(video_list_diff2, video.video_id);
if (index >= 0)
{
video.point -= video_list_diff2[index].point;
video_list.Add(video);
}
else
{
if (exclusion_date <= video.submit_date)
{
video_list.Add(video);
}
}
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
rank_file.Sort(ranking_method);
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("差分を計算しました。\r\n");
}
// is_save_to_rank_file : true ならランクファイル、false ならデータベース
// is_point : true ならポイント方式、false なら実数方式
public void AnalyzeRanking(InputOutputOption iooption, RankingMethod ranking_method, ParseRankingKind kind, string ranking_dir_name)
{
msgout_.Write("ランキング解析中…\r\n");
// ランキングHTMLを全ポイント解析しようとしている場合は警告メッセージを出力
if (kind == ParseRankingKind.TotalPoint && IsRankingHtml(ranking_dir_name))
{
msgout_.Write("ランキングHTML解析では全ポイント解析は使用できません。期間ポイントを選択して解析しなおしてください。\r\n");
return;
}
List <Video> video_list = ParseRanking(ranking_dir_name, DateTime.Now, kind);
if (ranking_method.sort_kind != SortKind.Nothing)
{
video_list.Sort(ranking_method.GetComparer());
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("ランキングの解析が終了しました。\r\n");
}
public void LoadForSpecial(string rank_file_path)
{
string str = "";
try
{
str = IJFile.Read(rank_file_path);
}
catch (System.IO.FileNotFoundException)
{
return;
}
string[] line = IJStringUtil.SplitWithCRLF(str);
if (video_list_ == null)
{
video_list_ = new List <Video>();
}
for (int i = 0; i < line.Length; ++i)
{
string[] s_array = line[i].Split('\t');
if (RankFile.SearchVideo(video_list_, s_array[0]) < 0) // 存在しないなら
{
Video video = new Video();
video.video_id = s_array[0];
try
{
video.point.view = IJStringUtil.ToIntFromCommaValue(s_array[3]);
video.point.res = IJStringUtil.ToIntFromCommaValue(s_array[4]);
video.point.mylist = IJStringUtil.ToIntFromCommaValue(s_array[5]);
video.title = s_array[6];
video.submit_date = NicoUtil.StringToDate(s_array[7]);
video.tag_set.ParseBlank(s_array[14]);
}
catch (System.IndexOutOfRangeException) { }
video_list_.Add(video);
}
}
}
public void MergeRankFileA(string rank_file_diff1_path, string rank_file_diff2_path, InputOutputOption iooption, RankingMethod ranking_method)
{
bool exists_rank_file1 = File.Exists(rank_file_diff1_path);
bool exists_rank_file2 = File.Exists(rank_file_diff2_path);
if (!exists_rank_file1 && !exists_rank_file2)
{
throw new Exception("ランクファイル(1),(2)が存在しません。");
}
msgout_.Write("マージ中…\r\n");
if (!exists_rank_file1 && rank_file_diff1_path != "")
{
msgout_.WriteLine("ランクファイル(1)は存在しません。");
}
if (!exists_rank_file2 && rank_file_diff2_path != "")
{
msgout_.WriteLine("ランクファイル(2)は存在しません。");
}
RankFile rank_file_diff1 = (exists_rank_file1 ? new RankFile(rank_file_diff1_path, iooption.GetRankFileCustomFormat()) : null);
RankFile rank_file_diff2 = (exists_rank_file2 ? new RankFile(rank_file_diff2_path, iooption.GetRankFileCustomFormat()) : null);
List <Video> video_list = new List <Video>();
if (exists_rank_file1)
{
MergeToList(rank_file_diff1, video_list, ranking_method);
}
if (exists_rank_file2)
{
MergeToList(rank_file_diff2, video_list, ranking_method);
}
RankFile rank_file = new RankFile(video_list, iooption.GetRankFileCustomFormat());
rank_file.Sort(ranking_method);
iooption.OutputRankFile(rank_file, ranking_method);
msgout_.Write("マージが終了しました。\r\n");
}
private void MergeToList(RankFile rank_file, List <Video> video_list, RankingMethod ranking_method)
{
for (int i = 0; i < rank_file.Count; ++i)
{
Video video = rank_file.GetVideo(i);
int index = RankFile.SearchVideo(video_list, video.video_id);
if (index >= 0)
{
int point_new = video.point.CalcScore(ranking_method);
int point_old = video_list[index].point.CalcScore(ranking_method);
if (point_new > point_old)
{
video_list.RemoveAt(index);
video_list.Add(video);
}
}
else
{
video_list.Add(video);
}
}
}
public void Parse(string str)
{
string[] line = IJStringUtil.SplitWithCRLF(str);
for (int i = 0; i < line.Length; ++i)
{
if (custom_format_.IsUsingCustomFormat())
{
Video video = custom_format_.GetVideo(line[i]);
video_list_.Add(video);
}
else
{
string[] s_array = line[i].Split('\t');
string[] info = new string[18];
for (int j = 0; j < 18; ++j)
{
info[j] = (j < s_array.Length) ? s_array[j] : "";
}
if (RankFile.SearchVideo(video_list_, info[0]) < 0) // 存在しないなら
{
Video video = new Video();
video.video_id = info[0];
video.point.view = IJStringUtil.ToIntFromCommaValueWithDef(info[2], 0);
video.point.res = IJStringUtil.ToIntFromCommaValueWithDef(info[3], 0);
video.point.mylist = IJStringUtil.ToIntFromCommaValueWithDef(info[4], 0);
video.title = info[8];
if (info[9] != "")
{
video.submit_date = NicoUtil.StringToDate(info[9]);
}
video.pname = info[11];
video.tag_set.Parse(info[12]);
video_list_.Add(video);
}
}
}
}
public void SetRankFile(RankFile rank_file)
{
rank_file_ = rank_file;
}
public InputRankFile(RankFile rank_file) : base(false, false)
{
rank_file_ = rank_file;
}
public void AddMultipleMylist(InputOutputOption iooption, List <string> mylist_id_list, List <int> mylist_count_list)
{
if (mylist_id_list.Count == 0)
{
return;
}
const int try_times = 5;
const int large_wait_time = 50000;
const int large_wait_num = 70;
RankFile rank_file = iooption.GetRankFile();
if (rank_file.Count == 0)
{
msgout_.Write("追加する動画がありません。\r\n");
return;
}
msgout_.Write("マイリストへの追加を開始します。\r\n");
// 最小3秒、最大10秒
int wait_time = Math.Min(Math.Max(rank_file.Count * 1000 * 8 / 10, 3000), 10000);
int total_wait_time = (wait_time * rank_file.Count + large_wait_time * (rank_file.Count / large_wait_num)) / 1000 / 60 + 1;
msgout_.Write(rank_file.Count + "件の追加には推定" + total_wait_time.ToString() + "分かかります。\r\n");
//List<Video> exist_video_list = new List<Video>();
//NicoListManager.ParsePointRss(niconico_network_.GetMylistHtml(mylist_id, true), DateTime.Now, exist_video_list, false, true);
//cancel_object_.Wait(1000);
string mylist_id = mylist_id_list[0];
int count = mylist_count_list[0];
int current_mylist_index = 0;
for (int i = 0; i < rank_file.Count; ++i)
{
//if (RankFile.SearchVideo(exist_video_list, rank_file[i]) >= 0)
//{
// msgout_.Write(rank_file[i] + " はすでに存在します。\r\n");
// continue;
//}
for (int j = 0; j < try_times; ++j)
{
try
{
niconico_network_.AddMylist(mylist_id, rank_file[i]);
msgout_.Write(rank_file[i] + " をマイリストに追加しました。\r\n");
}
catch (NiconicoAddingMylistExistException) // 上でチェックしているが念のためもう一度チェック
{
msgout_.Write(rank_file[i] + " はすでに存在します。\r\n");
}
catch (Exception e)
{
if (j < try_times - 1)
{
msgout_.Write("エラー：" + e.Message + "\r\n3秒後に再試行します。\r\n");
cancel_object_.Wait(3000);
continue;
}
else
{
msgout_.Write("エラー：" + e.Message + "\r\n");
cancel_object_.Wait(3000);
}
}
if (i < rank_file.Count - 1)
{
msgout_.Write("待機しています。\r\n");
// 基本的には wait_time ミリ秒だけ待つが、large_wait_num 回に1回は
// large_wait_time ミリ秒待つ。
if ((i + 1) % large_wait_num == 0)
{
cancel_object_.Wait(large_wait_time, large_wait_time + 1000);
}
else
{
cancel_object_.Wait(wait_time, wait_time + 1000);
}
}
break;
}
--count;
if (count == 0)
{
++current_mylist_index;
if (current_mylist_index >= mylist_id_list.Count)
{
break;
}
mylist_id = mylist_id_list[current_mylist_index];
count = mylist_count_list[current_mylist_index];
}
}
msgout_.Write("マイリストへの追加を終了します。\r\n");
}
