void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
{
writer.WriteStartObject();
if (Keywords.Any())
{
writer.WritePropertyName("keywords");
writer.WriteStartArray();
foreach (var item in Keywords)
{
writer.WriteStringValue(item);
}
writer.WriteEndArray();
}
else
{
writer.WriteNull("keywords");
}
if (IgnoreCase != null)
{
writer.WritePropertyName("ignoreCase");
writer.WriteBooleanValue(IgnoreCase.Value);
}
writer.WritePropertyName("@odata.type");
writer.WriteStringValue(ODataType);
writer.WritePropertyName("name");
writer.WriteStringValue(Name);
writer.WriteEndObject();
}
public override string ToString()
{
if (Keywords == null || !Keywords.Any())
{
return(String.Format("{0} {1} {2}", AccessModifier, ReturnType, Name));
}
return(String.Format("{0} {1} {2} {3}", AccessModifier, String.Join(" ", Keywords), ReturnType, Name));
}
public bool HasKeyword(int variation, string keyword)
{
if (ManualVariations.Any(v => v.Id == variation))
{
return(ManualVariations.First(v => v.Id == variation).Keywords.Any(k => k.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) >= 0));
}
return(Keywords.Any(k => k.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) >= 0));
}
public string ToFinalString()
{
//string result = String.Format("{0} {1} class {2}", AccessModifier, String.Join(" ", Keywords), Name);
string result = AccessModifier;
if (Keywords != null && Keywords.Any())
{
result += " " + String.Join(" ", Keywords);
}
result += " " + Type + " " + Name + " ";
if (Inheritance != null && Inheritance != "")
{
result += " : " + Inheritance;
}
if (Interfaces != null && Interfaces.Any())
{
if (Inheritance == null || Inheritance == "")
{
result += " : ";
}
else
{
result += ", ";
}
result += String.Join(", ", Interfaces);
}
result += "\n{\n";
if (Properties != null && Properties.Any())
{
foreach (var item in Properties)
{
result += "\t" + item.ToString() + " { get; set; }\n";
}
result += "\n";
}
if (Methods != null && Methods.Any())
{
foreach (var item in Methods)
{
result += item.ToFinalString() + "\n";
}
result += "\n";
}
result += "}\n";
return(result);
}
public ReplayInfo[] Collect(int startPage, int pagesCount)
{
var watch = Stopwatch.StartNew();
var parallelResult = new BlockingCollection <ReplayInfo>();
var replayPagesUrls = new BlockingCollection <string>();
var parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = 16;
Parallel.ForEach(Enumerable.Range(startPage, pagesCount), parallelOptions,
(page) =>
{
// Build URL for page with replays.
var pageUrl = GetUrlFor(page);
var replaysPageListHtml = GetReplayListPage(pageUrl);
if (replaysPageListHtml != null)
{
List <ReplayInfo> replays = GetReplayInfoList(replaysPageListHtml);
foreach (var replay in replays)
{
parallelResult.Add(replay);
}
}
else
{
Debug.Print("Could not parse HTML page!");
}
});
var result = new List <ReplayInfo>();
foreach (var replay in parallelResult)
{
var allKeywords = new List <string>(DescKeywords.Concat(TitleKeywords));
foreach (var keyword in allKeywords)
{
if (Keywords.Any(s => replay.Description.ToUpper().Contains(s) ||
replay.Title.ToUpper().Contains(s)))
{
if (!result.Contains(replay))
{
result.Add(replay);
}
}
}
}
watch.Stop();
Debug.Print(String.Format("{0} replays has been collected in {1} ms", result.Count, watch.ElapsedMilliseconds));
return(result.ToArray());
}
/// <summary>
/// Translates RankingGroup in xml
/// </summary>
/// <returns>map xml</returns>
public XElement ToXml()
{
return(new XElement(
"RankingGroup",
new XAttribute("Id", Id),
new XAttribute("Version", Version),
Rules,
Keywords.Any() ? new XElement("Keywords", Keywords.Select(_ => new XElement("Keyword", _))) : null,
Categories.Any() ? new XElement("Categories", Categories.Select(_ => new XElement("Category", new XAttribute("Name", _)))) : null,
NotFilteredId != null ? new XAttribute("NotFilteredId", NotFilteredId) : null,
NotFilteredVersion != null ? new XAttribute("NotFilteredVersion", NotFilteredVersion) : null));
}
public Dictionary <string, int> ParseURLS(string htmlContent, int depth)
{
Dictionary <string, string> urls = new Dictionary <string, string> ();
Dictionary <string, int> res = new Dictionary <string, int> ();
var match = Regex.Match(htmlContent, @"(?i)<a .*?href=\""([^\""]+)\""[^>]*>(.*?)</a>");
while (match.Success)
{
urls [match.Groups [1].Value] = Regex.Replace(match.Groups [2].Value, "(?i)<.*?>", "");
match = match.NextMatch();
}
foreach (var link in urls)
{
string href = link.Key;
string linkDescription = link.Value;
if (!string.IsNullOrEmpty(href))
{
bool canBeAdded = true;
if (EscapeWords != null)
{
canBeAdded = !EscapeWords.Any(linkDescription.Contains);
}
if (Keywords != null && !Keywords.Any(linkDescription.Contains))
{
canBeAdded = false;
}
if (canBeAdded)
{
string url = contentStandarlize(href);
if (String.IsNullOrEmpty(url) ||
url.StartsWith("#") ||
url.StartsWith("mailto:", StringComparison.OrdinalIgnoreCase) ||
url.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase))
{
continue;
}
url = URLSdantarlize(url);
if (URLRegexFilter != null && !URLRegexFilter.Any(f => Regex.IsMatch(url, f, RegexOptions.IgnoreCase)))
{
continue;
}
if (!res.Keys.Contains(url))
{
res.Add(url, depth + 1);
}
}
}
}
return(res);
}
public bool Check(T value)
{
if (value == null)
{
return(true);
}
var str = value as string;
if (Keywords.Count() > 0 && Keywords.Any(v => str.ToLower().Contains(v.ToLower())))
{
return(false);
}
return(true);
}
internal override HttpContent GenerateRequestBody()
{
var form = new MultipartFormDataContent();
if (Title != null)
{
form.Add(new StringContent(Title), "title");
}
if (RssSourceUrl != null)
{
form.Add(new StringContent(RssSourceUrl), "rss_source_url");
}
if (ParentDirectoryId != null)
{
form.Add(new StringContent(ParentDirectoryId.Value.ToString()), "parent_dir_id");
}
if (DeleteOldFiles != null)
{
form.Add(new StringContent(DeleteOldFiles.ToString()), "delete_old_files");
}
if (DontProcessWholeFeed != null)
{
form.Add(new StringContent(DontProcessWholeFeed.ToString()), "dont_process_whole_feed");
}
if (Keywords.Any())
{
form.Add(new StringContent(string.Join(",", Keywords)), "keyword");
}
if (UnwantedKeywords.Any())
{
form.Add(new StringContent(string.Join(",", UnwantedKeywords)), "unwanted_keywords");
}
if (Paused != null)
{
form.Add(new StringContent(Paused.ToString()), "paused");
}
return(form);
}
/// <summary>Helper method to get the next token</summary>
Token GetToken()
{
while (Skip.Any(s => s == mText[mIdx]))
{
if (++mIdx >= mText.Length)
{
return(null);
}
}
// Punc
if (Punctuations.Any(p => p == mText[mIdx]))
{
return(new Token(E.Punctuation, $"'{mText[mIdx]}'"));
}
// String
var str = GetString(); if (str != null)
{
return(new Token(E.String, $"\"{str}\""));
}
// Number
var num = GetNumberString(); if (num != null)
{
return(new Token(E.Number, num));
}
// Keyword
int end = mText.IndexOfAny(Delims, mIdx); if (end == -1)
{
return(null);
}
str = mText.Substring(mIdx, end - mIdx); mIdx = end - 1;
if (Keywords.Any(k => k == str))
{
return(new Token(E.Keyword, str));
}
// This token does not satisfy any of the above criteria
InvalidToken(str); return(new Token(E.Invalid, str));
}
public string ToFinalString()
{
//result = String.Format("\t{0} {1} {2} {3} ({4})", AccessModifier, String.Join(" ", Keywords), ReturnType, Name, String.Join(", ", Arguments));
string result = "\t" + AccessModifier + " ";
if (Keywords != null || !Keywords.Any())
{
result += String.Join(" ", Keywords) + " ";
}
result += ReturnType + " " + Name + "(";
if (Arguments != null || !Arguments.Any())
{
result += String.Join(", ", Arguments);
}
result += ")\n\t{\n\n\t}";
return(result);
}
private Tuple <string, string> GeneratePost()
{
string PostTitle = string.Empty;
string PostBody = string.Empty;
// Construct PostTitle
PostTitle += "[Spoilers] " + DisplayedTitle + " - " + ShowType + " " + FUNimationEpisodeNumber;
if (FUNimationEpisodeNumber == EpisodeCount)
{
PostTitle += " - FINAL";
}
PostTitle += " [Discussion]";
// Display alternative episode number
if (AKAOffset != 0)
{
PostBody += "**Also known as:** " + ShowType + ' ' + (FUNimationEpisodeNumber + AKAOffset) + "\n\n";
}
// Display episode title if there is one
if (FUNimationEpisodeTitle != string.Empty)
{
PostBody += "**Episode title:** " + FUNimationEpisodeTitle + " \n";
}
// Display episode duration if it's available
if (FUNimationDuration != 0)
{
PostBody += "**Episode duration:** " + SecondsToMinutes(FUNimationDuration) + '\n';
}
// Always show streaming section since the FUNimation link will always be available at this point
// Insert section divider
PostBody += "\n";
PostBody += "**Streaming:** \n";
// Add FUNimation link
PostBody += "**FUNimation:** [" + EscapeString(Title)
+ "](" + EscapeString(FUNimationURL) + ") \n";
foreach (Information Streaming in Streamings)
{
PostBody += "**" + EscapeString(Streaming.Website) + ":** [" + EscapeString(Streaming.Title)
+ "](" + ReplaceProtocol(EscapeString(Streaming.TitleURL), Streaming.Website) + ") \n";
}
// Display information section if it's not empty
if (Informations.Any())
{
// Insert section divider
PostBody += "\n";
PostBody += "**Information:** \n";
foreach (Information Information in Informations)
{
PostBody += "**" + EscapeString(Information.Website) + ":** [" + EscapeString(Information.Title)
+ "](" + ReplaceProtocol(EscapeString(Information.TitleURL), Information.Website) + ") \n";
}
}
// Display subreddits section if it's not empty
if (Subreddits.Any())
{
// Insert section divider
PostBody += "\n";
if (Subreddits.Count == 1)
{
PostBody += "**Subreddit:** /r/" + Subreddits[0] + "\n";
}
else
{
PostBody += "**Subreddits:**\n\n";
foreach (string Subreddit in Subreddits)
{
PostBody += "* /r/" + Subreddit + "\n";
}
}
}
// Insert previous episodes table if it's not empty
PostBody += GenerateTable();
// Display spoiler warning when source material exists
if (SourceExists)
{
// Insert line section divider
PostBody += "\n\n---\n\n";
PostBody += "**Reminder:** \n"
+ "Please do not discuss any plot points which haven't appeared in the anime yet. "
+ "Try not to confirm or deny any theories, encourage people to read the source material instead. "
+ "Minor spoilers are generally ok but should be tagged accordingly. "
+ "Failing to comply with the rules may result in your comment being removed.";
}
if (Keywords.Any())
{
// Insert line section divider
PostBody += "\n\n---\n\n";
PostBody += "**Keywords:** \n";
foreach (string Keyword in Keywords)
{
PostBody += Keyword + ", ";
}
PostBody = PostBody.TrimEnd(new char[] { ',', ' ' });
}
return(Tuple.Create(PostTitle, PostBody));
}
public bool HasKeyword(string keyword)
{
return(Keywords.Any(x => x != null && string.Equals(x.Trim().Replace("~", string.Empty), keyword)));
}
public bool CanParse(string command)
{
return(Keywords.Any(hook => hook.Matches(Normalize(command))));
}
public ReplayInfo[] Collect(int pages)
{
var watch = Stopwatch.StartNew();
var result = new List <ReplayInfo>();
var client = new HttpClient();
bool endOfCollecting = false;
int pageIndex = 0;
var currentUrl = Url;
while (!endOfCollecting)
{
if (pages <= 0)
{
endOfCollecting = true;
continue;
}
var htmlResult = client.GetAsync(currentUrl).Result;
if (htmlResult.StatusCode != System.Net.HttpStatusCode.OK)
{
Console.WriteLine("Could not fetch HTML page (status code = {0}", htmlResult.StatusCode.ToString());
endOfCollecting = true;
continue;
}
var html = htmlResult.Content.ReadAsStringAsync().Result;
List <ReplayInfo> replays = GetReplayInfoList(html);
if (TitleKeywords != null && TitleKeywords.Length > 0)
{
foreach (var replay in replays)
{
foreach (var keyword in TitleKeywords)
{
var keywords = new String[]
{
};
if (keywords.Any(s => replay.Description.ToUpper().Contains(s) ||
replay.Title.ToUpper().Contains(s)))
{
replay.Keywords.Add(keyword);
result.Add(replay);
}
}
}
}
if (DescKeywords != null && DescKeywords.Length > 0)
{
foreach (var replay in replays)
{
foreach (var keyword in DescKeywords)
{
if (Keywords.Any(s => replay.Description.ToUpper().Contains(s) ||
replay.Title.ToUpper().Contains(s)))
{
replay.Keywords.Add(keyword);
result.Add(replay);
}
}
}
}
if ((TitleKeywords == null || TitleKeywords.Length == 0) &&
(DescKeywords == null || DescKeywords.Length == 0))
{
result.AddRange(replays);
}
--pages;
++pageIndex;
Console.WriteLine("Collecting page #{0}. Number of replays collected: {1}\n", pageIndex + 1, result.Count);
/* set up next page to be fetched */
currentUrl = GetUrlFor(pageIndex);
}
client.Dispose();
watch.Stop();
Debug.Print(String.Format("{0} replays has been collected in {1} ms", result.Count, watch.ElapsedMilliseconds));
return(result.ToArray());
}
public bool IsKeyword(KeywordType type, string value) => Keywords.Any(key => key.Type == type && key.Value == value);
