private static void check(string githubProblemURL, string githubProblemID, string githubProblemExt, HashSet <string> ignore, HashSet <string> IDsAddedToTable, SortedDictionary <string, SortedSet <KattisProblem> > table, Dictionary <string, string> idToNameMap)
{
log.WriteLine("Checking: " + githubProblemID + githubProblemExt);
// check if valid extension
if (!Extensions.names.ContainsKey(githubProblemExt))
{
log.WriteLine("Found invalid extention '" + githubProblemExt + "'");
return;
}
// check if need to ignore the problem
if (ignore.Contains(githubProblemID) || ignore.Contains(githubProblemExt) || ignore.Contains(githubProblemID + githubProblemExt))
{
log.WriteLine("Ignored " + githubProblemID + githubProblemExt);
return;
}
// check if proper formatting
if (!Regex.IsMatch(githubProblemID + githubProblemExt, @"^[a-zA-Z\d]+\.[a-zA-Z\d]+$"))
{
log.WriteLine("Incorrect format found '" + githubProblemID + githubProblemExt + "'");
return;
}
// check if this ID is already in the table
if (IDsAddedToTable.Contains(githubProblemID))
{
// use foreach loop since the key value of table is a SortedSet, and you cannot access indexes.
// could be improved but most problems dont have the same name, so doesn't impact performance enough
foreach (KattisProblem p in table[idToNameMap[githubProblemID]])
{
if (p.id == githubProblemID)
{
string actualLang = Extensions.names[githubProblemExt];
// check if the actual language is already recorded - if not, add it; otherwise check more stuff
if (p.langs.ContainsKey(actualLang))
{
// check if the extension is already in - if it is, dont re-add it; otherwise add it
if (!p.langs[actualLang].ContainsKey(githubProblemExt))
{
p.langs[actualLang].Add(githubProblemExt, githubProblemURL);
}
}
else
{
p.langs.Add(actualLang, new SortedDictionary <string, string> {
{ githubProblemExt, githubProblemURL }
});
}
break;
}
}
}
else
{
string kattisName = string.Empty;
// this is true if user hasn't updated their KattisIDNameMapping.txt
if (!idToNameMap.ContainsKey(githubProblemID))
{
Stream kattisStream = getURLStream("https://open.kattis.com/problems/" + githubProblemID);
// this is true if the ID is a nonexistent kattis problem
if (kattisStream == null)
{
log.WriteLine("Invalid link found 'https://open.kattis.com/problems/" + githubProblemID + "'");
return;
}
log.WriteLine("Recommendation: You should update KattisIDNameMapping.txt to ensure fast speeds");
CustomReader kattisReader = new CustomReader(kattisStream);
string line;
while ((line = kattisReader.NextLine()) != null)
{
// a line with "headline-wrapper" has the name of the problem in it
if (!line.Contains("headline-wrapper", StringComparison.Ordinal))
{
continue;
}
assignKattisName(line, out kattisName);
break;
}
kattisStream.Close();
kattisReader.Close();
// fix weird html that causes apostrophes to be '
if (kattisName.Contains("'", StringComparison.Ordinal))
{
kattisName = kattisName.Replace("'", "'");
}
}
else
{
kattisName = idToNameMap[githubProblemID];
}
IDsAddedToTable.Add(githubProblemID);
SortedDictionary <string, SortedDictionary <string, string> > langs = new SortedDictionary <string, SortedDictionary <string, string> > {
{ Extensions.names[githubProblemExt], new SortedDictionary <string, string> {
{ githubProblemExt, githubProblemURL }
} }
};
KattisProblem prob = new KattisProblem(githubProblemID, kattisName, langs);
// check if the name is already in the table since some problems can have the same names but different IDs
if (table.ContainsKey(kattisName))
{
table[kattisName].Add(prob);
}
else
{
table.Add(kattisName, new SortedSet <KattisProblem> {
prob
});
}
}
}
public static void Main(string[] args)
{
log.StartTimer();
Dictionary <string, string> idToNameMap = new Dictionary <string, string> (2500);
const string mapfile = @"KattisIDNameMapping.txt";
if (!File.Exists(mapfile))
{
log.WriteLine("Recommendation: You should run KattisMapGenerator.dll to ensure fast speeds");
File.Create(mapfile).Close();
}
CustomReader sc = new CustomReader(mapfile);
string line;
while ((line = sc.NextLine()) != null)
{
string id = line;
string name = sc.NextLine();
if (name == null)
{
throw new EndOfStreamException("Missing name for ID '" + id + "'. To fix this, you could delete KattisIDNameMapping.txt and run KattisMappingProgram.cs, or manually fix it by editing KattisIDNameMapping.txt.");
}
idToNameMap.Add(id, name);
}
const string file = @"Config.txt";
if (!File.Exists(file))
{
log.WriteLine("No Config.txt found");
File.Create(file).Close();
}
HashSet <string> ignore, urls;
Stack <Tuple <string, Queue <string> > > folders;
ignore = new HashSet <string> (10);
urls = new HashSet <string> (5);
folders = new Stack <Tuple <string, Queue <string> > > ();
KattisState state = KattisState.NONE;
sc = new CustomReader(file);
while ((line = sc.NextLine()) != null)
{
// skip blank lines
if (line.Length == 0)
{
continue;
}
if (!isKattisState(line, ref state))
{
if (state == KattisState.IGNORE)
{
if (line.Length > 0 && Regex.IsMatch(line, @"^[a-zA-Z\d]*(\.[a-zA-Z\d]+)?$"))
{
if (!ignore.Contains(line))
{
ignore.Add(line);
}
}
else
{
log.WriteLine("'" + line + "' does not match IGNORE regex (^[a-zA-Z\\d]*(\\.[a-zA-Z\\d]+)?$). Valid examples (ensure they're on separate lines and no commas): 2048, .cpp, abc.java");
throw new InvalidDataException("'" + line + "' does not match IGNORE regex (^[a-zA-Z\\d]*(\\.[a-zA-Z\\d]+)?$). Valid examples (ensure they're on separate lines and no commas): 2048, .cpp, abc.java");
}
}
else if (state == KattisState.URL)
{
if (!line.StartsWith("https://github.com", StringComparison.Ordinal))
{
log.WriteLine("'" + line + "' does not begin with https://github.com.");
throw new InvalidDataException("'" + line + "' does not begin with https://github.com.");
}
if (!urls.Contains(line))
{
urls.Add(line);
}
}
else if (state == KattisState.FOLDER)
{
if (line.StartsWith("to:", StringComparison.OrdinalIgnoreCase))
{
if (line.Length > 3)
{
folders.Push(new Tuple <string, Queue <string> > (line.Substring(3), new Queue <string> ()));
}
else
{
log.WriteLine("'to:' requires a folder");
throw new InvalidDataException("'to:' requires a folder");
}
}
else
{
folders.Peek().Item2.Enqueue(line);
}
}
else
{
log.WriteLine(line + " | First line read should be an action.");
throw new InvalidDataException(line + " | First line read should be an action.");
}
}
}
sc.Close();
SortedDictionary <string, SortedSet <KattisProblem> > table = new SortedDictionary <string, SortedSet <KattisProblem> > ();
HashSet <string> IDsAddedToTable = new HashSet <string> ();
// folder check
while (folders.Count > 0)
{
Tuple <string, Queue <string> > t = folders.Pop();
string loc = t.Item1;
// try to replace the tree to blob in link
// example: https://github.com/MiniDomo/Kattis/tree/master/Java
// to https://github.com/MiniDomo/Kattis/blob/master/Java/
Match match = Regex.Match(loc, "^https://github.com/[^/]*/[^/]*/tree/");
if (match.Success)
{
string res = match.Value;
int len = res.Length;
res = res.Substring(0, len - 5) + "blob/";
loc = res + loc.Substring(len) + "/";
}
else
{
log.WriteLine("Invalid link for FOLDER: " + loc + " did not match regex ^https://github.com/[^/]*/[^/]*/tree/ resulting in skipping the link.");
continue;
}
while (t.Item2.Count > 0)
{
string folder = t.Item2.Dequeue();
DirectoryInfo dir = new DirectoryInfo(folder);
if (!dir.Exists)
{
log.WriteLine(folder + " was not found, resulting in being skipped");
continue;
}
foreach (FileInfo info in dir.GetFiles())
{
string githubProblemURL, githubProblemID, githubProblemExt;
// check if proper formatting
if (!Regex.IsMatch(info.Name, @"^[a-zA-Z\d]+\.[a-zA-Z\d]+$"))
{
log.WriteLine("File " + info.Name + " does not match ^[a-zA-Z\\d]+\\.[a-zA-Z\\d]+$ regex, skipping");
continue;
}
int periodPos = info.Name.IndexOf(".", StringComparison.Ordinal);
githubProblemID = info.Name.Substring(0, periodPos);
githubProblemExt = info.Name.Substring(periodPos);
githubProblemURL = loc + githubProblemID + githubProblemExt;
check(githubProblemURL, githubProblemID, githubProblemExt, ignore, IDsAddedToTable, table, idToNameMap);
}
}
}
// folder end
// URL check
if (urls.Count > 0)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)3072;
foreach (string link in urls)
{
// check if it's an actual URL like https://www.google.com/ and not <sfghsdh-3wtsdfg>asd,.sg
if (!Uri.IsWellFormedUriString(link, UriKind.Absolute))
{
continue;
}
Stream githubStream = getURLStream(link);
// true if github link is nonexistent
if (githubStream == null)
{
log.WriteLine("Invalid link found '" + link + "'");
continue;
}
CustomReader githubReader = new CustomReader(githubStream);
while ((line = githubReader.NextLine()) != null)
{
// indicates that we've reached towards the bottom of the page where nothing below it will have problems/solutions to account for
if (line.Contains("model-backdrop", StringComparison.Ordinal))
{
break;
}
// if the line does have 'a class="js-navigation-open"', then it has the information we need
if (!line.Contains("a class=\"js-navigation-open\""))
{
continue;
}
string githubProblemURL, githubProblemID, githubProblemExt;
assignURLandIDandLang(line, out githubProblemURL, out githubProblemID, out githubProblemExt);
check(githubProblemURL, githubProblemID, githubProblemExt, ignore, IDsAddedToTable, table, idToNameMap);
}
githubStream.Close();
githubReader.Close();
}
}
// URL end
string output = "# Kattis Solutions\nSome solutions may be outdated and could be improved.\n\n";
string other = "";
int count = 0;
foreach (KeyValuePair <string, SortedSet <KattisProblem> > pair in table)
{
foreach (KattisProblem p in pair.Value)
{
other += p.ToString() + "\n";
count++;
}
}
output += "| " + count + " Problem" + (count == 1 ? "" : "s") + " | Languages |\n| - | - |\n";
const string readme = @"README.md";
StreamWriter dc = new StreamWriter(readme, false, UnicodeEncoding.Default, 1 << 16);
dc.Write(output + other);
dc.Close();
log.StopTimer();
log.WriteLine("Program finished");
log.Close();
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}