internal static async Task DownloadImdbAsync()
{
string[] imdbFiles = Directory.GetFiles(ImdbDirectory, $"*{JsonExtension}").Select(Path.GetFileNameWithoutExtension).OrderBy(file => file).ToArray();
await Directory
.GetFiles(ItemsDirectory)
.ForEachAsync(async file =>
{
CQ cqPage = CQ.CreateDocumentFromFile(file);
string url = cqPage.Find(@"a.icon[title=""IMDb Rating""]").Attr <string>("href");
string imdbId = Path.GetFileName(new Uri(url).LocalPath.TrimEnd('/'));
if (imdbFiles.Any(existingFile => existingFile.Split(".").First().Equals(Path.GetFileNameWithoutExtension(file))))
{
return;
}
using WebClient webClient = new WebClient();
string imdbHtml = await webClient.DownloadStringTaskAsync(url);
CQ cqImdb = new CQ(imdbHtml);
string imdbJson = cqImdb.Find(@"script[type=""application/ld+json""]").Text();
JsonDocument document = JsonDocument.Parse(imdbJson);
string contentRating = document.RootElement.TryGetProperty("contentRating", out JsonElement ratingElement)
? ratingElement.GetString()
: DefaultContentRating;
if (string.IsNullOrWhiteSpace(contentRating))
{
contentRating = DefaultContentRating;
}
string imdbFile = Path.Combine(ImdbDirectory, $"{Path.GetFileNameWithoutExtension(file)}.{imdbId}.{contentRating}{HtmlExtension}");
await File.WriteAllTextAsync(imdbFile, imdbHtml);
string jsonFile = Path.Combine(ImdbDirectory, $"{Path.GetFileNameWithoutExtension(file)}.{imdbId}.{contentRating}{JsonExtension}");
await File.WriteAllTextAsync(jsonFile, imdbJson);
});
}
internal static async Task SaveYtsSpecialTitles()
{
string[] ytsTitles = Directory
.GetFiles(ItemsDirectory)
.Select(file =>
{
string url = CQ.CreateDocumentFromFile(file)?.Find(@"a.icon[title=""IMDb Rating""]")?.Attr <string>("href")?.Replace("../../external.html?link=", string.Empty) ?? string.Empty;
if (Uri.TryCreate(url, UriKind.Absolute, out Uri uri))
{
try
{
string imdbId = Path.GetFileName(uri?.LocalPath?.TrimEnd('/')) ?? string.Empty;
if (!Regex.IsMatch(imdbId, @"tt[0-9]+"))
{
Trace.WriteLine($"Invalid IMDB Id: {file}, {url}.");
}
return(imdbId);
}
catch
{
Trace.WriteLine($"Invalid IMDB Id: {file}, {url}.");
return(string.Empty);
}
}
else
{
Trace.WriteLine($"Invalid IMDB Id: {file}, {url}.");
return(string.Empty);
}
})
.Where(imdbId => !string.IsNullOrWhiteSpace(imdbId))
.ToArray();
string json = JsonSerializer.Serialize(ytsTitles, new JsonSerializerOptions()
{
WriteIndented = true
});
await File.WriteAllTextAsync(Path.Combine(DocumentsDirectory, $"YtsTitles{JsonExtension}"), json);
}
public void LauchSite()
{
var doc = CQ.CreateDocumentFromFile(File2Parse.FullName);
doc.Find("style").Remove();
doc.Find("link").Remove();
foreach (var script in this.Scripts)
{
doc.Find("body").Append(script.AbsoluteUri);
}
doc.Find("body").Append(
CQ.CreateFragment("<script/>")
.Attr("type", "text/javascript")
.Text(@"
var window.WDC=" + (Json.Encode(new Dictionary <string, string>
{
{ "ServiceUrl", "http://localhost:" + this.ServerPort + "/" },
{ "Identity", this.Identity }
})) + ";")
).Append(CQ.CreateFragment("<script/>")
.Attr("type", "text/javascript")
.Text(File.ReadAllText("WDC.js")))
.Append(CQ.CreateFragment("<script/>")
.Attr("type", "text/javascript")
.Text(" window." + this.Site.CallBackName + "();"));
doc.Save(File2Parse.FullName);
Browser = new WebBrowser();
Browser.Navigate(File2Parse.FullName);
while (IsRunning)
{
Thread.Sleep(100);
}
}
public static void GenerateMap(string username, string board, string outputFile)
{
Console.WriteLine("1. Getting pins...");
var dom = new CQ();
try
{
dom = CQ.CreateFromUrl(string.Format("http://pinterest.com/{0}/{1}/", username, board));
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ProtocolError)
{
Console.WriteLine("User or board could not be found");
}
Environment.Exit(0);
}
StringBuilder builder = new StringBuilder();
builder.AppendLine("function points() {");
int count = dom[".pin"].Count();
IList <string> unclearPins = new List <string>();
Console.WriteLine("2. {0} pins found, gettings positions... (this could take a while depending on the number of pins)", count);
Console.WriteLine();
dom[".pin"].Each((i, e) =>
{
// 2 seconde slapen om limiet geocoding te voorkomen
Thread.Sleep(2000);
var elem = CQ.Create(e);
var pinDesc = elem.Find("p.description").First().Text();
var pinImg = elem.Find("img.PinImageImg").First().Attr("src");
try
{
var location = GetLatLongFromAddress(pinDesc);
builder.AppendFormat("var marker{0} = new google.maps.Marker({{position: new google.maps.LatLng({1}), map: map, title: '{2}'}});\n", i, location, pinDesc);
builder.AppendFormat("google.maps.event.addListener(marker{0}, 'click', function() {{\n", i);
builder.AppendFormat(" var info{0} = new google.maps.InfoWindow({{content: '<img src=\"{1}\">', disableAutoPan: true}});\n", i, pinImg);
builder.AppendFormat(" info{0}.open(map, marker{0});\n", i);
builder.AppendLine("});");
}
catch (Exception err)
{
unclearPins.Add(pinDesc);
}
int percentage = (int)((((i + 1) * 1d) / (count * 1d)) * 100);
RenderConsoleProgress(percentage, '\u2592', ConsoleColor.Green, string.Format("{0} of {1} done ({2}%)", i + 1, count, percentage));
});
Console.WriteLine();
Console.WriteLine();
Console.WriteLine();
if (unclearPins.Count > 0)
{
Console.WriteLine("The following pins could not be located on a map:");
foreach (string unclearPin in unclearPins)
{
Console.WriteLine(string.Format("- {0}", unclearPin));
}
}
builder.AppendLine("}");
string combine = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, InputFile);
var doc = CQ.CreateDocumentFromFile(combine);
doc["script#points"].Text(builder.ToString());
doc.Save(outputFile);
ConsoleColor originalColor = Console.ForegroundColor;
Console.Write("3. Done, results written to ");
Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write(outputFile);
Console.ForegroundColor = originalColor;
Console.WriteLine("!");
}
public override String get()
{
CQ dom = CQ.CreateDocumentFromFile(file);
return(dom["rss channel item title"].First().Text());
}
/// <summary>
/// Opens the file denoted by path, and parses the Open Graph parameters
/// inside.
/// </summary>
/// <param name="path">Path to the file.</param>
/// <returns>OpenGraph data.</returns>
public OpenGraphData ParseFromPath(string path)
{
return(this.ParseDocument(CQ.CreateDocumentFromFile(path)));
}
