public AttackResult Check(AttackRequest request)
{
var result = new AttackResult();
WebPageRequest webRequest = new WebPageRequest(request.URL);
WebPageLoader.Load(webRequest);
List <string> links = new List <string>();
if (!String.IsNullOrEmpty(webRequest.Response.Body))
{
links = LinkFinder.Parse(webRequest.Response.Body, request.URL);
}
//Do javascript files too, expand library to check load/ajax/etc
foreach (var link in links)
{
var testLink = link.Remove(0, 5);
foreach (var fp in _fingerprints)
{
if (testLink.Contains(fp))
{
result.Success = true;
result.Results.Enqueue("Possible SSRF: " + link);
}
}
}
return(result);
}
public void Run(string url)
{
rtx.Invoke(method: (MethodInvoker) delegate
{
rtx.AppendText(text: $"[{DateTime.Now.ToString("hh:mm:ss tt")}]\t[+] Crawling...\n");
});
try
{
requests = 1;
HttpResponseMessage responseMessage = HttpClient.GetAsync(requestUri: url).Result;
string html = responseMessage.Content.ReadAsStringAsync().Result;
int maxThreads = LinkFinder.Find(html).Count;
if (Threads > maxThreads)
{
rtx.Invoke(method: (MethodInvoker) delegate
{
rtx.AppendText(text: $"[{DateTime.Now.ToString("hh:mm:ss tt")}]\tAlert: Decreasing threads to {maxThreads}\n");
});
}
foreach (var s in LinkFinder.Find(html))
{
new Thread(new ThreadStart(() =>
{
FollowLink(s.Href);
}))
{
IsBackground = true
}.Start();
}
}
catch { };
Crawl(url);
}
private void GatherLinks(string source)
{
foreach (LinkItem i in LinkFinder.Find(source))
{
string u = i.Href;
if (i.Href.StartsWith(value: "/") || i.Href.Contains(value: Url))
{
if (linkDb.IndexOf(item: i.Href) < 0)
{
if (CheckBlackList(url: i.Href))
{
if (GetLinkParameteres(url: i.Href))
{
linkDb.Add(item: i.Href);
if (i.Href.StartsWith("/"))
{
u = "http://" + Url + i.Href;
}
trw.Invoke(method: (MethodInvoker) delegate
{
trw.Nodes.Add(u);
});
FollowLink(url: i.Href);
}
}
}
}
}
}
private release DownloadVersion(release r)
{
try
{
using (var myWebClient = new WebClient())
{
myWebClient.Headers["User-Agent"] = "MOZILLA/5.0 (WINDOWS NT 6.1; WOW64) APPLEWEBKIT/537.1 (KHTML, LIKE GECKO) CHROME/21.0.1180.75 SAFARI/537.1";
string page = myWebClient.DownloadString(r.urlVersion);
foreach (LinkItem i in LinkFinder.Find(page))
{
int q = i.Href.IndexOf("yuzu-windows-mingw");
if (q > 0)
{
string[] ver = i.Href.Split('/', '-');
r.urlFile = i.Href;
r.newversion = ver[8] + "-" + ver[12];
break;
}
}
}
}
catch
{
r.newversion = versionError;
}
return(r);
}
public void FindTest_negative()
{
string content = @"<b href=""https://www.w3schools.com"">Visit W3Schools</b>";
HashSet <string> pageLinks = LinkFinder.Find(content);
bool b = pageLinks.Contains("https://www.w3schools.com");
Assert.IsFalse(b);
}
public void FindTest()
{
string content = @"<a href=""https://www.w3schools.com"">Visit W3Schools</a>" +
@"<a href=""https://www.Link1.com"">Link1</a>";
HashSet <string> pageLinks = LinkFinder.Find(content);
bool b = pageLinks.Contains("https://www.w3schools.com");
bool b1 = pageLinks.Contains("https://www.Link1.com");
Assert.IsTrue(b && b1);
}
private List <string> LoadLink(int 大本等级)
{
Logger.Instance.AddLog(LogType.Debug, "正在获取" + 大本等级 + "本的阵型");
string 网页 = string.Empty;
using (WebClient 卧槽 = new WebClient())
{
卧槽.Encoding = Encoding.UTF8;
卧槽.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36");
try
{
switch (大本等级)
{
case 8:
网页 = 卧槽.DownloadString("https://shimo.im/docs/Ct3c6VYyGvWxJhcg/read");
break;
case 9:
网页 = 卧槽.DownloadString("https://shimo.im/docs/qGKhqyck3Th68ygy/read");
break;
case 10:
网页 = 卧槽.DownloadString("https://shimo.im/docs/PJd8D3JghjXhxCG8//read");
break;
case 11:
网页 = 卧槽.DownloadString("https://shimo.im/docs/WwgChTtHqXCyJGcP/read");
break;
case 12:
网页 = 卧槽.DownloadString("https://shimo.im/docs/vQy9kJQQhr9Vq8v8/read");
break;
case 13:
网页 = 卧槽.DownloadString("https://shimo.im/docs/Gj8wxTcQRwhwhJ69/read");
break;
}
}
catch (Exception ex)
{
Logger.Instance.AddLog(LogType.Error, ex.Message);
}
}
var 链接 = LinkFinder.Find(网页);
List <string> TM的部落链接 = new List <string>();
foreach (var 阵型链接 in 链接)
{
if (阵型链接.Href.Contains("https://link.clashofclans.com/"))
{
TM的部落链接.Add(阵型链接.Href);
}
}
return(TM的部落链接);
}
private void button2_Click(object sender, EventArgs e)
{
string metin = richTextBox1.Text;
listBox1.Items.Clear();
txtSonuclar.Text = "";
// 2.
foreach (LinkItem i in LinkFinder.Find(metin, txtFacebook.Text, txtInstagram.Text))
{
if (i.Text != "Fan Klüb" && i.Text != "Erzurum" && i.Text != "Erzurum'u Keşfe Çıktık!" &&
i.Text != "Osman Çelik" && i.Text != "Said Erzene" && i.Text != "erzurumportali" && i.Text != "erzurumportakali")
{
listBox1.Items.Add(i.Text);
}
}
string[] arr = new string[listBox1.Items.Count];
listBox1.Items.CopyTo(arr, 0);
var arr2 = arr.Distinct();
listBox1.Items.Clear();
foreach (string s in arr2)
{
listBox1.Items.Add(s);
// txtSonuclar.Text += s + Environment.NewLine;
}
label1.Text = "Çekilişe Katılan Kişi Sayısı :" + listBox1.Items.Count;
txtSonuclar.Text += "--------ASILLAR---------" + Environment.NewLine;
txtSonuclar.Text += Environment.NewLine;
for (int j = 1; j <= ndAsil.Value; j++)
{
Random rnd = new Random();
int i = rnd.Next(listBox1.Items.Count);
txtSonuclar.Text += j + " - " + listBox1.Items[i] + Environment.NewLine;
listBox1.Items.Remove(listBox1.Items[i]);
}
txtSonuclar.Text += Environment.NewLine;
txtSonuclar.Text += "--------YEDEKLER---------" + Environment.NewLine;
txtSonuclar.Text += Environment.NewLine;
for (int y = 1; y <= ndYedek.Value; y++)
{
Random rnd = new Random();
int i = rnd.Next(listBox1.Items.Count);
txtSonuclar.Text += y + " - " + listBox1.Items[i] + Environment.NewLine;
listBox1.Items.Remove(listBox1.Items[i]);
}
//Clipboard.SetText(txtSonuclar.Text);
}
private void Main_Load(object sender, EventArgs e)
{
WebClient w = new WebClient();
string s = w.DownloadString("http://readberserk.com/chapter/berserk-chapter-a0/");
foreach (string i in LinkFinder.Find(s))
{
Debug.WriteLine(i);
}
MessageBox.Show("done");
}
static void Main(string[] args)
{
var running = true;
Download(searchUrl) // Invoke a google search.
.Then(html => // Transforms search results and extract links.
{
return(LinkFinder
.Find(html)
.Select(link => link.Href));
})
// Download all extracted links.
.Then(links => Promise <string> .All( // Combine multiple promises into a single async operation.
links
.Where(link => link.StartsWith("http")) // Filter out relative links.
.Select( // Convert collection of links to a collection of promises that are downloading the links.
link => Download(link) // Download each link.
)
))
.Then(htmls => // Display html from the link that was followed.
{
Console.WriteLine("Async operation completed.");
foreach (var html in htmls)
{
Console.WriteLine("---------------");
Console.WriteLine(html.Substring(0, 250) + "...");
}
Console.WriteLine("---------------");
Console.WriteLine("Downloaded " + htmls.Count() + " pages");
running = false;
})
.Catch(exception => // Catch any errors that happen during download or transform.
{
Console.WriteLine("Async operation errorred.");
Console.WriteLine(exception);
running = false;
})
.Done();
Console.WriteLine("Waiting");
while (running)
{
Thread.Sleep(10);
}
Console.WriteLine("Exiting");
}
public void LoadPlugIns(Uri uri)
{
string baseURL = uri.ToString();
var client = new WebClient();
string content = client.DownloadString(baseURL);
foreach (LinkItem i in LinkFinder.Find(content, uri.ToString()))
{
if (i.Href.EndsWith(".dll"))
{
LoadPlugIn(new Uri(i.Href));
}
}
}
public static void Main()
{
// Scrape links from wikipedia.org
// 1.
// URL: http://csgolounge.com/match?m=4961
WebClient w = new WebClient();
string s = w.DownloadString("http://csgolounge.com/match?m=4961");
// 2.
foreach (LinkItem i in LinkFinder.Find(s))
{
Console.WriteLine(i);
}
}
static void Main()
{
// Scrape links from wikipedia.org
// 1.
// URL: http://en.wikipedia.org/wiki/Main_Page
System.Net.WebClient w = new WebClient();
string s = w.DownloadString("http://en.wikipedia.org/wiki/Main_Page");
// 2.
foreach (LinkItem i in LinkFinder.Find(s))
{
Debug.WriteLine(i);
}
}
public static List <string> GetUKE()
{
List <string> linkx = new List <string>();
string website = @"https://bip.uke.gov.pl/pozwolenia-radiowe/wykaz-pozwolen-radiowych-tresci/stacje-gsm-umts-lte-oraz-cdma,12.html";
WebClient wc = new WebClient();
string str = wc.DownloadString(website);
foreach (string li in LinkFinder.Find(str))
{
//Console.WriteLine(li);
linkx.Add(li);
}
//Console.WriteLine(string.Join("\n", linkx));
return(linkx);
}
static void Main(string[] args)
{
List <string> listen = new List <string>();
string textFile = "C:\\textlines.txt";
string[] lines = File.ReadAllLines(textFile);
foreach (string line in lines)
{
string webpageUrl = "https://www.google.com/search?q=" + line.Replace(" ", "+") + "+ oldschoolrunescape";
string targetDomain = "exposureroom.com";
var linkFinder = new LinkFinder();
var links = linkFinder.FindLinksToDomainOnWebPage(webpageUrl, targetDomain);
bool o = false;
foreach (var link in links)
{
if (link.InnerText.Contains("oldschool.runescape") && link.InnerText.Contains("https") && o == false || link.InnerText.Contains("oldschoolrunescape") && link.InnerText.Contains("https") && o == false)
{
o = true;
string toFind1 = "https";
int start = link.InnerText.IndexOf(toFind1) + toFind1.Length;
start = start - 5;
int end = link.InnerText.Length;
string string2 = link.InnerText.Substring(start, end - start);
string2 = string2.Replace(" ", string.Empty);
Console.WriteLine(string2);
listen.Add(string2);
}
else
{
}
}
}
using (TextWriter tw = new StreamWriter("C:\\Users\\dev7\\Desktop\\textlines2.txt"))
{
foreach (String s in listen)
{
tw.WriteLine(s);
}
}
Console.ReadLine();
}
/// <summary>
/// Attempts to extract hot linked and thumb->FullScale images.
/// </summary>
/// <param name="htmlDump">
/// The html Dump.
/// </param>
/// <param name="postId">
/// The Post Id.
/// </param>
/// <returns>
/// The extract images links html.
/// </returns>
public static List <ImageInfo> ExtractImagesLinksHtml(string htmlDump, string postId)
{
if (!string.IsNullOrEmpty(postId) && postId.StartsWith("http://"))
{
postId = postId.Substring(postId.IndexOf("#post") + 5);
}
htmlDump = htmlDump.Replace("&", "&");
// use only message content
var sMessageStart = string.Format("<div id=\"post_message_{0}\">", postId);
var sMessageEnd = "</blockquote>";
var iStart = htmlDump.IndexOf(sMessageStart);
iStart += sMessageStart.Length;
var iEnd = htmlDump.IndexOf(sMessageEnd, iStart);
htmlDump = htmlDump.Substring(iStart, iEnd - iStart);
///////////////////////////////////////////////
// Parse all Links <a>
var rtnList =
LinkFinder.ListAllLinks(htmlDump).Select(
link =>
new ImageInfo
{
ImageUrl = RemoveRedirectLink(Utility.ReplaceHexWithAscii(link.Href)),
ThumbnailUrl = Utility.ReplaceHexWithAscii(link.Text)
}).Where(newPicPoolItem => !Utility.IsImageNoneSense(newPicPoolItem.ImageUrl) && !Utility.IsImageNoneSense(newPicPoolItem.ThumbnailUrl)).ToList();
// Parse all Image <a>
rtnList.AddRange(
LinkFinder.ListAllImages(htmlDump).Select(
link =>
new ImageInfo
{
ImageUrl = RemoveRedirectLink(Utility.ReplaceHexWithAscii(link.Href)),
ThumbnailUrl = Utility.ReplaceHexWithAscii(link.Text)
}));
return(rtnList);
}
void getValuesFromYFinance()
{
WebClient w = new WebClient();
string s = w.DownloadString("https://finance.yahoo.com/quote/CSV/history/?guccounter=1");
// 2.
foreach (csLinkItem i in LinkFinder.Find(s))
{
city1.Add(i.ToString());
}
for (int i = 0; i < city1.Count; i++)
{
numb.Add(CleanStringOfNonDigits_V3(city1[i]));
if (numb[i] == "")
{
numb[i] = "10";
}
}
listBox1.DataSource = numb;
}
/// <summary>
/// extracts the link to the created topic
/// </summary>
/// <param name="responseString">the html code of the codestriker response</param>
/// <returns>the specific link</returns>
private string ExtractUriFromResponse(string responseString)
{
// last item is the link to the topic in Codestriker
try
{
LinkItem topicLinkString = LinkFinder.Find(responseString.ToLower()).Last();
string originalLink = topicLinkString.Text;
if (RequestUri.ToString().Contains(":"))
{
originalLink = m_Regex.Replace(originalLink, $"{RequestUri.OriginalString}?");
}
string linkToProperties = originalLink.Replace("view", "view_topic_properties");
var topicUri = linkToProperties;
return(topicUri);
}
catch (InvalidOperationException e)
{
throw new ArgumentException($"Invalid Argument {nameof(responseString)}", e);
}
}
protected void Page_Load(object sender, EventArgs e)
{
WebClient w = new WebClient();
string s = w.DownloadString("http://en.wikipedia.org/wiki/List_of_current_AFL_team_squads");
StringBuilder sb = new StringBuilder();
foreach (LinkItem i in LinkFinder.Find(s).GetRange(1000, 50))
{
try
{
WebClient ww = new WebClient();
string ss = ww.DownloadString("http://en.wikipedia.org" + i.Href);
//sb.Append(i.Href);
//sb.Append("<br />");
sb2 = LinkFinder.CallMe(ss);
}
catch (Exception exc) { }
}
Literal1.Text = sb2.ToString();
}
static void Main(string[] args)
{
var running = true;
Download(searchUrl) // Invoke a google search.
.Transform(html => // Transforms search results and extract links.
{
return(LinkFinder
.Find(html)
.Select(link => link.Href)
.ToArray());
})
.Catch(exception => // Catch any errors that happen during download or transform.
{
Console.WriteLine("Async operation errorred.");
Console.WriteLine(exception);
running = false;
})
.Then(links => // Display the links that were extracted.
{
Console.WriteLine("Async operation completed.");
foreach (var link in links)
{
Console.WriteLine(link);
}
running = false;
})
.Done();
Console.WriteLine("Waiting");
while (running)
{
Thread.Sleep(10);
}
Console.WriteLine("Exiting");
}
static void Main(string[] args)
{
//PrintDisclaimer();
// Create web client
using (WebClient client = new WebClient())
{
// Set user agent.
client.Headers["User-Agent"] =
"Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko";
//"Mozilla/4.0 (Compatible; Windows NT 5.1; MSIE 6.0) " +
//"(compatible; MSIE 6.0; Windows NT 5.1; " +
//".NET CLR 1.1.4322; .NET CLR 2.0.50727)";
// Accept-encoding headers.
client.Headers["Accept-Encoding"] = "html";
client.Headers["Accept-Encoding"] = "text";
// Download data.
byte[] arr = client.DownloadData("http://blog.mosinu.com/");
string sTr = client.DownloadString("http://blog.mosinu.com/");
// Get response header.
string contentEncoding = client.ResponseHeaders["Content-Encoding"];
// Write values.
//Console.WriteLine("--- WebClient result ---");
PrintRedAttentionText(" *** Ferret result *** ");
foreach (LinkItem i in LinkFinder.Find(sTr))
{
Console.WriteLine(arr.Length);
client.DownloadDataAsync(i);
//Debug.WriteLine(i);
Console.WriteLine(i);
}
Console.ReadLine();
}
}
static void Main(string[] args)
{
var running = true;
Download(searchUrl) // Invoke a google search.
.Transform(html => // Transforms search results and extract links.
{
return(LinkFinder
.Find(html)
.Select(link => link.Href)
.Skip(5)
.First()); // Grab the 6th link.
})
.Then(firstLink => Download(firstLink)) // Follow the first link and download it.
.Catch(exception => // Catch any errors that happen during download or transform.
{
Console.WriteLine("Async operation errorred.");
Console.WriteLine(exception);
running = false;
})
.Then(html => // Display html from the link that was followed.
{
Console.WriteLine("Async operation completed.");
Console.WriteLine(html.Substring(0, 250) + "...");
running = false;
})
.Done();
Console.WriteLine("Waiting");
while (running)
{
Thread.Sleep(10);
}
Console.WriteLine("Exiting");
}
public void Start()
{
try
{
// Foreach link we find on the pastebinarchive page
// TODO: replace this with a htmlagility pack call
foreach (LinkItem LI in LinkFinder.Find(PastebinArchive()))
{
string sHref = StripLinkItem(LI.Href);
// If the stripd down link is excluded, or irrelevant go to the next.
if (string.IsNullOrEmpty(sHref) || exclusionList.Contains(sHref))
{
continue;
}
// Check if we proccesed the link before
if (previous.Add(sHref))
{
_urls.Add(new Uri("http://pastebin.com/raw.php?i=" + sHref));
Stats.ScraperStats.TotalScraped++;
}
else
{
faulty.Add(sHref);
Stats.ScraperStats.TotalFaulty++;
}
}
// Exclude the 8 faulty caused by the 8 featured files
Stats.ScraperStats.TotalFaulty -= 8;
}
catch
{
// Implement 3 strike out system
Console.WriteLine("Could not download archive, due to a time out");
}
}
public void Scrape()
{
try
{
foreach (LinkItem LI in LinkFinder.Find(PastebinArchive()))
{
string sHref = StripLinkItem(LI.Href);
// If the stripd down link is excluded, or irrelevant go to the next.
if (string.IsNullOrEmpty(sHref) || exclusionList.Contains(sHref))
{
continue;
}
// If we did not previously handle this, add it to our current.
if (previous.Add(sHref))
{
current.Add(sHref);
Stats.ScraperStats.TotalScraped++;
}
else
{
faulty.Add(sHref);
Stats.ScraperStats.TotalFaulty++;
}
}
// Exclude the 8 faulty caused by the 8 featured files
Stats.ScraperStats.TotalFaulty -= 8;
DownloadPastes(ref current);
}
catch
{
Console.WriteLine("Could not download archive, due to a time out");
}
}
/// <summary>
/// Attempts to extract hot linked and thumb->FullScale images.
/// </summary>
/// <param name="htmlDump">The html Dump.</param>
/// <param name="postId">The post identifier.</param>
/// <returns>
/// The extract attachment images html.
/// </returns>
public static List <ImageInfo> ExtractAttachmentImagesHtml(string htmlDump, string postId)
{
var rtnList = new List <ImageInfo>();
htmlDump = htmlDump.Replace("&", "&");
var start = "<div class=\"attachments\">";
var end = "<!-- / attachments -->";
// use only message content
var iStart = htmlDump.IndexOf(start, System.StringComparison.Ordinal);
if (iStart < 0)
{
// fix post id
if (postId.Contains("#post"))
{
postId = postId.Substring(postId.IndexOf("#post", System.StringComparison.Ordinal) + 5);
}
start = string.Format("<div id=\"post_message_{0}\">", postId);
end = "</blockquote>";
iStart = htmlDump.IndexOf(start, System.StringComparison.Ordinal);
if (iStart < 0)
{
// Return Empty List
return(rtnList);
}
iStart += start.Length;
var startDump = htmlDump.Substring(iStart);
var iEnd = startDump.IndexOf(end, System.StringComparison.Ordinal);
if (iEnd > 0)
{
htmlDump = startDump.Remove(iEnd);
}
}
else
{
iStart += start.Length;
var iEnd = htmlDump.IndexOf(end, System.StringComparison.Ordinal);
if (iEnd > 0)
{
htmlDump = htmlDump.Substring(iStart, iEnd - iStart);
}
}
///////////////////////////////////////////////
rtnList.AddRange(
LinkFinder.ListAllLinks(htmlDump)
.Select(
link =>
new ImageInfo
{
ImageUrl =
link.Href.StartsWith("http://")
? link.Href
: CacheController.Instance().UserSettings.CurrentForumUrl
+ Utility.ReplaceHexWithAscii(link.Href),
ThumbnailUrl = string.Empty
})
.Where(newPicPoolItem => !Utility.IsImageNoneSense(newPicPoolItem.ImageUrl)));
return(rtnList);
}
private void ParseFiling(IDbConnection db, SyndicationItem item)
{
try
{
// get filing data
EdgarFiling filing = EdgarFiling.Deserialize(item.Content);
filing.Ticker = this._ticker;
// if it's a quarterly or annual report, save info
if (null != filing && filing.FilingType == "10-K" || filing.FilingType == "10-Q")
{
//if (!string.IsNullOrEmpty(filing.FilingDirectory))
if (!string.IsNullOrEmpty(filing.FilingHref))
{
// find root directory of each filing
//RestClient client = new RestClient(filing.FilingDirectory);
RestClient client = new RestClient(filing.FilingHref);
var request = new RestRequest();
request.Method = Method.GET;
var response = client.Execute(request);
string content = response.Content;
foreach (LinkItem i in LinkFinder.Find(content))
{
//TODO not a beauty
if (i.Href.Substring(i.Href.Length - 4, 4) == ".xml")
{
filing.FileName = i.Text;
filing.FilingUrl = "https://www.sec.gov" + i.Href;
filing.PathOnDisk = DownloadXml(filing.FilingUrl, DownloadPath(_basePath, _ticker, filing.FileName));
if (!string.IsNullOrEmpty(filing.PathOnDisk))
{
db.Insert(filing);
}
else
{
ErrorLog.HandleError(this._ticker, "EdgarDownloader", string.Format("Unable to download xml for filing: {0} on ticker {1}", filing.FilingDate, this._ticker));
}
//one item is enough
break;
}
}
//old
//Match match = Regex.Match(content, @"<a href=""([\w]+-[\d]+\.xml)"">[\w]+-[\d]+\.xml</a>", RegexOptions.IgnoreCase);
//if (match.Success)
//{
// filing.FileName = match.Groups[1].Value;
// filing.PathOnDisk = DownloadXml(filing.FilingUrl, DownloadPath(_basePath, _ticker, filing.FileName));
// if (!string.IsNullOrEmpty(filing.PathOnDisk))
// {
// db.Insert(filing);
// }
// else
// {
// ErrorLog.HandleError(this._ticker, "EdgarDownloader", string.Format("Unable to download xml for filing: {0} on ticker {1}", filing.FilingDate, this._ticker));
// }
//}
//else
//{
// ErrorLog.HandleError(this._ticker, "EdgarDownloader", string.Format("Unable to find xml for filing: {0} on ticker {1}", filing.FilingDate, this._ticker));
//}
}
else
{
ErrorLog.HandleError(this._ticker, "EdgarDownloader", string.Format("Unable to find base directory for filing: {0} on ticker {1}", filing.FilingDate, this._ticker));
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
public void RunScrapper()
{
int pageNumber = 1; //starter page number
//create client
WebClient client = new WebClient();
client.Headers.Add(HttpRequestHeader.Cookie, "over18=1"); //add over18 cookie to client
string website = client.DownloadString(address); //download strings from website
OutputWriteLine("Scanning subreddit...");
//page scanner
LinkFinder mainPage = new LinkFinder();
//converted strings
string trimmedLink = null;
string savedLinkName = null;
//loop through pages until max is reached, then end program
while (pageNumber <= maxPages)
{
OutputWriteLine("Page: " + pageNumber);
//loop through links with imgur and download them
foreach (LinkItem i in mainPage.Find(website))
{
//check if link is an imgur link
if (i.ToString().Contains("http://i.imgur.com/") || i.ToString().Contains("https://i.imgur.com/"))
{
//set link var to file link
trimmedLink = i.ToString();
//check if file is a jpg or gif and then save if true
if (SupportedExtension(trimmedLink))
{
//save gifv and gif as webm for smaller files
trimmedLink = trimmedLink.Replace("gifv", "webm");
trimmedLink = trimmedLink.Replace("gif", "webm");
//trim link for save format
savedLinkName = SaveFormat(trimmedLink);
//try to download or write error
DownloadFile(client, trimmedLink, savedLinkName, fileDir);
}
}
//check if link is a gfycat link and filter out links with text
if (i.ToString().Contains("http://gfycat.com/") && !i.ToString().Contains(" ") ||
i.ToString().Contains("https://gfycat.com/") && !i.ToString().Contains(" "))
{
//trim link
trimmedLink = i.ToString();
//try different gfycat webm servers
trimmedLink += ".webm"; //make gfycat a webm
//try different gfycat hosting servers servers
trimmedLink = trimmedLink.Replace("https://", "https://zippy.");
trimmedLink = trimmedLink.Replace("http://", "http://zippy.");
if (!FileExists(trimmedLink))
{
trimmedLink = trimmedLink.Replace("https://zippy.", "https://fat.");
trimmedLink = trimmedLink.Replace("http://zippy.", "http://fat.");
}
if (!FileExists(trimmedLink))
{
trimmedLink = trimmedLink.Replace("https://fat.", "https://giant.");
trimmedLink = trimmedLink.Replace("http://fat.", "http://giant.");
}
if (!FileExists(trimmedLink)) //if file still isnt found
{
OutputWriteLine("ERROR: Could not get hosting server for " + trimmedLink);
}
else
{
savedLinkName = SaveFormat(trimmedLink); //trim link for save format
DownloadFile(client, trimmedLink, savedLinkName, fileDir); //try to download or write error
}
}
//get next page
if (i.ToString().Contains("count=") && i.ToString().Contains("after="))
{
OutputWriteLine("Getting next page...");
pageNumber++;
address = i.Href; //set new address to new page's href
}
}
mainPage = new LinkFinder(); //new linkfinder object for link list
//try to download the next page strings, if execption, exit loop and end
try
{
website = client.DownloadString(address);
}
catch
{
OutputWriteLine("Couldn't Get Next Page");
break;
}
}
//end program after desired pages has been met
OutputWriteLine("Finished Scrapping");
ResetData();
DialogResult message = System.Windows.Forms.MessageBox.Show("Do you wish to see the output folder?", "Scrapper Completed", MessageBoxButtons.YesNo);
if (message == DialogResult.Yes)
{
try
{
Process.Start(fileDir);
ChangeView(Visibility.Visible, Visibility.Hidden);
}
catch
{
DialogResult errorMessage = System.Windows.Forms.MessageBox.Show("Failed to open directory!", "Error!", MessageBoxButtons.OK);
ChangeView(Visibility.Visible, Visibility.Hidden);
}
}
else if (message == DialogResult.No ||
message == DialogResult.Abort)
{
ChangeView(Visibility.Visible, Visibility.Hidden);
}
}
private void UpdatePacketstorm()
{
string totalExploits;
int maxPageNumber;
int[] pageNumbers;
List <string> linksDatabase = new List <string>();
try
{
new Thread(new ThreadStart(() =>
{
UpdateRtx("[+] Downloading database from packetstormsecurity.com... ");
using (WebClient webClient = new WebClient())
{
ErrorCode = 0x1; // error in connecting to site - parsing page and exploits.
webClient.Headers.Add(name: "user-agent",
value: " Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0");
string webpageSource = webClient.DownloadString(address: Packetstorm_Server);
maxPageNumber = int.Parse(Regex.Match(input: webpageSource,
pattern: Page_Number_Pattern).Value.Split(' ')[3]);
pageNumbers = new int[maxPageNumber]; // creating pages number
for (int i = 0; i < maxPageNumber; i++)
{
pageNumbers[i] = i + 1;
}
totalExploits = Regex.Match(input: webpageSource,
pattern: Number_Of_Exploits)
.Value.Split(' ')[1];
UpdateRtx(txt: $"[+] Found {totalExploits} exploits in {maxPageNumber} pages.");
UpdateRtx(txt: $"[+] Reading database...");
UpdateRtx(txt: $"[+] Gathering links: 0");
}
int sleepCounter = 0;
object sleepLock = new object();
ParallelOptions parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = Threads
};
try
{
Parallel.ForEach(source: pageNumbers.ToList(), parallelOptions: parallelOptions, body: pageNumber =>
{
ErrorCode = 0x2;
string link = Packestorm_Page_Address.Replace(oldValue: "$page",
newValue: $"page{pageNumber}");
using (WebClient webClient = new WebClient())
{
webClient.Headers.Add(name: "user-agent",
value: " Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0");
try
{
string webpageSource = webClient.DownloadString(address: link);
foreach (LinkItem i in LinkFinder.Find(file: webpageSource))
{
if (i.ToString().Contains(".html"))
{
if (linksDatabase.IndexOf(item: i.Href) < 0)
{
linksDatabase.Add(i.Href);
Rtx.Invoke((System.Windows.Forms.MethodInvoker) delegate
{
Rtx.Text = (Rtx.Text.Replace(Regex.Match(input: Rtx.Text,
pattern: @".*Gathering links.*").Value, newValue: $"[{DateTime.Now.ToString("hh:mm:ss tt")}]\t[+] Gathering links: {linksDatabase.Count}"));
});
}
}
}
}
catch (Exception ex) { UpdateRtx(txt: $"[-] Error code: {ErrorCode}"); UpdateRtx(txt: $"Error description: Problem in gathering links from page.{ex.Message}"); };
}
lock (sleepLock)
{
sleepCounter++;
}
if (sleepCounter > MaxSleep)
{
Thread.Sleep(Timeout);
lock (sleepLock)
{
sleepCounter = 0;
}
}
});
UpdateRtx(txt: $"[+] Packetstorm Components: {PacketstormList.Count}");
sleepCounter = 0;
object rtxLock = new object();
Parallel.ForEach(source: linksDatabase.ToList(), parallelOptions: parallelOptions, body: link =>
{
using (WebClient webClient = new WebClient())
{
ErrorCode = 0x3; // happens for wrong htmlagility dll version.
try
{
webClient.Headers.Add(name: "user-agent",
value: " Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0");
string webpageSource = webClient.DownloadString(address: $"https://packetstormsecurity.com{link}");
if (Regex.IsMatch(
input: webpageSource,
pattern: Component_Pattern))
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html: webpageSource);
var a = doc.DocumentNode.SelectNodes("//h1").Skip(0).Take(1).Single();
string version = "n\a";
if (Regex.IsMatch(
input: webpageSource,
pattern: Date_Pattern))
{
version = versionDetector.GetVersion(Regex.Match(input: webpageSource, pattern: Date_Pattern).Value);
}
lock (rtxLock)
{
PacketstormList.Add($"{link}:{a.InnerText}:{Regex.Match(input: webpageSource, pattern: Component_Pattern).Value}:{version}");
Rtx.Invoke((System.Windows.Forms.MethodInvoker) delegate
{
Rtx.Text = (Rtx.Text.Replace(oldValue: Regex.Match(input: Rtx.Text,
pattern: @".*Packetstorm Components.*").Value, newValue: $"[{DateTime.Now.ToString("hh:mm:ss tt")}]\t[+] Packetstorm Components: {PacketstormList.Count}"));
});
}
}
}
catch (Exception ex)
{
if (ErrorCode == 0x3)
{
UpdateRtx(txt: $"[-] Error code: {ErrorCode}");
UpdateRtx(txt: $"[-] Error description: Problem in gathering components.{ex.Message}");
}
else
{
UpdateRtx(txt: $"[-] Error code: Unkown");
UpdateRtx(txt: $"[-] Error description: {ex.Message}");
}
};
}
lock (sleepLock)
{
sleepCounter++;
}
if (sleepCounter > MaxSleep)
{
Thread.Sleep(Timeout);
lock (sleepLock)
{
sleepCounter = 0;
}
}
});
UpdateRtx("[+] Database updated from packetstormsecurity.com.");
Packetstorm = false;
CheckPriority();
}
catch (AggregateException ex)
{
UpdateRtx(txt: $"[-] Error code: {ErrorCode}");
UpdateRtx(txt: $"[-] Error description: Probably you're using wrong dll version.{ex.Message}");
}
catch (Exception ex)
{
UpdateRtx(txt: $"[-] Error code: Unkown");
UpdateRtx(txt: $"[-] Error description: {ex.Message}");
}
}))
{
IsBackground = true
}.Start();
}
catch (Exception ex)
{
if (ErrorCode == 0x1)
{
UpdateRtx(txt: $"[-] Error code: {ErrorCode}");
UpdateRtx(txt: $"[-] Error description: Problem in connecting to packetstormsecurity.com.{ex.Message}");
}
else
{
UpdateRtx(txt: $"[-] Error code: Unkown");
UpdateRtx(txt: $"[-] Error description: {ex.Message}");
}
}
}
private List <EPGuigeReturnObject> parse(string fileLocation)
{
List <EPGuigeReturnObject> AllEpisodes = new List <EPGuigeReturnObject>();
string ShowName = "";
string htmlCode = File.ReadAllText(fileLocation);
StringReader sr = new StringReader(htmlCode);
string line;
while ((line = sr.ReadLine()) != null)
{
// title
if (line.Contains("<title>"))
{
int start = line.IndexOf("<title>") + 7;
int end = line.IndexOf("(a Titles &");
ShowName = line.Substring(start, end - start);
}
// show and episode content
if (line.Contains("<pre>"))
{
line = sr.ReadLine();
// skip empty lines
while (line != null && line.Length == 0)
{
line = sr.ReadLine();
}
// skip episode table header
line = sr.ReadLine();
line = sr.ReadLine();
line = sr.ReadLine();
// skip empty lines
while (line != null && line.Length == 0)
{
line = sr.ReadLine();
}
// cycle through all season and episode lines
while (line != null && !line.Contains("</pre>"))
{
// season name
if (line.StartsWith("• ")) // sometimes season names start with •
{
line = line.Substring(7).Trim();
}
line = sr.ReadLine();
// skip empty lines
while (line != null && line.Length == 0)
{
line = sr.ReadLine();
}
// episodes
while (line != null && !(line.Length == 0))
{
DateTime airDate = DateTime.MinValue;
string episodeNumber = "";
string episodeNumber2 = "";
string episode = "";
string episodeTitle = "";
Regex r = new Regex(@"\s{2,}");
string[] parts = r.Split(line);
foreach (string ppp in parts)
{
// get episode number
if (line.IndexOf(ppp) == 0)
{
episodeNumber = ppp;
}
// get 2nd episode number
if (line.IndexOf(ppp) > 0 && line.IndexOf(ppp) < 10)
{
episodeNumber2 = ppp;
}
// get airdate
if (airDate == DateTime.MinValue)
{
if (!DateTime.TryParseExact(ppp, "dd/MMM/yy", CultureInfo.CreateSpecificCulture("en"), System.Globalization.DateTimeStyles.None, out airDate))
{
DateTime.TryParseExact(ppp, "dd MMM yy", CultureInfo.CreateSpecificCulture("en"), System.Globalization.DateTimeStyles.None, out airDate);
}
}
// get episode info
if (line.IndexOf(ppp) > 35 && (!ppp.Contains("#trailer") && !ppp.Contains("recap")))
{
episode = ppp;
break;
}
}
// get episode title
List <LinkItem> list = LinkFinder.Find(episode);
foreach (LinkItem item in list)
{
if ((!item.Href.Contains("#trailer")) && (!item.Href.Contains("recap")))
{
episodeTitle = item.Text;
}
}
AllEpisodes.Add(new EPGuigeReturnObject(ShowName, episodeNumber, episodeNumber2, airDate.ToShortDateString(), episodeTitle));
line = sr.ReadLine();
}
// skip empty lines until next season
while (line != null && line.Length == 0)
{
line = sr.ReadLine();
}
}
break; // got all episodes - we are done
}
}
return(AllEpisodes);
}
private static void Main()
{
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
using (Process process = Process.GetCurrentProcess())
{
process.PriorityClass = ((ProcessPriorityClass)ProcessPriorityClass.High);
}
string str3 = Directory.GetParent(Assembly.GetExecutingAssembly().Location).FullName + Path.DirectorySeparatorChar.ToString() + "data";
try
{
new Thread(() => {
string conncode1 = string.Format(@"URI = file:{0}\\media-movies-links.db", str3);
SQLiteConnection cn = new SQLiteConnection(conncode1);
cn.Open();
SQLiteCommand cd = new SQLiteCommand(cn);
cd.CommandText = "delete from movie_links";
try
{
cd.ExecuteNonQuery();
SQLiteCommand command = new SQLiteCommand(cn);
string address = "https://erchatt.webs.com/movies";
string file = new WebClient().DownloadString(address);
foreach (string str4 in LinkFinder.Find(file))
{
if (!str3.Contains("webs.com"))
{
try
{
command.CommandText = $"insert into movie_links values ('{str4}', 'no-genre','no-desc')";
command.ExecuteNonQuery();
}
catch (Exception exception)
{
MessageBox.Show(exception.Message);
}
}
}
}
catch (Exception exception2)
{
Console.WriteLine(exception2.Message);
}
cn.Close();
}).Start();
new Thread(() =>
{
if (!File.Exists("lstupdate"))
{
SQLiteConnection updateReaderCreate = new SQLiteConnection($"URI=file:{str3}\\lstpdate");
updateReaderCreate.Open();
SQLiteCommand sql = new SQLiteCommand(updateReaderCreate);
sql.CommandText = "create table updates(lstpdate text)";
sql.ExecuteNonQuery();
updateReaderCreate.Close();
}
else
{
}
string lstupdate1 = "";
string nextupdate = "Wednesday, March 25, 2020";
SQLiteConnection updateReader = new SQLiteConnection($"URI=file:{str3}\\lstpdate");
updateReader.Open();
SQLiteCommand cmd = new SQLiteCommand(updateReader);
cmd.CommandText = "select * from updates";
SQLiteDataReader dr = cmd.ExecuteReader();
if (dr.Read())
{
lstupdate1 = (string)dr["lstupdate"];
}
if ((DateTime.Parse(nextupdate).Date) <= DateTime.Now)
{
if (lstupdate1 == nextupdate)
{
Thread.CurrentThread.Priority = ThreadPriority.Highest;
string[] strArray = new string[] { "updater.exe" };
foreach (string str in strArray)
{
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.FileName = str;
startInfo.UseShellExecute = false;
startInfo.WindowStyle = ProcessWindowStyle.Hidden;
//Start the process
Process proc = Process.Start(startInfo);
}
}
else
{
}
}
else
{
}
}).Start();
}
catch (Exception exception)
{
MessageBox.Show(exception.ToString());
}
if (!File.Exists(@"data\cred98"))
{
Application.Run(new welcome("welcome new user", "Welcome !!"));
}
else
{
try
{
SQLiteConnection connection = new SQLiteConnection($"URI=file:{str3}\\cred98");
connection.Open();
SQLiteCommand command = new SQLiteCommand(connection)
{
CommandText = "select * from credentials"
};
command.Prepare();
SQLiteDataReader reader = command.ExecuteReader();
if (reader.Read())
{
string str5 = (string)reader["name"];
string str6 = (string)reader["package"];
string str7 = (string)reader["signature"];
string s = (string)reader["expiry"];
string str9 = (string)reader["pcuid"];
if ((str7 != Environment.UserDomainName) | (str9 != Environment.UserName))
{
connection.Close();
clearAccounts();
}
if (str6 == "expired")
{
MessageBox.Show("Please Subscribe to use our Service");
Application.Run(new createAccount());
connection.Close();
}
if (DateTime.Parse(s).Date <= DateTime.Now.Date)
{
new SQLiteCommand(connection)
{
CommandText = "update credentials set package = 'expired'"
}.ExecuteNonQuery();
MessageBox.Show("Expiry date reached\nPlease subscribe");
Application.Restart();
connection.Close();
}
else
{
Application.Run(new load_main());
connection.Close();
}
connection.Close();
}
connection.Close();
}
catch (Exception exception2)
{
MessageBox.Show(exception2.Message);
}
}
}
