/// <summary> /// Load the jobs from the file jobs.csv /// /// </summary> /// <returns></returns> private WebsiteData GetData() { string filename = "data/jobs.csv"; WebsiteData websiteData = new WebsiteData(); using (TextFieldParser csvParser = new TextFieldParser(filename)) { csvParser.SetDelimiters(new string[] { "," }); csvParser.HasFieldsEnclosedInQuotes = true; // Skip the row with the column names csvParser.ReadLine(); int id = 0; while (!csvParser.EndOfData) { // Read current line fields, pointer moves to the next line. string[] fields = csvParser.ReadFields(); Job job = new Job(id, fields); id++; websiteData.Jobs.Add(job); } } User u = new User(); u.Name = "Admin"; u.Password = "******"; websiteData.Users.Add(u); return(websiteData); }
public async Task <List <WebsiteData> > getWebsiteData() { List <WebsiteData> FoxNewsData = new List <WebsiteData>(); driver = new ChromeDriver(); var urls = await getUrls(); foreach (var item in urls) { driver.Navigate().GoToUrl(item.url); var headlineElement = driver.FindElement(By.ClassName("headline")); foreach (var term in _searchTerms) { if (headlineElement.Text.ToLower().Contains(term)) { WebsiteData websiteData = new WebsiteData(); websiteData.outlet = "Fox News"; websiteData.url = item.url; websiteData.headline = headlineElement.Text; websiteData.date = item.date; websiteData.body = driver.FindElement(By.ClassName("article-body")).Text; FoxNewsData.Add(websiteData); } } } return(FoxNewsData); }
public async Task <List <WebsiteData> > getWebsiteData() { List <WebsiteData> cbcData = new List <WebsiteData>(); driver = new ChromeDriver(); List <String> urls = await getUrls(); foreach (var item in urls) { driver.Navigate().GoToUrl(item); try { var headlineElement = driver.FindElement(By.ClassName("detailHeadline")); foreach (var term in _searchTerms) { if (headlineElement.Text.ToLower().Contains(term)) { WebsiteData websiteData = new WebsiteData(); websiteData.outlet = "CBC News"; websiteData.url = item; websiteData.headline = headlineElement.Text; try { websiteData.date = driver.FindElement(By.ClassName("timeStamp")).Text.Substring(8, 12); } catch { Console.WriteLine("No date found"); websiteData.date = ""; } try { websiteData.body = driver.FindElement(By.ClassName("story")).Text; } catch { Console.WriteLine("No article body found"); websiteData.body = ""; } cbcData.Add(websiteData); } } } catch { Console.WriteLine("No headline found."); } } return(cbcData); }
private void Update() { if (!PluginConfiguration.AutoUpdate) { return; } if (string.IsNullOrEmpty(PluginClass.BuildURL)) { return; } PointBlankLogging.LogImportant("Downloading " + Name + "..."); WebsiteData.DownloadFile(PluginClass.BuildURL, Location); PointBlankLogging.LogImportant(Name + " updated successfully! Please restart the server to finalize the update!"); }
private bool CheckUpdates() { if (!PluginConfiguration.NotifyUpdates && !PluginConfiguration.AutoUpdate) { return(false); } if (string.IsNullOrEmpty(PluginClass.VersionURL)) { return(false); } string bVersion = ""; WebsiteData.GetData(PluginClass.VersionURL, out bVersion); return(bVersion != Version); }
/// <summary> /// Getst the View for the display the image carousel & word count for the website /// </summary> /// <param name="websiteData"></param> /// <returns></returns> public ActionResult WebsiteContentCountResult(WebsiteData websiteData) { Uri uriResult; bool validUrl = ValidateURL(websiteData.WebsiteURL, out uriResult); if (ModelState.IsValid && validUrl && checkWebsiteExists(uriResult.AbsoluteUri)) { //Returns action to be executed on successfull completion of call return(View("~/Views/WebsiteContentReader/WebsiteContentCountResult.cshtml", this.WebsiteContentReaderRepository.GetWebsiteContent(uriResult.AbsoluteUri))); } //Returns the validation message ModelState.AddModelError("WebsiteURL", "please enter website url"); return(View("~/Views/WebsiteContentReader/WebsiteData.cshtml", websiteData)); }
private void CheckUpdates() { while (Running) { Thread.Sleep(300000); // Check every 5 minutes if (WebsiteData.GetData(URL, out string data)) { JObject info = JObject.Parse(data); if ((string)info["PointBlank_Version"] == "0") // Ignore if no version { continue; } if ((string)info["PointBlank_Version"] != PointBlankInfo.Version) { PointBlankLogging.LogImportant("A new update is available for PointBlank!"); } } } }
public static bool VerifyLicense(string file, string website, EHashType hash = EHashType.NONE) { if (!File.Exists(file)) { return(false); } string response = "false"; switch (hash) { case EHashType.MD5: WebsiteData.PostData(website, new NameValueCollection() { { "License", Hashing.CalculateMD5String(File.ReadAllBytes(file)) } }, out response); break; case EHashType.SHA1: WebsiteData.PostData(website, new NameValueCollection() { { "License", Hashing.CalculateSHA1String(File.ReadAllBytes(file)) } }, out response); break; case EHashType.SHA256: WebsiteData.PostData(website, new NameValueCollection() { { "License", Hashing.CalculateSHA256String(File.ReadAllBytes(file)) } }, out response); break; case EHashType.SHA512: WebsiteData.PostData(website, new NameValueCollection() { { "License", Hashing.CalculateSHA512String(File.ReadAllBytes(file)) } }, out response); break; default: if (WebsiteData.UploadFile(website, file, out byte[] data))
public List <WebsiteData> getWebsiteData() { List <WebsiteData> cnnData = new List <WebsiteData>(); driver = new ChromeDriver(); for (int pageNum = 1; pageNum < 100; pageNum++) { driver.Navigate().GoToUrl($"https://www.cnn.com/search?size=100&q=coronavirus&from={(pageNum - 1) * 100}&page={pageNum}"); var searchList = _websiteScraperShared.FindElements(driver, By.ClassName("cnn-search__result")); foreach (var item in searchList) { var headlineElement = item.FindElement(By.ClassName("cnn-search__result-headline")); var dateElement = item.FindElement(By.ClassName("cnn-search__result-publish-date")); var bodyElement = item.FindElement(By.ClassName("cnn-search__result-body")); foreach (var term in _searchTerms) { if (headlineElement.Text.ToLower().Contains(term)) { WebsiteData websiteData = new WebsiteData(); websiteData.outlet = "CNN"; websiteData.url = headlineElement.FindElement(By.TagName("a")).GetAttribute("href"); websiteData.headline = headlineElement.Text; websiteData.date = dateElement.Text; websiteData.body = bodyElement.Text; cnnData.Add(websiteData); break; } } } } driver.Close(); return(cnnData); }
public async Task <List <WebsiteData> > getUrls() { List <WebsiteData> urlAndDate = new List <WebsiteData>(); HttpClient client = new HttpClient(); for (int i = 1; i < 100; i++) { var date = DateTime.Today.AddDays(1 - i).ToString("yyyyMMdd"); var dateFormatted = DateTime.Today.AddDays(1 - i).ToString("MMM dd, yyyy"); HttpResponseMessage response = await client.GetAsync($"https://api.foxnews.com/search/web?q=coronavirus+more:pagemap:metatags-pagetype:article+more:pagemap:metatags-dc.type:Text.Article&siteSearch=foxnews.com&siteSearchFilter=i&sort=date:r:{date}:{date}"); var content = response.Content.ReadAsStringAsync().Result; var apiObjects = JsonConvert.DeserializeObject <FoxNewsAPI>(content); foreach (var item in apiObjects.items) { var foxNewsObject = new WebsiteData(); foxNewsObject.url = item.link; foxNewsObject.date = dateFormatted; urlAndDate.Add(foxNewsObject); } } return(urlAndDate); }
public void InitializeData() { this.site = new WebsiteData(); }
public async Task InsertWebsiteData(WebsiteData websiteData) { string sql = @"insert into websitedata (url, outlet, headline, date, body) values (@url, @outlet, @headline, @date, @body)"; await _sqlAccess.SaveData(sql, websiteData); }