/// <summary>
/// Load the jobs from the file jobs.csv
///
/// </summary>
/// <returns></returns>
private WebsiteData GetData()
{
string filename = "data/jobs.csv";
WebsiteData websiteData = new WebsiteData();
using (TextFieldParser csvParser = new TextFieldParser(filename))
{
csvParser.SetDelimiters(new string[] { "," });
csvParser.HasFieldsEnclosedInQuotes = true;
// Skip the row with the column names
csvParser.ReadLine();
int id = 0;
while (!csvParser.EndOfData)
{
// Read current line fields, pointer moves to the next line.
string[] fields = csvParser.ReadFields();
Job job = new Job(id, fields);
id++;
websiteData.Jobs.Add(job);
}
}
User u = new User();
u.Name = "Admin";
u.Password = "******";
websiteData.Users.Add(u);
return(websiteData);
}
public async Task <List <WebsiteData> > getWebsiteData()
{
List <WebsiteData> FoxNewsData = new List <WebsiteData>();
driver = new ChromeDriver();
var urls = await getUrls();
foreach (var item in urls)
{
driver.Navigate().GoToUrl(item.url);
var headlineElement = driver.FindElement(By.ClassName("headline"));
foreach (var term in _searchTerms)
{
if (headlineElement.Text.ToLower().Contains(term))
{
WebsiteData websiteData = new WebsiteData();
websiteData.outlet = "Fox News";
websiteData.url = item.url;
websiteData.headline = headlineElement.Text;
websiteData.date = item.date;
websiteData.body = driver.FindElement(By.ClassName("article-body")).Text;
FoxNewsData.Add(websiteData);
}
}
}
return(FoxNewsData);
}
public async Task <List <WebsiteData> > getWebsiteData()
{
List <WebsiteData> cbcData = new List <WebsiteData>();
driver = new ChromeDriver();
List <String> urls = await getUrls();
foreach (var item in urls)
{
driver.Navigate().GoToUrl(item);
try
{
var headlineElement = driver.FindElement(By.ClassName("detailHeadline"));
foreach (var term in _searchTerms)
{
if (headlineElement.Text.ToLower().Contains(term))
{
WebsiteData websiteData = new WebsiteData();
websiteData.outlet = "CBC News";
websiteData.url = item;
websiteData.headline = headlineElement.Text;
try
{
websiteData.date = driver.FindElement(By.ClassName("timeStamp")).Text.Substring(8, 12);
}
catch
{
Console.WriteLine("No date found");
websiteData.date = "";
}
try
{
websiteData.body = driver.FindElement(By.ClassName("story")).Text;
}
catch
{
Console.WriteLine("No article body found");
websiteData.body = "";
}
cbcData.Add(websiteData);
}
}
}
catch
{
Console.WriteLine("No headline found.");
}
}
return(cbcData);
}
private void Update()
{
if (!PluginConfiguration.AutoUpdate)
{
return;
}
if (string.IsNullOrEmpty(PluginClass.BuildURL))
{
return;
}
PointBlankLogging.LogImportant("Downloading " + Name + "...");
WebsiteData.DownloadFile(PluginClass.BuildURL, Location);
PointBlankLogging.LogImportant(Name + " updated successfully! Please restart the server to finalize the update!");
}
private bool CheckUpdates()
{
if (!PluginConfiguration.NotifyUpdates && !PluginConfiguration.AutoUpdate)
{
return(false);
}
if (string.IsNullOrEmpty(PluginClass.VersionURL))
{
return(false);
}
string bVersion = "";
WebsiteData.GetData(PluginClass.VersionURL, out bVersion);
return(bVersion != Version);
}
/// <summary>
/// Getst the View for the display the image carousel & word count for the website
/// </summary>
/// <param name="websiteData"></param>
/// <returns></returns>
public ActionResult WebsiteContentCountResult(WebsiteData websiteData)
{
Uri uriResult;
bool validUrl = ValidateURL(websiteData.WebsiteURL, out uriResult);
if (ModelState.IsValid && validUrl && checkWebsiteExists(uriResult.AbsoluteUri))
{
//Returns action to be executed on successfull completion of call
return(View("~/Views/WebsiteContentReader/WebsiteContentCountResult.cshtml",
this.WebsiteContentReaderRepository.GetWebsiteContent(uriResult.AbsoluteUri)));
}
//Returns the validation message
ModelState.AddModelError("WebsiteURL", "please enter website url");
return(View("~/Views/WebsiteContentReader/WebsiteData.cshtml", websiteData));
}
private void CheckUpdates()
{
while (Running)
{
Thread.Sleep(300000); // Check every 5 minutes
if (WebsiteData.GetData(URL, out string data))
{
JObject info = JObject.Parse(data);
if ((string)info["PointBlank_Version"] == "0") // Ignore if no version
{
continue;
}
if ((string)info["PointBlank_Version"] != PointBlankInfo.Version)
{
PointBlankLogging.LogImportant("A new update is available for PointBlank!");
}
}
}
}
public static bool VerifyLicense(string file, string website, EHashType hash = EHashType.NONE)
{
if (!File.Exists(file))
{
return(false);
}
string response = "false";
switch (hash)
{
case EHashType.MD5:
WebsiteData.PostData(website, new NameValueCollection()
{
{ "License", Hashing.CalculateMD5String(File.ReadAllBytes(file)) }
}, out response);
break;
case EHashType.SHA1:
WebsiteData.PostData(website, new NameValueCollection()
{
{ "License", Hashing.CalculateSHA1String(File.ReadAllBytes(file)) }
}, out response);
break;
case EHashType.SHA256:
WebsiteData.PostData(website, new NameValueCollection()
{
{ "License", Hashing.CalculateSHA256String(File.ReadAllBytes(file)) }
}, out response);
break;
case EHashType.SHA512:
WebsiteData.PostData(website, new NameValueCollection()
{
{ "License", Hashing.CalculateSHA512String(File.ReadAllBytes(file)) }
}, out response);
break;
default:
if (WebsiteData.UploadFile(website, file, out byte[] data))
public List <WebsiteData> getWebsiteData()
{
List <WebsiteData> cnnData = new List <WebsiteData>();
driver = new ChromeDriver();
for (int pageNum = 1; pageNum < 100; pageNum++)
{
driver.Navigate().GoToUrl($"https://www.cnn.com/search?size=100&q=coronavirus&from={(pageNum - 1) * 100}&page={pageNum}");
var searchList = _websiteScraperShared.FindElements(driver, By.ClassName("cnn-search__result"));
foreach (var item in searchList)
{
var headlineElement = item.FindElement(By.ClassName("cnn-search__result-headline"));
var dateElement = item.FindElement(By.ClassName("cnn-search__result-publish-date"));
var bodyElement = item.FindElement(By.ClassName("cnn-search__result-body"));
foreach (var term in _searchTerms)
{
if (headlineElement.Text.ToLower().Contains(term))
{
WebsiteData websiteData = new WebsiteData();
websiteData.outlet = "CNN";
websiteData.url = headlineElement.FindElement(By.TagName("a")).GetAttribute("href");
websiteData.headline = headlineElement.Text;
websiteData.date = dateElement.Text;
websiteData.body = bodyElement.Text;
cnnData.Add(websiteData);
break;
}
}
}
}
driver.Close();
return(cnnData);
}
public async Task <List <WebsiteData> > getUrls()
{
List <WebsiteData> urlAndDate = new List <WebsiteData>();
HttpClient client = new HttpClient();
for (int i = 1; i < 100; i++)
{
var date = DateTime.Today.AddDays(1 - i).ToString("yyyyMMdd");
var dateFormatted = DateTime.Today.AddDays(1 - i).ToString("MMM dd, yyyy");
HttpResponseMessage response = await client.GetAsync($"https://api.foxnews.com/search/web?q=coronavirus+more:pagemap:metatags-pagetype:article+more:pagemap:metatags-dc.type:Text.Article&siteSearch=foxnews.com&siteSearchFilter=i&sort=date:r:{date}:{date}");
var content = response.Content.ReadAsStringAsync().Result;
var apiObjects = JsonConvert.DeserializeObject <FoxNewsAPI>(content);
foreach (var item in apiObjects.items)
{
var foxNewsObject = new WebsiteData();
foxNewsObject.url = item.link;
foxNewsObject.date = dateFormatted;
urlAndDate.Add(foxNewsObject);
}
}
return(urlAndDate);
}
public void InitializeData()
{
this.site = new WebsiteData();
}
public async Task InsertWebsiteData(WebsiteData websiteData)
{
string sql = @"insert into websitedata (url, outlet, headline, date, body)
values (@url, @outlet, @headline, @date, @body)";
await _sqlAccess.SaveData(sql, websiteData);
}
