/// <summary>
/// Compare new extracted data with exisiting data in database
/// </summary>
/// <param name="url"></param>
/// <param name="content"></param>
/// <param name="date"></param>
public void Compare(string url, string content, string date)
{
con.Open();
cmd.Reset();
cmd.Parameters.AddWithValue("@urlname", url);
cmd.Parameters.AddWithValue("@urlcontent", content);
cmd.Parameters.AddWithValue("@contentdate", date);
cmd.CommandText = "SELECT COUNT(*) FROM WebsiteData WHERE URL=@urlname AND DATE=@contentdate AND MESSAGE=@urlcontent;";
if (Convert.ToInt32(cmd.ExecuteScalar()) == 0)
{
MyNotification.Push(url, content, date);
Add(url, content, date);
}
con.Close();
}
/// <summary>
/// Will execute after a loop of urls are visited
/// Check all the urls in database and remove the redundant rows
/// </summary>
public void Update()
{
List <string> to_remove = new List <string>();
// Check for updates
con.Open();
cmd.Reset();
cmd.CommandText = "SELECT DISTINCT URL FROM WebsiteData;";
SQLiteDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
string distinct_url = reader.GetString(0);
if (!URLS.Contains(distinct_url))
{
to_remove.Add(distinct_url);
}
}
reader.Close();
// Updates
foreach (string url in to_remove)
{
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("DatabaseChange"), InfoFetchConsole.Program.localeM.GetString("DatabaseOldDelete") + "\nURL: " + url);
cmd.Reset();
cmd.Parameters.Add("@urlname", DbType.String).Value = url;
cmd.CommandText = "DELETE FROM WebsiteData WHERE URL=@urlname;";
cmd.ExecuteNonQuery();
}
// Remove redundant content of urls (if > 50 rows)
cmd.Reset();
cmd.CommandText = "SELECT URL, COUNT(*) FROM WebsiteData GROUP BY URL HAVING COUNT(URL) > 50;";
reader = cmd.ExecuteReader();
while (reader.Read())
{
string url = reader.GetString(0);
int occurences = reader.GetInt32(1);
int head = occurences - 30; // Delete to 30 rows
cmd.Reset();
cmd.Parameters.AddWithValue("@urlname", url);
cmd.Parameters.AddWithValue("@limitnum", head);
cmd.CommandText = "DELETE FROM WebsiteData WHERE rowid IN (SELECT rowid FROM WebsiteData WHERE URL=@urlname LIMIT @limitnum);";
cmd.ExecuteNonQuery();
}
reader.Close();
con.Close();
}
/// <summary>
/// check local chromedriver version
/// </summary>
/// <returns>
/// true if match with chrome version
/// false if not match
/// </returns>
private static bool CheckChromeDriverVersion()
{
Process p = new Process();
p.StartInfo.UseShellExecute = false;
p.StartInfo.RedirectStandardOutput = true;
p.StartInfo.FileName = "chromedriver.exe";
p.StartInfo.Arguments = "-v";
p.Start();
string driveroutput = p.StandardOutput.ReadToEnd();
p.WaitForExit();
p.Dispose();
Regex versionReg = new Regex(@"\d+.\d+.\d+.\d+");
Match versionMatch = versionReg.Match(driveroutput);
string driverVersion;
if (versionMatch.Success)
{
driverVersion = versionMatch.Value;
}
else
{
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("UnknownChromedriverVersion"));
return(false);
}
if (string.IsNullOrEmpty(localChromeVersion))
{
return(false);
}
string[] chromeV = localChromeVersion.Split('.');
string[] driverV = driverVersion.Split('.');
if (chromeV[0] == driverV[0] && chromeV[1] == driverV[1] && chromeV[2] == driverV[2])
{
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// check local chromedriver
/// </summary>
/// <returns>
/// true if chromedriver is found and has correct version
/// false if not found
/// </returns>
public static bool CheckChromeDriver()
{
bool driverOK = true;
if (!File.Exists("chromedriver.exe"))
{
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("ChromedriverNotFound"));
driverOK = false;
}
else if (!CheckChromeDriverVersion())
{
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("ChromedriverWrongVersion"));
driverOK = false;
}
if (!driverOK)
{
return(DownloadChromeDriver());
}
return(driverOK);
}
/// <summary>
/// Send html to HtmlAgilityPack
/// Then request the selected sections
/// At last let database update content
/// </summary>
/// <param name="web"></param>
/// <param name="direction"></param>
public void Read(Website web, string direction, Database database)
{
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(web.Content);
HtmlNodeCollection htmlNodes = htmlDoc.DocumentNode.SelectNodes(direction);
if (htmlNodes == null)
{
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("ParseFail"),
string.Format(InfoFetchConsole.Program.localeM.GetString("URLXpath") + "\n" +
InfoFetchConsole.Program.localeM.GetString("XPath"), web.Url, direction));
return;
}
// reverse the order, so that the latest date appears at the end of the database
for (int i = htmlNodes.Count - 1; i >= 0; i--)
{
ParseContent(htmlNodes[i].InnerText, out string newContent, out string newDate);
database.Compare(web.Url, newContent, newDate);
}
}
/// <summary>
/// Show an input box to change interval
/// Idea comes from https://www.csharp-examples.net/inputbox/
/// </summary>
/// <returns></returns>
public static DialogResult IntervalInput(long currentInterval, out long newInterval)
{
Form form = new Form();
form.Text = InfoFetchConsole.Program.localeM.GetString("ChangeInterval");
Button buttonOK = new Button();
Button buttonCancel = new Button();
buttonOK.Text = InfoFetchConsole.Program.localeM.GetString("OK");
buttonCancel.Text = InfoFetchConsole.Program.localeM.GetString("Cancel");
buttonOK.DialogResult = DialogResult.OK;
buttonCancel.DialogResult = DialogResult.Cancel;
buttonOK.SetBounds(20, 70, 70, 20);
buttonCancel.SetBounds(110, 70, 70, 20);
IntervalFormat format = ConvertTime(currentInterval, out int txtBoxValue);
TextBox txtBox = new TextBox();
txtBox.Text = txtBoxValue.ToString();
txtBox.Focus();
txtBox.AcceptsReturn = false;
txtBox.AcceptsTab = false;
txtBox.SetBounds(10, 10, 80, 50);
txtBox.Font = new Font(FontFamily.GenericSansSerif, 20);
txtBox.KeyPress += Txtbox_KeyPress;
txtBox.MaxLength = 4;
ComboBox comboBox = new ComboBox();
string[] formats = new string[] {
InfoFetchConsole.Program.localeM.GetString("Day"),
InfoFetchConsole.Program.localeM.GetString("Hour"),
InfoFetchConsole.Program.localeM.GetString("Minute"),
InfoFetchConsole.Program.localeM.GetString("Second")
};
comboBox.Items.AddRange(formats);
comboBox.SetBounds(100, 10, 90, 50);
switch (format)
{
case IntervalFormat.IntervalDay:
comboBox.SelectedIndex = 0;
break;
case IntervalFormat.IntervalHour:
comboBox.SelectedIndex = 1;
break;
case IntervalFormat.IntervalMinute:
comboBox.SelectedIndex = 2;
break;
case IntervalFormat.IntervalSecond:
comboBox.SelectedIndex = 3;
break;
}
comboBox.Font = new Font(FontFamily.GenericSansSerif, 18);
comboBox.DropDownStyle = ComboBoxStyle.DropDownList;
form.ClientSize = new Size(200, 100);
form.Controls.AddRange(new Control[] { buttonOK, buttonCancel, txtBox, comboBox });
form.FormBorderStyle = FormBorderStyle.FixedSingle;
form.StartPosition = FormStartPosition.CenterScreen;
form.ShowInTaskbar = true;
form.MinimizeBox = false;
form.MaximizeBox = false;
form.AcceptButton = buttonOK;
form.CancelButton = buttonCancel;
DialogResult result = form.ShowDialog();
if (result == DialogResult.OK)
{
string notificationFormat = "";
switch (comboBox.SelectedIndex)
{
case 0:
format = IntervalFormat.IntervalDay;
notificationFormat = InfoFetchConsole.Program.localeM.GetString("Day");
break;
case 1:
format = IntervalFormat.IntervalHour;
notificationFormat = InfoFetchConsole.Program.localeM.GetString("Hour");
break;
case 2:
format = IntervalFormat.IntervalMinute;
notificationFormat = InfoFetchConsole.Program.localeM.GetString("Minute");
break;
case 3:
format = IntervalFormat.IntervalSecond;
notificationFormat = InfoFetchConsole.Program.localeM.GetString("Second");
break;
default:
format = IntervalFormat.IntervalSecond;
break;
}
int num = System.Convert.ToInt32(txtBox.Text);
newInterval = ConvertBackTime(num, format);
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("IntervalUpdate"),
InfoFetchConsole.Program.localeM.GetString("IntervalNew") + " " + num.ToString() + " " + notificationFormat);
}
else
{
newInterval = 0;
}
return(result);
}
/// <summary>
/// download the correct chromedriver and remove the one with wrong version
/// </summary>
/// <returns>
/// true if download succeeded
/// false if download failed
/// </returns>
private static bool DownloadChromeDriver()
{
MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("ChromedriverDownload"));
string webContent;
if (File.Exists(localBackupFile))
{
StreamReader sr = new StreamReader(localBackupFile);
webContent = sr.ReadToEnd();
sr.Close();
}
else
{
return(false);
}
if (string.IsNullOrEmpty(webContent))
{
return(false);
}
Regex versionReg = new Regex(@"\d+.\d+.\d+.\d+");
Match versionMatch = versionReg.Match(webContent);
string targetVersion = string.Empty;
string[] chromeV = localChromeVersion.Split('.');
while (versionMatch.Success)
{
string version = versionMatch.Value;
string[] driverV = version.Split('.');
if (chromeV[0] == driverV[0] && chromeV[1] == driverV[1] && chromeV[2] == driverV[2])
{
targetVersion = version;
break;
}
versionMatch = versionMatch.NextMatch();
}
if (string.IsNullOrEmpty(targetVersion))
{
return(false);
}
string targetURL = downloadURL + targetVersion + "/chromedriver_win32.zip";
if (File.Exists("chromedriver_win32.zip"))
{
File.Delete("chromedriver_win32.zip");
}
using (var downloader = new WebClient())
{
downloader.DownloadFile(targetURL, "chromedriver_win32.zip");
}
if (!File.Exists("chromedriver_win32.zip"))
{
return(false);
}
if (File.Exists("chromedriver.exe"))
{
File.Delete("chromedriver.exe");
}
ZipFile.ExtractToDirectory("chromedriver_win32.zip", ".");
if (File.Exists("chromedriver.exe"))
{
File.Delete("chromedriver_win32.zip");
}
else
{
return(false);
}
return(true);
}
