Beispiel #1
0
 /// <summary>
 /// Compare new extracted data with exisiting data in database
 /// </summary>
 /// <param name="url"></param>
 /// <param name="content"></param>
 /// <param name="date"></param>
 public void Compare(string url, string content, string date)
 {
     con.Open();
     cmd.Reset();
     cmd.Parameters.AddWithValue("@urlname", url);
     cmd.Parameters.AddWithValue("@urlcontent", content);
     cmd.Parameters.AddWithValue("@contentdate", date);
     cmd.CommandText = "SELECT COUNT(*) FROM WebsiteData WHERE URL=@urlname AND DATE=@contentdate AND MESSAGE=@urlcontent;";
     if (Convert.ToInt32(cmd.ExecuteScalar()) == 0)
     {
         MyNotification.Push(url, content, date);
         Add(url, content, date);
     }
     con.Close();
 }
Beispiel #2
0
        /// <summary>
        /// Will execute after a loop of urls are visited
        /// Check all the urls in database and remove the redundant rows
        /// </summary>
        public void Update()
        {
            List <string> to_remove = new List <string>();

            // Check for updates
            con.Open();
            cmd.Reset();
            cmd.CommandText = "SELECT DISTINCT URL FROM WebsiteData;";
            SQLiteDataReader reader = cmd.ExecuteReader();

            while (reader.Read())
            {
                string distinct_url = reader.GetString(0);
                if (!URLS.Contains(distinct_url))
                {
                    to_remove.Add(distinct_url);
                }
            }
            reader.Close();
            // Updates
            foreach (string url in to_remove)
            {
                MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("DatabaseChange"), InfoFetchConsole.Program.localeM.GetString("DatabaseOldDelete") + "\nURL: " + url);
                cmd.Reset();
                cmd.Parameters.Add("@urlname", DbType.String).Value = url;
                cmd.CommandText = "DELETE FROM WebsiteData WHERE URL=@urlname;";
                cmd.ExecuteNonQuery();
            }

            // Remove redundant content of urls (if > 50 rows)
            cmd.Reset();
            cmd.CommandText = "SELECT URL, COUNT(*) FROM WebsiteData GROUP BY URL HAVING COUNT(URL) > 50;";
            reader          = cmd.ExecuteReader();
            while (reader.Read())
            {
                string url        = reader.GetString(0);
                int    occurences = reader.GetInt32(1);
                int    head       = occurences - 30; // Delete to 30 rows
                cmd.Reset();
                cmd.Parameters.AddWithValue("@urlname", url);
                cmd.Parameters.AddWithValue("@limitnum", head);
                cmd.CommandText = "DELETE FROM WebsiteData WHERE rowid IN (SELECT rowid FROM WebsiteData WHERE URL=@urlname LIMIT @limitnum);";
                cmd.ExecuteNonQuery();
            }
            reader.Close();

            con.Close();
        }
Beispiel #3
0
        /// <summary>
        /// check local chromedriver version
        /// </summary>
        /// <returns>
        /// true if match with chrome version
        /// false if not match
        /// </returns>
        private static bool CheckChromeDriverVersion()
        {
            Process p = new Process();

            p.StartInfo.UseShellExecute        = false;
            p.StartInfo.RedirectStandardOutput = true;
            p.StartInfo.FileName  = "chromedriver.exe";
            p.StartInfo.Arguments = "-v";
            p.Start();
            string driveroutput = p.StandardOutput.ReadToEnd();

            p.WaitForExit();
            p.Dispose();

            Regex  versionReg   = new Regex(@"\d+.\d+.\d+.\d+");
            Match  versionMatch = versionReg.Match(driveroutput);
            string driverVersion;

            if (versionMatch.Success)
            {
                driverVersion = versionMatch.Value;
            }
            else
            {
                MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("UnknownChromedriverVersion"));
                return(false);
            }

            if (string.IsNullOrEmpty(localChromeVersion))
            {
                return(false);
            }

            string[] chromeV = localChromeVersion.Split('.');
            string[] driverV = driverVersion.Split('.');

            if (chromeV[0] == driverV[0] && chromeV[1] == driverV[1] && chromeV[2] == driverV[2])
            {
                return(true);
            }
            else
            {
                return(false);
            }
        }
Beispiel #4
0
        /// <summary>
        /// check local chromedriver
        /// </summary>
        /// <returns>
        /// true if chromedriver is found and has correct version
        /// false if not found
        /// </returns>
        public static bool CheckChromeDriver()
        {
            bool driverOK = true;

            if (!File.Exists("chromedriver.exe"))
            {
                MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("ChromedriverNotFound"));
                driverOK = false;
            }
            else if (!CheckChromeDriverVersion())
            {
                MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("ChromedriverWrongVersion"));
                driverOK = false;
            }
            if (!driverOK)
            {
                return(DownloadChromeDriver());
            }

            return(driverOK);
        }
Beispiel #5
0
        /// <summary>
        /// Send html to HtmlAgilityPack
        /// Then request the selected sections
        /// At last let database update content
        /// </summary>
        /// <param name="web"></param>
        /// <param name="direction"></param>
        public void Read(Website web, string direction, Database database)
        {
            HtmlDocument htmlDoc = new HtmlDocument();

            htmlDoc.LoadHtml(web.Content);
            HtmlNodeCollection htmlNodes = htmlDoc.DocumentNode.SelectNodes(direction);

            if (htmlNodes == null)
            {
                MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("ParseFail"),
                                    string.Format(InfoFetchConsole.Program.localeM.GetString("URLXpath") + "\n" +
                                                  InfoFetchConsole.Program.localeM.GetString("XPath"), web.Url, direction));
                return;
            }
            // reverse the order, so that the latest date appears at the end of the database
            for (int i = htmlNodes.Count - 1; i >= 0; i--)
            {
                ParseContent(htmlNodes[i].InnerText, out string newContent, out string newDate);
                database.Compare(web.Url, newContent, newDate);
            }
        }
Beispiel #6
0
        /// <summary>
        /// Show an input box to change interval
        /// Idea comes from https://www.csharp-examples.net/inputbox/
        /// </summary>
        /// <returns></returns>
        public static DialogResult IntervalInput(long currentInterval, out long newInterval)
        {
            Form form = new Form();

            form.Text = InfoFetchConsole.Program.localeM.GetString("ChangeInterval");

            Button buttonOK     = new Button();
            Button buttonCancel = new Button();

            buttonOK.Text             = InfoFetchConsole.Program.localeM.GetString("OK");
            buttonCancel.Text         = InfoFetchConsole.Program.localeM.GetString("Cancel");
            buttonOK.DialogResult     = DialogResult.OK;
            buttonCancel.DialogResult = DialogResult.Cancel;

            buttonOK.SetBounds(20, 70, 70, 20);
            buttonCancel.SetBounds(110, 70, 70, 20);

            IntervalFormat format = ConvertTime(currentInterval, out int txtBoxValue);

            TextBox txtBox = new TextBox();

            txtBox.Text = txtBoxValue.ToString();
            txtBox.Focus();
            txtBox.AcceptsReturn = false;
            txtBox.AcceptsTab    = false;
            txtBox.SetBounds(10, 10, 80, 50);
            txtBox.Font      = new Font(FontFamily.GenericSansSerif, 20);
            txtBox.KeyPress += Txtbox_KeyPress;
            txtBox.MaxLength = 4;

            ComboBox comboBox = new ComboBox();

            string[] formats = new string[] {
                InfoFetchConsole.Program.localeM.GetString("Day"),
                InfoFetchConsole.Program.localeM.GetString("Hour"),
                InfoFetchConsole.Program.localeM.GetString("Minute"),
                InfoFetchConsole.Program.localeM.GetString("Second")
            };
            comboBox.Items.AddRange(formats);
            comboBox.SetBounds(100, 10, 90, 50);
            switch (format)
            {
            case IntervalFormat.IntervalDay:
                comboBox.SelectedIndex = 0;
                break;

            case IntervalFormat.IntervalHour:
                comboBox.SelectedIndex = 1;
                break;

            case IntervalFormat.IntervalMinute:
                comboBox.SelectedIndex = 2;
                break;

            case IntervalFormat.IntervalSecond:
                comboBox.SelectedIndex = 3;
                break;
            }
            comboBox.Font          = new Font(FontFamily.GenericSansSerif, 18);
            comboBox.DropDownStyle = ComboBoxStyle.DropDownList;

            form.ClientSize = new Size(200, 100);
            form.Controls.AddRange(new Control[] { buttonOK, buttonCancel, txtBox, comboBox });
            form.FormBorderStyle = FormBorderStyle.FixedSingle;
            form.StartPosition   = FormStartPosition.CenterScreen;
            form.ShowInTaskbar   = true;
            form.MinimizeBox     = false;
            form.MaximizeBox     = false;
            form.AcceptButton    = buttonOK;
            form.CancelButton    = buttonCancel;

            DialogResult result = form.ShowDialog();

            if (result == DialogResult.OK)
            {
                string notificationFormat = "";
                switch (comboBox.SelectedIndex)
                {
                case 0:
                    format             = IntervalFormat.IntervalDay;
                    notificationFormat = InfoFetchConsole.Program.localeM.GetString("Day");
                    break;

                case 1:
                    format             = IntervalFormat.IntervalHour;
                    notificationFormat = InfoFetchConsole.Program.localeM.GetString("Hour");
                    break;

                case 2:
                    format             = IntervalFormat.IntervalMinute;
                    notificationFormat = InfoFetchConsole.Program.localeM.GetString("Minute");
                    break;

                case 3:
                    format             = IntervalFormat.IntervalSecond;
                    notificationFormat = InfoFetchConsole.Program.localeM.GetString("Second");
                    break;

                default:
                    format = IntervalFormat.IntervalSecond;
                    break;
                }
                int num = System.Convert.ToInt32(txtBox.Text);
                newInterval = ConvertBackTime(num, format);

                MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("IntervalUpdate"),
                                    InfoFetchConsole.Program.localeM.GetString("IntervalNew") + " " + num.ToString() + " " + notificationFormat);
            }
            else
            {
                newInterval = 0;
            }
            return(result);
        }
Beispiel #7
0
        /// <summary>
        /// download the correct chromedriver and remove the one with wrong version
        /// </summary>
        /// <returns>
        /// true if download succeeded
        /// false if download failed
        /// </returns>
        private static bool DownloadChromeDriver()
        {
            MyNotification.Push(InfoFetchConsole.Program.localeM.GetString("Chromedriver"), InfoFetchConsole.Program.localeM.GetString("ChromedriverDownload"));

            string webContent;

            if (File.Exists(localBackupFile))
            {
                StreamReader sr = new StreamReader(localBackupFile);
                webContent = sr.ReadToEnd();
                sr.Close();
            }
            else
            {
                return(false);
            }
            if (string.IsNullOrEmpty(webContent))
            {
                return(false);
            }

            Regex  versionReg    = new Regex(@"\d+.\d+.\d+.\d+");
            Match  versionMatch  = versionReg.Match(webContent);
            string targetVersion = string.Empty;

            string[] chromeV = localChromeVersion.Split('.');
            while (versionMatch.Success)
            {
                string   version = versionMatch.Value;
                string[] driverV = version.Split('.');
                if (chromeV[0] == driverV[0] && chromeV[1] == driverV[1] && chromeV[2] == driverV[2])
                {
                    targetVersion = version;
                    break;
                }
                versionMatch = versionMatch.NextMatch();
            }
            if (string.IsNullOrEmpty(targetVersion))
            {
                return(false);
            }

            string targetURL = downloadURL + targetVersion + "/chromedriver_win32.zip";

            if (File.Exists("chromedriver_win32.zip"))
            {
                File.Delete("chromedriver_win32.zip");
            }

            using (var downloader = new WebClient())
            {
                downloader.DownloadFile(targetURL, "chromedriver_win32.zip");
            }

            if (!File.Exists("chromedriver_win32.zip"))
            {
                return(false);
            }

            if (File.Exists("chromedriver.exe"))
            {
                File.Delete("chromedriver.exe");
            }

            ZipFile.ExtractToDirectory("chromedriver_win32.zip", ".");

            if (File.Exists("chromedriver.exe"))
            {
                File.Delete("chromedriver_win32.zip");
            }
            else
            {
                return(false);
            }

            return(true);
        }