private void jobToRun_GrabComplete(object sender, GrabCompleteEventArgs e) { BackgroundWorker saveWorker = new BackgroundWorker(); saveWorker.DoWork += (s, dwe) => { if (_config.LogToDatabase) { var db = new SqLiteDal(_config.DatabasePath); db.EnsureExists(); db.AddJobHistory(new[] { e.Job }); } if (_config.LogToFile) { string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault(); if (string.IsNullOrWhiteSpace(filePath)) filePath = Lime49.Utils.GetApplicationPath(); string fileName = Path.GetExtension(filePath); var logDirectory = string.IsNullOrWhiteSpace(fileName)? filePath: Path.GetDirectoryName(filePath); if (!Directory.Exists(logDirectory)) Directory.CreateDirectory(logDirectory); switch (e.Job.ScheduledGrab.GrabMode) { case Constants.GrabModes.Regex: string logFilePath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now)); CSVExporter exporter = new CSVExporter(logFilePath, _config.AppendLogFile) { IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace = _config.TrimExtraWhitespace }; exporter.Save(e.Job.Response); break; case Constants.GrabModes.Scrape: //todo: test-untested string dumpPath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now)); File.WriteAllBytes(dumpPath, e.Job.Response.BinaryResponse); break; } } }; saveWorker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { Logger.Log(string.Format("Failed saving output for job #{0} - {1}: {2}", e.Job.ScheduledGrab.GrabID, e.Job.ScheduledGrab.Name, rwe.Error.Message)); GrabFailed(sender, e); } _handles[e.Job.WaitIndex].Set(); if (e.Job.WaitIndex == _handles.Length - 1) { Dispatcher.Invoke(new Action(() => { IsBusy = false; })); } }; saveWorker.RunWorkerAsync(); if (e.Job.ScheduledGrab.GrabSchedule == Constants.GrabSchedules.OneTime) Jobs.Remove(e.Job); if (GrabComplete != null) GrabComplete(sender, e); }
/// <summary> /// Loads saved settings when the Window loads. /// </summary> /// <param name="sender">The source of the event.</param> /// <param name="e">The <see cref="System.Windows.RoutedEventArgs"/> instance containing the event data.</param> private void Window_Loaded(object sender, RoutedEventArgs e) { try { //string serializedSavedUrls = Settings.Default.UrlList; //IEnumerable<GrabbableUrl> savedUrls = CsvGrabber.Core.Utils.DeserializeUrls(serializedSavedUrls); //ScheduledGrabs = new ObservableCollection<ScheduledGrab>(savedUrls); WPFConfiguration.LoadConfiguration(); _config = WPFConfiguration.GetConfig(); var db = new SqLiteDal(); if (!db.EnsureExists()) { DialogBox.ShowAlert(this, string.Format("The SQLite Database {0} could not be created", db.FileName), "Error"); Close(); } RefreshScheduledGrabs(); } catch (Exception ex) { Console.WriteLine("Error loading saved URL list: " + ex.Message); } try { string serializedFilenames = Settings.Default.FilePath; IEnumerable<string> items = CsvGrabber.Core.Utils.DeserializeList(serializedFilenames); if (items.Any()) { foreach (string item in items) { cboFileName.Items.Add(item); } cboFileName.SelectedIndex = 0; } else { cboFileName.Text = System.IO.Path.Combine(Lime49.Utils.GetApplicationPath(), string.Format("grabbeddata{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now)); } } catch (Exception ex) { Console.WriteLine("Error loading saved expression list: " + ex.Message); Settings.Default.FilePath = CsvGrabber.Core.Utils.SerializeList(new string[0]); } cboDbPath.Text = Settings.Default.LogDbPath; /*string lastFilename = Settings.Default.FilePath; if (string.IsNullOrWhiteSpace(lastFilename)) { Settings.Default.FilePath = System.IO.Path.Combine(Lime49.Utils.GetApplicationPath(), string.Format("grabbeddata{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now)); } txtFilePath.Text = Settings.Default.FilePath;*/ chkLogDatabase.IsChecked = Settings.Default.LogToDatabase; chkLogFile.IsChecked = Settings.Default.LogToFile; chkTrimWhitespace.IsChecked = Settings.Default.TrimExtraWhitespace; Settings.Default.Save(); }
/// <summary> /// Starts threads for grabbing Urls using the specified arguments. /// </summary> public void Start() { _cancellationPending = false; IsBusy = true; BackgroundWorker worker = new BackgroundWorker(); worker.DoWork += (s, dwe) => { var db = new SqLiteDal(); var jobs = db.GetScheduledGrabs(true); dwe.Result = jobs; //ManualResetEvent[] waitEvents = new ManualResetEvent[ThreadCount]; //for (int i = 0; i < ThreadCount; i++) { // try { // ManualResetEvent doneEvent = new ManualResetEvent(false); // if (ThreadPool.QueueUserWorkItem(new WaitCallback(ParseUrl), new UrlGrabThreadInfo(doneEvent, i))) { // waitEvents[i] = doneEvent; // _logger.Log(string.Format("Started download thread #{0}", i)); // } else { // _logger.Log(string.Format("Error starting download thread #{0}", i), true); // } // } catch (Exception ex) { // _logger.Log(string.Format("Error starting download thread #{0}", i, true) + ": " + ex.Message, true); // } //} //WaitHandle.WaitAll(waitEvents); //dwe.Result = new GrabCompleteEventArgs(ParsedUrls); }; worker.RunWorkerCompleted += (s, rwe) => { if (rwe.Error != null) { Logger.Log("Error starting grabbing URLs" + ": " + rwe.Error.Message, true); if (GrabFailed != null) GrabFailed(this, EventArgs.Empty); IsBusy = false; } else { Jobs = new ObservableCollection<GrabJob>(); var jobsToRun = rwe.Result as IEnumerable<ScheduledGrab>; GrabUrls(jobsToRun); } }; worker.RunWorkerAsync(); }
/// <summary> /// Saves the results of a grab. /// </summary> /// <param name="sender">The source of the event.</param> /// <param name="e">The <see cref="CsvGrabber.Core.GrabCompleteEventArgs"/> instance containing the event data.</param> private void grabber_GrabComplete(object sender, GrabCompleteEventArgs e) { BackgroundWorker saveWorker = new BackgroundWorker(); saveWorker.DoWork += (s, dwe) => { if(_config.LogToDatabase) { var db = new SqLiteDal(); db.AddJobHistory(new[] {e.Job}); } if(_config.LogToFile) { string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault(); DirectoryInfo jobLogDir = new DirectoryInfo(System.IO.Path.Combine(filePath, Core.Utils.SanitizeFileName(e.Job.ScheduledGrab.Name))); if (!jobLogDir.Exists) Directory.CreateDirectory(jobLogDir.FullName); //jobLogDir.Create(); switch (e.Job.ScheduledGrab.GrabMode) { case Constants.GrabModes.Regex: string logPath = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now)); CSVExporter exporter = new CSVExporter(logPath, _config.AppendLogFile) {IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace=_config.TrimExtraWhitespace}; exporter.Save(e.Job.Response); break; case Constants.GrabModes.Scrape: int read = 0; byte[] buffer = new byte[1024]; string fileName = System.IO.Path.GetFileNameWithoutExtension(e.Job.ScheduledGrab.GrabParams.Url.Url); string extension = System.IO.Path.GetExtension(e.Job.ScheduledGrab.GrabParams.Url.Url); string outputFile = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0}-{1:yyyy-MM-ddTHHmm}{2}", fileName, DateTime.Now, extension)); using (MemoryStream ms = new MemoryStream(e.Job.Response.BinaryResponse)) using(FileStream fs = new FileStream (outputFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)){//= new BinaryWriter()) { while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, read); } } break; } } }; saveWorker.RunWorkerCompleted += (Fs, rwe) => { if(rwe.Error != null) { Dispatcher.BeginInvoke(new ThreadStart(() => { DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); })); } }; saveWorker.RunWorkerAsync(); CommandManager.InvalidateRequerySuggested(); }
private void RefreshScheduledGrabs() { BackgroundWorker worker = new BackgroundWorker(); ObservableCollection<ScheduledGrab> grabs = null; ObservableCollection<GrabTemplate> templates = null; worker.DoWork += (s, dwe) => { var db = new SqLiteDal(); grabs = new ObservableCollection<ScheduledGrab>(db.GetScheduledGrabs(null)); templates = new ObservableCollection<GrabTemplate>(db.GetTemplates()); }; worker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error"); } else { ScheduledGrabs = grabs; TemplatesProvider.ObjectInstance = templates; } }; worker.RunWorkerAsync(); WPFUtils.WaitForWorker(worker); CommandManager.InvalidateRequerySuggested(); }
/// <summary> /// Deletes a URL from the Queue. /// </summary> /// <param name="sender">The sender.</param> /// <param name="e">The <see cref="System.Windows.Input.ExecutedRoutedEventArgs"/> instance containing the event data.</param> private void DeleteUrl(object sender, ExecutedRoutedEventArgs e) { if(DialogBox.Show(this, "Really delete this URL", "Confirm Deletion", DialogBoxType.YesNo, DialogBoxIcon.Question, DialogBoxButton.No) != DialogBoxButton.Yes) return; ScheduledGrab url = lstUrls.SelectedValue as ScheduledGrab; BackgroundWorker worker = new BackgroundWorker(); worker.DoWork += (s, dwe) => { var db = new SqLiteDal(); db.DeleteScheduledGrab(url); }; worker.RunWorkerCompleted += (Fs, rwe) => { if(rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error"); } }; worker.RunWorkerAsync(); WPFUtils.WaitForWorker(worker); CommandManager.InvalidateRequerySuggested(); RefreshScheduledGrabs(); }
private void EditUrl(object sender, ExecutedRoutedEventArgs e) { ScheduledGrab url = lstUrls.SelectedValue as ScheduledGrab; WinAddJob dlg = new WinAddJob(url.Clone()) { Owner = this, }; if (dlg.ShowDialog() == true) { BackgroundWorker worker = new BackgroundWorker(); ScheduledGrab jobToGrab = dlg.CurrentGrab; worker.DoWork += (s, dwe) => { var db = new SqLiteDal(); db.EditScheduledGrab(jobToGrab); }; worker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error"); } }; worker.RunWorkerAsync(); WPFUtils.WaitForWorker(worker); CommandManager.InvalidateRequerySuggested(); RefreshScheduledGrabs(); } }
private void CopyForUrls(object sender, ExecutedRoutedEventArgs e) { ScheduledGrab url = e.Parameter as ScheduledGrab; if (url == null) return; var dlg = new WinUserInput() { Prompt = "Enter URLs", AcceptsReturn = true, Owner = this }; if(dlg.ShowDialog()==true) { var rawUrls = dlg.UserInput; if(string.IsNullOrWhiteSpace(rawUrls)) return; BackgroundWorker saveWorker = new BackgroundWorker(); saveWorker.DoWork += (s, dwe) => { var db = new SqLiteDal(); var urls = rawUrls.Split(new[] {"\r\n", "\n"}, StringSplitOptions.RemoveEmptyEntries); db.DuplicateGrabs(url, urls); }; saveWorker.RunWorkerCompleted += (Fs, rwe) => { if(rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); } NavigationCommands.Refresh.Execute(null, this); }; saveWorker.RunWorkerAsync(); } }
///// <summary> ///// Adds a new category when the return key is pressed. ///// </summary> ///// <param name="sender">The source of the event.</param> ///// <param name="e">The <see cref="System.Windows.Input.KeyEventArgs"/> instance containing the event data.</param> //private void txtUrl_KeyDown(object sender, System.Windows.Input.KeyEventArgs e) //{ // if (e.Key == Key.Enter || e.Key == Key.Return || e.Key == Key.OemComma) { // ApplicationCommands.New.Execute(this, null); // e.Handled = true; // } //} /// <summary> /// Adds a URL to the queue. /// </summary> /// <param name="sender">The sender.</param> /// <param name="e">The <see cref="System.Windows.Input.ExecutedRoutedEventArgs"/> instance containing the event data.</param> private void AddUrl(object sender, ExecutedRoutedEventArgs e) { WinAddJob dlg = new WinAddJob() { Owner = this, }; if(e.Parameter is GrabTemplate) { WinAddFromTemplate dlgTemplate = new WinAddFromTemplate(){ Owner=this}; dlgTemplate.GrabTemplate = e.Parameter as GrabTemplate; if (dlg.ShowDialog() == true) { dlg.CurrentGrab = dlgTemplate.ScheduledGrab; } else { return; } } if (dlg.ShowDialog() == true) { BackgroundWorker worker = new BackgroundWorker(); ScheduledGrab jobToGrab = dlg.CurrentGrab; worker.DoWork += (s, dwe) => { var db = new SqLiteDal(); db.AddScheduledGrabs(new[]{jobToGrab}); }; worker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error"); } }; worker.RunWorkerAsync(); WPFUtils.WaitForWorker(worker); CommandManager.InvalidateRequerySuggested(); RefreshScheduledGrabs(); } //string url = txtUrl.Text; //if (UrlsToGrab.Any(u=>u.Url.ToLowerInvariant() == url.ToLowerInvariant())) { // DialogBox.ShowAlert(this, "URL already in queue", "Error"); //} else if (!string.IsNullOrWhiteSpace(url)) { // UrlsToGrab.Add(new GrabbableUrl(url)); // txtUrl.Text = string.Empty; //} }
private void DeleteTemplate(object sender, ExecutedRoutedEventArgs e) { if (DialogBox.Show(this, "Really delete this template", "Confirm Deletion", DialogBoxType.YesNo, DialogBoxIcon.Question, DialogBoxButton.No) != DialogBoxButton.Yes) return; var template = CurrentTemplate; BackgroundWorker worker = new BackgroundWorker(); worker.DoWork += (s, dwe) => { var db = new SqLiteDal(); db.DeleteGrabTemplate(template); }; worker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error"); } }; worker.RunWorkerAsync(); WPFUtils.WaitForWorker(worker); CommandManager.InvalidateRequerySuggested(); NavigationCommands.Refresh.Execute(null,this); }
private void SaveTemplate(object sender, ExecutedRoutedEventArgs e) { if (!Validate()) return; //Settings.Default.UrlList = CsvGrabber.Core.Utils.SerializeUrls(UrlsToGrab); List<string> expressions = new List<string>(); foreach (var exp in cboExpressions.Items) { string expression = Convert.ToString(exp); if (!string.IsNullOrWhiteSpace(expression) && !expressions.Contains(expression)) { expressions.Insert(0, expression); } } if (!expressions.Contains(cboExpressions.Text) && !string.IsNullOrWhiteSpace(cboExpressions.Text)) { expressions.Add(cboExpressions.Text); cboExpressions.Items.Add(cboExpressions.Text); } Settings.Default.GrabExpressions = CsvGrabber.Core.Utils.SerializeList(expressions); Settings.Default.Save(); GrabEventArgs args = new GrabEventArgs(new GrabbableUrl(txtUrlFormatString.Text)); RegexOptions options = 0; if (chkCaseInsensitive.IsChecked == true) { options |= RegexOptions.IgnoreCase; } if (chkDotNewLine.IsChecked == true) { options |= RegexOptions.Singleline; } if (chkFreeSpacing.IsChecked == true) { options |= RegexOptions.IgnorePatternWhitespace; } args.GrabExpression = new Regex(cboExpressions.Text, options); CurrentTemplate.GrabParameters = args; var templateToSave = CurrentTemplate; BackgroundWorker saveWorker = new BackgroundWorker(); saveWorker.DoWork += (s, dwe) => { var db = new SqLiteDal(); if (string.IsNullOrWhiteSpace(templateToSave.TemplateID)) { templateToSave.TemplateID = Lime49.Utils.GenID(true); db.AddGrabTemplates(new[] { templateToSave }); } else { db.EditGrabTemplate(templateToSave); } }; saveWorker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); } else { CurrentTemplate = templateToSave; } }; saveWorker.RunWorkerAsync(); }
private void RefreshTemplates(object sender, RoutedEventArgs e) { List<GrabTemplate> templates = new List<GrabTemplate>(); BackgroundWorker saveWorker = new BackgroundWorker(); saveWorker.DoWork += (s, dwe) => { var db = new SqLiteDal(); templates = db.GetTemplates(); }; saveWorker.RunWorkerCompleted += (Fs, rwe) => { if (rwe.Error != null) { DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); } else { this.Templates = new ObservableCollection<GrabTemplate>(templates); } }; saveWorker.RunWorkerAsync(); }