Example #1
0
 private void jobToRun_GrabComplete(object sender, GrabCompleteEventArgs e)
 {
     BackgroundWorker saveWorker = new BackgroundWorker();
     saveWorker.DoWork += (s, dwe) =>
     {
         if (_config.LogToDatabase) {
             var db = new SqLiteDal(_config.DatabasePath);
             db.EnsureExists();
                 db.AddJobHistory(new[] { e.Job });
         }
         if (_config.LogToFile) {
             string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault();
             if (string.IsNullOrWhiteSpace(filePath))
                 filePath = Lime49.Utils.GetApplicationPath();
             string fileName = Path.GetExtension(filePath);
             var logDirectory = string.IsNullOrWhiteSpace(fileName)? filePath: Path.GetDirectoryName(filePath);
             if (!Directory.Exists(logDirectory))
                 Directory.CreateDirectory(logDirectory);
             switch (e.Job.ScheduledGrab.GrabMode) {
                 case Constants.GrabModes.Regex:
                     string logFilePath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now));
                     CSVExporter exporter = new CSVExporter(logFilePath, _config.AppendLogFile) { IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace = _config.TrimExtraWhitespace };
                     exporter.Save(e.Job.Response);
                     break;
                 case Constants.GrabModes.Scrape:
                     //todo: test-untested
                     string dumpPath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now));
                     File.WriteAllBytes(dumpPath, e.Job.Response.BinaryResponse);
                     break;
             }
         }
     };
     saveWorker.RunWorkerCompleted += (Fs, rwe) =>
     {
         if (rwe.Error != null) {
             Logger.Log(string.Format("Failed saving output for job #{0} - {1}: {2}", e.Job.ScheduledGrab.GrabID, e.Job.ScheduledGrab.Name, rwe.Error.Message));
             GrabFailed(sender, e);
         }
         _handles[e.Job.WaitIndex].Set();
         if (e.Job.WaitIndex == _handles.Length - 1)
         {
             Dispatcher.Invoke(new Action(() =>
                                              {
                                                  IsBusy = false;
                                              }));
         }
     };
     saveWorker.RunWorkerAsync();
     if (e.Job.ScheduledGrab.GrabSchedule == Constants.GrabSchedules.OneTime)
         Jobs.Remove(e.Job);
     if (GrabComplete != null)
         GrabComplete(sender, e);
 }
Example #2
0
 /// <summary>
 /// Saves the results of a grab.
 /// </summary>
 /// <param name="sender">The source of the event.</param>
 /// <param name="e">The <see cref="CsvGrabber.Core.GrabCompleteEventArgs"/> instance containing the event data.</param>
 private void grabber_GrabComplete(object sender, GrabCompleteEventArgs e)
 {
     BackgroundWorker saveWorker = new BackgroundWorker();
     saveWorker.DoWork += (s, dwe) => {
                              if(_config.LogToDatabase) {
                                  var db = new SqLiteDal();
                                  db.AddJobHistory(new[] {e.Job});
                              }
                              if(_config.LogToFile) {
                                  string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault();
                                  DirectoryInfo jobLogDir = new DirectoryInfo(System.IO.Path.Combine(filePath, Core.Utils.SanitizeFileName(e.Job.ScheduledGrab.Name)));
                                  if (!jobLogDir.Exists)
                                      Directory.CreateDirectory(jobLogDir.FullName); //jobLogDir.Create();
                                  switch (e.Job.ScheduledGrab.GrabMode) {
                                      case Constants.GrabModes.Regex:
                                          string logPath = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now));
                                          CSVExporter exporter = new CSVExporter(logPath, _config.AppendLogFile) {IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace=_config.TrimExtraWhitespace};
                                          exporter.Save(e.Job.Response);
                                          break;
                                          case Constants.GrabModes.Scrape:
                                          int read = 0;
                                          byte[] buffer = new byte[1024];
                                          string fileName = System.IO.Path.GetFileNameWithoutExtension(e.Job.ScheduledGrab.GrabParams.Url.Url);
                                          string extension = System.IO.Path.GetExtension(e.Job.ScheduledGrab.GrabParams.Url.Url);
                                          string outputFile = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0}-{1:yyyy-MM-ddTHHmm}{2}", fileName, DateTime.Now, extension));
                                          using (MemoryStream ms = new MemoryStream(e.Job.Response.BinaryResponse))
                                          using(FileStream fs = new FileStream (outputFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)){//= new BinaryWriter()) {
                                              while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) {
                                                  ms.Write(buffer, 0, read);
                                              }
                                          }
                                          break;
                                  }
                              }
                          };
     saveWorker.RunWorkerCompleted += (Fs, rwe) => {
                                          if(rwe.Error != null) {
                                              Dispatcher.BeginInvoke(new ThreadStart(() =>
                                                  { DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); }));
                                          }
                                      };
     saveWorker.RunWorkerAsync();
     CommandManager.InvalidateRequerySuggested();
 }