private void jobToRun_GrabComplete(object sender, GrabCompleteEventArgs e)
{
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) =>
{
if (_config.LogToDatabase) {
var db = new SqLiteDal(_config.DatabasePath);
db.EnsureExists();
db.AddJobHistory(new[] { e.Job });
}
if (_config.LogToFile) {
string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault();
if (string.IsNullOrWhiteSpace(filePath))
filePath = Lime49.Utils.GetApplicationPath();
string fileName = Path.GetExtension(filePath);
var logDirectory = string.IsNullOrWhiteSpace(fileName)? filePath: Path.GetDirectoryName(filePath);
if (!Directory.Exists(logDirectory))
Directory.CreateDirectory(logDirectory);
switch (e.Job.ScheduledGrab.GrabMode) {
case Constants.GrabModes.Regex:
string logFilePath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now));
CSVExporter exporter = new CSVExporter(logFilePath, _config.AppendLogFile) { IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace = _config.TrimExtraWhitespace };
exporter.Save(e.Job.Response);
break;
case Constants.GrabModes.Scrape:
//todo: test-untested
string dumpPath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now));
File.WriteAllBytes(dumpPath, e.Job.Response.BinaryResponse);
break;
}
}
};
saveWorker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null) {
Logger.Log(string.Format("Failed saving output for job #{0} - {1}: {2}", e.Job.ScheduledGrab.GrabID, e.Job.ScheduledGrab.Name, rwe.Error.Message));
GrabFailed(sender, e);
}
_handles[e.Job.WaitIndex].Set();
if (e.Job.WaitIndex == _handles.Length - 1)
{
Dispatcher.Invoke(new Action(() =>
{
IsBusy = false;
}));
}
};
saveWorker.RunWorkerAsync();
if (e.Job.ScheduledGrab.GrabSchedule == Constants.GrabSchedules.OneTime)
Jobs.Remove(e.Job);
if (GrabComplete != null)
GrabComplete(sender, e);
}
/// <summary>
/// Saves the results of a grab.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="CsvGrabber.Core.GrabCompleteEventArgs"/> instance containing the event data.</param>
private void grabber_GrabComplete(object sender, GrabCompleteEventArgs e)
{
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) => {
if(_config.LogToDatabase) {
var db = new SqLiteDal();
db.AddJobHistory(new[] {e.Job});
}
if(_config.LogToFile) {
string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault();
DirectoryInfo jobLogDir = new DirectoryInfo(System.IO.Path.Combine(filePath, Core.Utils.SanitizeFileName(e.Job.ScheduledGrab.Name)));
if (!jobLogDir.Exists)
Directory.CreateDirectory(jobLogDir.FullName); //jobLogDir.Create();
switch (e.Job.ScheduledGrab.GrabMode) {
case Constants.GrabModes.Regex:
string logPath = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now));
CSVExporter exporter = new CSVExporter(logPath, _config.AppendLogFile) {IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace=_config.TrimExtraWhitespace};
exporter.Save(e.Job.Response);
break;
case Constants.GrabModes.Scrape:
int read = 0;
byte[] buffer = new byte[1024];
string fileName = System.IO.Path.GetFileNameWithoutExtension(e.Job.ScheduledGrab.GrabParams.Url.Url);
string extension = System.IO.Path.GetExtension(e.Job.ScheduledGrab.GrabParams.Url.Url);
string outputFile = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0}-{1:yyyy-MM-ddTHHmm}{2}", fileName, DateTime.Now, extension));
using (MemoryStream ms = new MemoryStream(e.Job.Response.BinaryResponse))
using(FileStream fs = new FileStream (outputFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)){//= new BinaryWriter()) {
while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) {
ms.Write(buffer, 0, read);
}
}
break;
}
}
};
saveWorker.RunWorkerCompleted += (Fs, rwe) => {
if(rwe.Error != null) {
Dispatcher.BeginInvoke(new ThreadStart(() =>
{ DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); }));
}
};
saveWorker.RunWorkerAsync();
CommandManager.InvalidateRequerySuggested();
}
