private void jobToRun_GrabComplete(object sender, GrabCompleteEventArgs e)
{
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) =>
{
if (_config.LogToDatabase) {
var db = new SqLiteDal(_config.DatabasePath);
db.EnsureExists();
db.AddJobHistory(new[] { e.Job });
}
if (_config.LogToFile) {
string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault();
if (string.IsNullOrWhiteSpace(filePath))
filePath = Lime49.Utils.GetApplicationPath();
string fileName = Path.GetExtension(filePath);
var logDirectory = string.IsNullOrWhiteSpace(fileName)? filePath: Path.GetDirectoryName(filePath);
if (!Directory.Exists(logDirectory))
Directory.CreateDirectory(logDirectory);
switch (e.Job.ScheduledGrab.GrabMode) {
case Constants.GrabModes.Regex:
string logFilePath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now));
CSVExporter exporter = new CSVExporter(logFilePath, _config.AppendLogFile) { IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace = _config.TrimExtraWhitespace };
exporter.Save(e.Job.Response);
break;
case Constants.GrabModes.Scrape:
//todo: test-untested
string dumpPath = System.IO.Path.Combine(logDirectory, string.Format("{0}-{1:yyyy-MM-ddTHHmm}.csv", Utils.SanitizeFileName(e.Job.ScheduledGrab.Name), DateTime.Now));
File.WriteAllBytes(dumpPath, e.Job.Response.BinaryResponse);
break;
}
}
};
saveWorker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null) {
Logger.Log(string.Format("Failed saving output for job #{0} - {1}: {2}", e.Job.ScheduledGrab.GrabID, e.Job.ScheduledGrab.Name, rwe.Error.Message));
GrabFailed(sender, e);
}
_handles[e.Job.WaitIndex].Set();
if (e.Job.WaitIndex == _handles.Length - 1)
{
Dispatcher.Invoke(new Action(() =>
{
IsBusy = false;
}));
}
};
saveWorker.RunWorkerAsync();
if (e.Job.ScheduledGrab.GrabSchedule == Constants.GrabSchedules.OneTime)
Jobs.Remove(e.Job);
if (GrabComplete != null)
GrabComplete(sender, e);
}
/// <summary>
/// Loads saved settings when the Window loads.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="System.Windows.RoutedEventArgs"/> instance containing the event data.</param>
private void Window_Loaded(object sender, RoutedEventArgs e)
{
try {
//string serializedSavedUrls = Settings.Default.UrlList;
//IEnumerable<GrabbableUrl> savedUrls = CsvGrabber.Core.Utils.DeserializeUrls(serializedSavedUrls);
//ScheduledGrabs = new ObservableCollection<ScheduledGrab>(savedUrls);
WPFConfiguration.LoadConfiguration();
_config = WPFConfiguration.GetConfig();
var db = new SqLiteDal();
if (!db.EnsureExists()) {
DialogBox.ShowAlert(this, string.Format("The SQLite Database {0} could not be created", db.FileName), "Error");
Close();
}
RefreshScheduledGrabs();
} catch (Exception ex) {
Console.WriteLine("Error loading saved URL list: " + ex.Message);
}
try {
string serializedFilenames = Settings.Default.FilePath;
IEnumerable<string> items = CsvGrabber.Core.Utils.DeserializeList(serializedFilenames);
if (items.Any()) {
foreach (string item in items) {
cboFileName.Items.Add(item);
}
cboFileName.SelectedIndex = 0;
} else {
cboFileName.Text = System.IO.Path.Combine(Lime49.Utils.GetApplicationPath(), string.Format("grabbeddata{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now));
}
} catch (Exception ex) {
Console.WriteLine("Error loading saved expression list: " + ex.Message);
Settings.Default.FilePath = CsvGrabber.Core.Utils.SerializeList(new string[0]);
}
cboDbPath.Text = Settings.Default.LogDbPath;
/*string lastFilename = Settings.Default.FilePath;
if (string.IsNullOrWhiteSpace(lastFilename)) {
Settings.Default.FilePath = System.IO.Path.Combine(Lime49.Utils.GetApplicationPath(), string.Format("grabbeddata{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now));
}
txtFilePath.Text = Settings.Default.FilePath;*/
chkLogDatabase.IsChecked = Settings.Default.LogToDatabase;
chkLogFile.IsChecked = Settings.Default.LogToFile;
chkTrimWhitespace.IsChecked = Settings.Default.TrimExtraWhitespace;
Settings.Default.Save();
}
/// <summary>
/// Starts threads for grabbing Urls using the specified arguments.
/// </summary>
public void Start()
{
_cancellationPending = false;
IsBusy = true;
BackgroundWorker worker = new BackgroundWorker();
worker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
var jobs = db.GetScheduledGrabs(true);
dwe.Result = jobs;
//ManualResetEvent[] waitEvents = new ManualResetEvent[ThreadCount];
//for (int i = 0; i < ThreadCount; i++) {
// try {
// ManualResetEvent doneEvent = new ManualResetEvent(false);
// if (ThreadPool.QueueUserWorkItem(new WaitCallback(ParseUrl), new UrlGrabThreadInfo(doneEvent, i))) {
// waitEvents[i] = doneEvent;
// _logger.Log(string.Format("Started download thread #{0}", i));
// } else {
// _logger.Log(string.Format("Error starting download thread #{0}", i), true);
// }
// } catch (Exception ex) {
// _logger.Log(string.Format("Error starting download thread #{0}", i, true) + ": " + ex.Message, true);
// }
//}
//WaitHandle.WaitAll(waitEvents);
//dwe.Result = new GrabCompleteEventArgs(ParsedUrls);
};
worker.RunWorkerCompleted += (s, rwe) =>
{
if (rwe.Error != null) {
Logger.Log("Error starting grabbing URLs" + ": " + rwe.Error.Message, true);
if (GrabFailed != null)
GrabFailed(this, EventArgs.Empty);
IsBusy = false;
} else {
Jobs = new ObservableCollection<GrabJob>();
var jobsToRun = rwe.Result as IEnumerable<ScheduledGrab>;
GrabUrls(jobsToRun);
}
};
worker.RunWorkerAsync();
}
/// <summary>
/// Saves the results of a grab.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="CsvGrabber.Core.GrabCompleteEventArgs"/> instance containing the event data.</param>
private void grabber_GrabComplete(object sender, GrabCompleteEventArgs e)
{
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) => {
if(_config.LogToDatabase) {
var db = new SqLiteDal();
db.AddJobHistory(new[] {e.Job});
}
if(_config.LogToFile) {
string filePath= CsvGrabber.Core.Utils.DeserializeList(_config.FilePath).FirstOrDefault();
DirectoryInfo jobLogDir = new DirectoryInfo(System.IO.Path.Combine(filePath, Core.Utils.SanitizeFileName(e.Job.ScheduledGrab.Name)));
if (!jobLogDir.Exists)
Directory.CreateDirectory(jobLogDir.FullName); //jobLogDir.Create();
switch (e.Job.ScheduledGrab.GrabMode) {
case Constants.GrabModes.Regex:
string logPath = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0:yyyy-MM-ddTHHmm}.csv", DateTime.Now));
CSVExporter exporter = new CSVExporter(logPath, _config.AppendLogFile) {IncludeRawResponse = _config.LogRawResponse, TrimExtraWhitespace=_config.TrimExtraWhitespace};
exporter.Save(e.Job.Response);
break;
case Constants.GrabModes.Scrape:
int read = 0;
byte[] buffer = new byte[1024];
string fileName = System.IO.Path.GetFileNameWithoutExtension(e.Job.ScheduledGrab.GrabParams.Url.Url);
string extension = System.IO.Path.GetExtension(e.Job.ScheduledGrab.GrabParams.Url.Url);
string outputFile = System.IO.Path.Combine(jobLogDir.FullName, string.Format("{0}-{1:yyyy-MM-ddTHHmm}{2}", fileName, DateTime.Now, extension));
using (MemoryStream ms = new MemoryStream(e.Job.Response.BinaryResponse))
using(FileStream fs = new FileStream (outputFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)){//= new BinaryWriter()) {
while ((read = ms.Read(buffer, 0, buffer.Length)) > 0) {
ms.Write(buffer, 0, read);
}
}
break;
}
}
};
saveWorker.RunWorkerCompleted += (Fs, rwe) => {
if(rwe.Error != null) {
Dispatcher.BeginInvoke(new ThreadStart(() =>
{ DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results"); }));
}
};
saveWorker.RunWorkerAsync();
CommandManager.InvalidateRequerySuggested();
}
private void RefreshScheduledGrabs()
{
BackgroundWorker worker = new BackgroundWorker();
ObservableCollection<ScheduledGrab> grabs = null;
ObservableCollection<GrabTemplate> templates = null;
worker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
grabs = new ObservableCollection<ScheduledGrab>(db.GetScheduledGrabs(null));
templates = new ObservableCollection<GrabTemplate>(db.GetTemplates());
};
worker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null) {
DialogBox.ShowAlert(this, rwe.Error.Message, "Error");
} else {
ScheduledGrabs = grabs;
TemplatesProvider.ObjectInstance = templates;
}
};
worker.RunWorkerAsync();
WPFUtils.WaitForWorker(worker);
CommandManager.InvalidateRequerySuggested();
}
/// <summary>
/// Deletes a URL from the Queue.
/// </summary>
/// <param name="sender">The sender.</param>
/// <param name="e">The <see cref="System.Windows.Input.ExecutedRoutedEventArgs"/> instance containing the event data.</param>
private void DeleteUrl(object sender, ExecutedRoutedEventArgs e)
{
if(DialogBox.Show(this, "Really delete this URL", "Confirm Deletion", DialogBoxType.YesNo, DialogBoxIcon.Question, DialogBoxButton.No) != DialogBoxButton.Yes)
return;
ScheduledGrab url = lstUrls.SelectedValue as ScheduledGrab;
BackgroundWorker worker = new BackgroundWorker();
worker.DoWork += (s, dwe) => {
var db = new SqLiteDal();
db.DeleteScheduledGrab(url);
};
worker.RunWorkerCompleted += (Fs, rwe) => {
if(rwe.Error != null) {
DialogBox.ShowAlert(this, rwe.Error.Message, "Error");
}
};
worker.RunWorkerAsync();
WPFUtils.WaitForWorker(worker);
CommandManager.InvalidateRequerySuggested();
RefreshScheduledGrabs();
}
private void EditUrl(object sender, ExecutedRoutedEventArgs e)
{
ScheduledGrab url = lstUrls.SelectedValue as ScheduledGrab;
WinAddJob dlg = new WinAddJob(url.Clone())
{
Owner = this,
};
if (dlg.ShowDialog() == true) {
BackgroundWorker worker = new BackgroundWorker();
ScheduledGrab jobToGrab = dlg.CurrentGrab;
worker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
db.EditScheduledGrab(jobToGrab);
};
worker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null) {
DialogBox.ShowAlert(this, rwe.Error.Message, "Error");
}
};
worker.RunWorkerAsync();
WPFUtils.WaitForWorker(worker);
CommandManager.InvalidateRequerySuggested();
RefreshScheduledGrabs();
}
}
private void CopyForUrls(object sender, ExecutedRoutedEventArgs e)
{
ScheduledGrab url = e.Parameter as ScheduledGrab;
if (url == null)
return;
var dlg = new WinUserInput()
{
Prompt = "Enter URLs",
AcceptsReturn = true,
Owner = this
};
if(dlg.ShowDialog()==true)
{
var rawUrls = dlg.UserInput;
if(string.IsNullOrWhiteSpace(rawUrls))
return;
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) => {
var db = new SqLiteDal();
var urls = rawUrls.Split(new[] {"\r\n", "\n"}, StringSplitOptions.RemoveEmptyEntries);
db.DuplicateGrabs(url, urls);
};
saveWorker.RunWorkerCompleted += (Fs, rwe) => {
if(rwe.Error != null) {
DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results");
}
NavigationCommands.Refresh.Execute(null, this);
};
saveWorker.RunWorkerAsync();
}
}
///// <summary>
///// Adds a new category when the return key is pressed.
///// </summary>
///// <param name="sender">The source of the event.</param>
///// <param name="e">The <see cref="System.Windows.Input.KeyEventArgs"/> instance containing the event data.</param>
//private void txtUrl_KeyDown(object sender, System.Windows.Input.KeyEventArgs e)
//{
// if (e.Key == Key.Enter || e.Key == Key.Return || e.Key == Key.OemComma) {
// ApplicationCommands.New.Execute(this, null);
// e.Handled = true;
// }
//}
/// <summary>
/// Adds a URL to the queue.
/// </summary>
/// <param name="sender">The sender.</param>
/// <param name="e">The <see cref="System.Windows.Input.ExecutedRoutedEventArgs"/> instance containing the event data.</param>
private void AddUrl(object sender, ExecutedRoutedEventArgs e)
{
WinAddJob dlg = new WinAddJob()
{
Owner = this,
};
if(e.Parameter is GrabTemplate)
{
WinAddFromTemplate dlgTemplate = new WinAddFromTemplate(){ Owner=this};
dlgTemplate.GrabTemplate = e.Parameter as GrabTemplate;
if (dlg.ShowDialog() == true) {
dlg.CurrentGrab = dlgTemplate.ScheduledGrab;
} else
{
return;
}
}
if (dlg.ShowDialog() == true) {
BackgroundWorker worker = new BackgroundWorker();
ScheduledGrab jobToGrab = dlg.CurrentGrab;
worker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
db.AddScheduledGrabs(new[]{jobToGrab});
};
worker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null) {
DialogBox.ShowAlert(this, rwe.Error.Message, "Error");
}
};
worker.RunWorkerAsync();
WPFUtils.WaitForWorker(worker);
CommandManager.InvalidateRequerySuggested();
RefreshScheduledGrabs();
}
//string url = txtUrl.Text;
//if (UrlsToGrab.Any(u=>u.Url.ToLowerInvariant() == url.ToLowerInvariant())) {
// DialogBox.ShowAlert(this, "URL already in queue", "Error");
//} else if (!string.IsNullOrWhiteSpace(url)) {
// UrlsToGrab.Add(new GrabbableUrl(url));
// txtUrl.Text = string.Empty;
//}
}
private void DeleteTemplate(object sender, ExecutedRoutedEventArgs e)
{
if (DialogBox.Show(this, "Really delete this template", "Confirm Deletion", DialogBoxType.YesNo, DialogBoxIcon.Question, DialogBoxButton.No) != DialogBoxButton.Yes)
return;
var template = CurrentTemplate;
BackgroundWorker worker = new BackgroundWorker();
worker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
db.DeleteGrabTemplate(template);
};
worker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null)
{
DialogBox.ShowAlert(this, rwe.Error.Message, "Error");
}
};
worker.RunWorkerAsync();
WPFUtils.WaitForWorker(worker);
CommandManager.InvalidateRequerySuggested();
NavigationCommands.Refresh.Execute(null,this);
}
private void SaveTemplate(object sender, ExecutedRoutedEventArgs e)
{
if (!Validate())
return;
//Settings.Default.UrlList = CsvGrabber.Core.Utils.SerializeUrls(UrlsToGrab);
List<string> expressions = new List<string>();
foreach (var exp in cboExpressions.Items) {
string expression = Convert.ToString(exp);
if (!string.IsNullOrWhiteSpace(expression) && !expressions.Contains(expression)) {
expressions.Insert(0, expression);
}
}
if (!expressions.Contains(cboExpressions.Text) && !string.IsNullOrWhiteSpace(cboExpressions.Text)) {
expressions.Add(cboExpressions.Text);
cboExpressions.Items.Add(cboExpressions.Text);
}
Settings.Default.GrabExpressions = CsvGrabber.Core.Utils.SerializeList(expressions);
Settings.Default.Save();
GrabEventArgs args = new GrabEventArgs(new GrabbableUrl(txtUrlFormatString.Text));
RegexOptions options = 0;
if (chkCaseInsensitive.IsChecked == true) {
options |= RegexOptions.IgnoreCase;
}
if (chkDotNewLine.IsChecked == true) {
options |= RegexOptions.Singleline;
}
if (chkFreeSpacing.IsChecked == true) {
options |= RegexOptions.IgnorePatternWhitespace;
}
args.GrabExpression = new Regex(cboExpressions.Text, options);
CurrentTemplate.GrabParameters = args;
var templateToSave = CurrentTemplate;
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
if (string.IsNullOrWhiteSpace(templateToSave.TemplateID)) {
templateToSave.TemplateID = Lime49.Utils.GenID(true);
db.AddGrabTemplates(new[] { templateToSave });
} else
{
db.EditGrabTemplate(templateToSave);
}
};
saveWorker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null) {
DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results");
} else
{
CurrentTemplate = templateToSave;
}
};
saveWorker.RunWorkerAsync();
}
private void RefreshTemplates(object sender, RoutedEventArgs e)
{
List<GrabTemplate> templates = new List<GrabTemplate>();
BackgroundWorker saveWorker = new BackgroundWorker();
saveWorker.DoWork += (s, dwe) =>
{
var db = new SqLiteDal();
templates = db.GetTemplates();
};
saveWorker.RunWorkerCompleted += (Fs, rwe) =>
{
if (rwe.Error != null)
{
DialogBox.ShowAlert(this, rwe.Error.Message, "Error Saving Job Results");
} else
{
this.Templates = new ObservableCollection<GrabTemplate>(templates);
}
};
saveWorker.RunWorkerAsync();
}
