void btnImport_Click(object sender, RoutedEventArgs e)
{
FeatureTrackingManager.Instance.UseFeature(Features.Library_ImportFromThirdParty);
IEnumerable <AugmentedBindable <BibTeXEntry> > allEntries = GetEntries().Where(x => x.Underlying.Selected);
if (allEntries.Count() == 0)
{
MessageBoxes.Error("Please select at least one entry to import, by checking the checkbox.");
return;
}
List <ImportingIntoLibrary.FilenameWithMetadataImport> filename_and_bibtex_imports = new List <ImportingIntoLibrary.FilenameWithMetadataImport>();
foreach (AugmentedBindable <BibTeXEntry> entry in allEntries)
{
ImportingIntoLibrary.FilenameWithMetadataImport filename_with_metadata_import = new ImportingIntoLibrary.FilenameWithMetadataImport
{
filename = entry.Underlying.Filename,
bibtex = entry.Underlying.BibTeX,
tags = entry.Underlying.Tags,
notes = entry.Underlying.Notes
};
filename_and_bibtex_imports.Add(filename_with_metadata_import);
}
StatusManager.Instance.UpdateStatus("ImportFromThirdParty", "Started importing documents");
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(_library, false, false, filename_and_bibtex_imports.ToArray());
MessageBoxes.Info("{0} files are now being imported - this may take a little while. You can track the import progress in the status bar.", filename_and_bibtex_imports.Count);
this.Close();
}
private void BtnImport_OnClick(object sender, RoutedEventArgs e)
{
try
{
var root_folder = bindable.Underlying.SelectedPath;
if (!Directory.Exists(root_folder))
{
return;
}
// do the import
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryFromFolder_ASYNCHRONOUS(library, root_folder, bindable.Underlying.RecurseSubfolders, bindable.Underlying.ImportTagsFromSubfolderNames, false, false);
// remember settings for next time
bindable.Underlying.DefaultSelectedPath = root_folder;
bindable.Underlying.DefaultRecurseSubfolders = bindable.Underlying.RecurseSubfolders;
bindable.Underlying.DefaultImportTagsFromSubfolderNames = bindable.Underlying.ImportTagsFromSubfolderNames;
Close();
}
catch (Exception exception)
{
Logging.Error(exception, "Problem importing files from {0}", bindable.Underlying.SelectedPath);
}
}
private void MoveGuestPreviewPDFDocument(WebLibraryDetail web_library_detail)
{
PDFDocument source_pdf_document = pdf_renderer_control_stats.pdf_document;
SafeThreadPool.QueueUserWorkItem(o =>
{
PDFDocument cloned_pdf_document = ImportingIntoLibrary.ClonePDFDocumentsFromOtherLibrary_SYNCHRONOUS(source_pdf_document, web_library_detail.library, false);
WPFDoEvents.InvokeInUIThread(() =>
{
// Open the new
if (null != cloned_pdf_document)
{
MainWindowServiceDispatcher.Instance.OpenDocument(cloned_pdf_document);
}
else
{
MessageBoxes.Warn("There was a problem moving this document to another library.");
}
// Close the old
MainWindowServiceDispatcher.Instance.ClosePDFReadingControl(this);
// Delete the old
if (cloned_pdf_document != source_pdf_document)
{
source_pdf_document.Deleted = true;
source_pdf_document.Bindable.NotifyPropertyChanged(nameof(source_pdf_document.Deleted));
}
});
});
}
private void ProcessTheNewDocuments(List <string> filenames_that_are_new)
{
if (0 == filenames_that_are_new.Count)
{
return;
}
if (Utilities.Shutdownable.ShutdownableManager.Instance.IsShuttingDown)
{
Logging.Info("FolderWatcher: Breaking out due to daemon termination");
return;
}
if (Qiqqa.Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
{
Logging.Info("FolderWatcher: Breaking out due to DisableAllBackgroundTasks");
return;
}
// Create the import records
List <FilenameWithMetadataImport> filename_with_metadata_imports = new List <FilenameWithMetadataImport>();
foreach (var filename in filenames_that_are_new)
{
filename_with_metadata_imports.Add(new FilenameWithMetadataImport
{
filename = filename,
tags = new HashSet <string>(tags)
});
// TODO: refactor this: delay until the PDF has actually been processed completely!
//
// Add this file to the list of processed files...
folder_watcher_manager.RememberProcessedFile(filename);
}
// Get the library to import all these new files
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(library, true, true, filename_with_metadata_imports.ToArray());
// TODO: refactor the ImportingIntoLibrary class
//
// HACK & QUICK PATCH until we have refactored this stuff:
filenames_that_are_new.Clear();
}
void ButtonGrabPDFs_Click(object sender, RoutedEventArgs e)
{
Uri current_uri = CurrentWebBrowserControl.CurrentUri;
string html = CurrentWebBrowserControl.PageHTML;
List <string> urls = DownloadableFileGrabber.Grab(html, "pdf");
List <Uri> uris = new List <Uri>();
foreach (string url in urls)
{
Uri uri;
if (Uri.TryCreate(current_uri, url, out uri))
{
uris.Add(uri);
}
}
if (0 < uris.Count)
{
string msg = String.Format(
"Qiqqa has found {0} PDFs on this page. Please choose the library into which you want to import them."
, uris.Count
);
WebLibraryDetail web_library_detail = WebLibraryPicker.PickWebLibrary(msg);
if (null != web_library_detail)
{
foreach (Uri uri in uris)
{
ImportingIntoLibrary.AddNewDocumentToLibraryFromInternet_ASYNCHRONOUS(web_library_detail.library, uri.ToString());
}
}
else
{
MessageBoxes.Warn("No PDFs have been imported.");
}
}
else
{
MessageBoxes.Info("Qiqqa could not find links to any PDFs on this page (with URLs ending in .pdf");
}
}
private void MoveOrCopyCommon(Feature feature, bool delete_source_pdf_documents)
{
WebLibraryDetail web_library_detail = WebLibraryPicker.PickWebLibrary();
if (null == web_library_detail)
{
Logging.Warn("User did not pick a library to copy or move to: pick = NULL.");
return;
}
// Check that we are not moving any docs into the same library
bool same_library = false;
foreach (var pdf_document in pdf_documents)
{
if (pdf_document.Library.WebLibraryDetail == web_library_detail)
{
same_library = true;
}
}
if (same_library)
{
MessageBoxes.Error("You can not move/copy a PDF from/to the same library.");
return;
}
// Copying / Moving PDFDocuments takes a while, particularly if it's a large set.
//
// Hence this WORK should be executed by a background task.
SafeThreadPool.QueueUserWorkItem(o =>
{
FeatureTrackingManager.Instance.UseFeature(feature);
ImportingIntoLibrary.ClonePDFDocumentsFromOtherLibrary_SYNCHRONOUS(pdf_documents, web_library_detail.library, delegate(PDFDocument target, PDFDocument source)
{
if (delete_source_pdf_documents && null != target && null != source && target != source)
{
source.Library.DeleteDocument(source);
}
});
});
}
private static void DoImportMyDocuments(object obj)
{
if (null == mdd)
{
Logging.Warn("Not sure how MendeleyImporter.MendeleyDatabaseDetails is null if we got a command to import...");
return;
}
FeatureTrackingManager.Instance.UseFeature(Features.Library_ImportAutoFromMendeley);
WebLibraryDetail web_library_detail = null;
WPFDoEvents.InvokeInUIThread(() =>
web_library_detail = WebLibraryPicker.PickWebLibrary()
);
if (null != web_library_detail)
{
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(web_library_detail.library, false, false, mdd.metadata_imports.ToArray());
}
}
static void DoImportMyDocuments(object obj)
{
if (null == mdd)
{
Logging.Warn("Not sure how EndnoteImporter.EndnoteDatabaseDetails is null if we got a command to import...");
return;
}
Qiqqa.UtilisationTracking.FeatureTrackingManager.Instance.UseFeature(Features.Library_ImportAutoFromEndNote);
WebLibraryDetail web_library_detail = null;
Application.Current.Dispatcher.Invoke(((Action)(() =>
web_library_detail = WebLibraryPicker.PickWebLibrary()
)));
if (null != web_library_detail)
{
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(web_library_detail.library, false, false, mdd.metadata_imports.ToArray());
}
}
private void MoveOrCopyCommon(Feature feature, bool delete_source_pdf_documents)
{
WebLibraryDetail web_library_detail = WebLibraryPicker.PickWebLibrary();
if (null == web_library_detail)
{
return;
}
// Check that we are not moving any docs into the same library
bool same_library = false;
foreach (var pdf_document in pdf_documents)
{
if (pdf_document.Library.WebLibraryDetail == web_library_detail)
{
same_library = true;;
}
}
if (same_library)
{
MessageBoxes.Error("You can not move/copy a PDF from/to the same library.");
return;
}
FeatureTrackingManager.Instance.UseFeature(feature);
ImportingIntoLibrary.ClonePDFDocumentsFromOtherLibrary_SYNCHRONOUS(pdf_documents, web_library_detail.library);
if (delete_source_pdf_documents)
{
foreach (var pdf_document in pdf_documents)
{
pdf_document.Deleted = true;
pdf_document.Bindable.NotifyPropertyChanged(() => pdf_document.Deleted);
}
}
}
void CmdAutomaticEndnoteImport_Click(object sender, RoutedEventArgs e)
{
Qiqqa.UtilisationTracking.FeatureTrackingManager.Instance.UseFeature(Features.Library_ImportAutoFromEndNote);
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(this._library, false, false, edd.metadata_imports.ToArray());
this.Close();
}
private void CmdAutomaticMendeleyImport_Click(object sender, RoutedEventArgs e)
{
FeatureTrackingManager.Instance.UseFeature(Features.Library_ImportAutoFromMendeley);
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(_library, false, false, mdd.metadata_imports.ToArray());
Close();
}
/// <summary>
/// The daemon code calls this occasionally to poke it into action to do work
/// </summary>
/// <param name="daemon"></param>
public void ExecuteBackgroundProcess(Daemon daemon)
{
// We don't want to start watching files until the library is loaded...
if (!(LibraryRef?.Xlibrary.LibraryIsLoaded ?? false))
{
Logging.Info("Library is not yet loaded, so waiting before watching...");
// Indicate that the library may still not have been changed...
FolderContentsHaveChanged = true;
return;
}
// Update our folder system watcher if necessary
CheckIfFolderNameHasChanged();
// If the current folder is blank, do nothing
if (String.IsNullOrEmpty(configured_folder_to_watch))
{
return;
}
// If the folder does not exist, do nothing
if (!Directory.Exists(configured_folder_to_watch))
{
Logging.Info("Watched folder {0} does not exist: watching this directory has been disabled.", configured_folder_to_watch);
return;
}
// If the folder or its contents has not changed since the last time, do nothing
if (!FolderContentsHaveChanged)
{
return;
}
if (!ConfigurationManager.IsEnabled(nameof(FolderWatcher)))
{
Logging.Info("Watched folder {0} will not be watched/scanned due to Developer Override setting {1}=false", configured_folder_to_watch, nameof(FolderWatcher));
return;
}
Stopwatch breathing_time = Stopwatch.StartNew();
Logging.Debug("FolderWatcher BEGIN");
// To recover from a fatal library failure and re-indexing attempt for very large libraries,
// we're better off processing a limited number of source files as we'll be able to see
// *some* results more quickly and we'll have a working, though yet incomplete,
// index in *reasonable time*.
//
// To reconstruct the entire index will take a *long* time. We grow the index and other meta
// stores a bunch-of-files at a time and then repeat the entire maintenance process until
// we'll be sure to have run out of files to process for sure...
// Mark that we are now processing the folder
while (TestAndReset_FolderContentsHaveChanged())
{
// If this library is busy, skip it for now
if (Library.IsBusyAddingPDFs || Library.IsBusyRegeneratingTags)
{
Logging.Debug特("FolderWatcher: Not daemon processing any library that is busy with adds...");
FolderContentsHaveChanged = true;
break;
}
if (ShutdownableManager.Instance.IsShuttingDown)
{
Logging.Debug特("FolderWatcher: Breaking out of outer processing loop due to daemon termination");
FolderContentsHaveChanged = true;
break;
}
if (Qiqqa.Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
{
Logging.Debug特("FolderWatcher: Breaking out of outer processing loop due to DisableAllBackgroundTasks");
FolderContentsHaveChanged = true;
break;
}
if (LibraryRef == null || folder_watcher_manager?.TypedTarget == null)
{
Logging.Debug特("FolderWatcher: Breaking out of outer processing loop due to disposed library and/or watch manager");
FolderContentsHaveChanged = true;
break;
}
if (!ConfigurationManager.IsEnabled(nameof(FolderWatcher)))
{
Logging.Info("Watched folder {0} will not be watched/scanned due to Developer Override setting {1}=false", configured_folder_to_watch, nameof(FolderWatcher));
break;
}
// reset counters for logging/reporting:
watch_stats.Reset(daemon);
// If we get this far then there might be some work to do in the folder...
Stopwatch clk = Stopwatch.StartNew();
//
// Summary:
// [AlphaFS] Specifies a set of custom filters to be used with enumeration methods
// of Alphaleonis.Win32.Filesystem.Directory, e.g., Alphaleonis.Win32.Filesystem.Directory.EnumerateDirectories(System.String),
// Alphaleonis.Win32.Filesystem.Directory.EnumerateFiles(System.String), or Alphaleonis.Win32.Filesystem.Directory.EnumerateFileSystemEntries(System.String).
//
// Remarks:
// Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters allows scenarios in
// which files/directories being enumerated by the methods of Alphaleonis.Win32.Filesystem.Directory
// class are accepted only if they match the search pattern, attributes (see Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.SkipReparsePoints),
// and optionally also the custom criteria tested in the method whose delegate is
// specified in Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.InclusionFilter.
// These criteria could be, e.g., file size exceeding some threshold, pathname matches
// a complex regular expression, etc. If the enumeration process is set to be recursive
// (see Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.Recursive) and
// Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.RecursionFilter is specified,
// the directory is traversed recursively only if it matches the custom criteria
// in Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.RecursionFilter method.
// This allows, for example, custom handling of junctions and symbolic links, e.g.,
// detection of cycles. If any error occurs during the enumeration and the enumeration
// process is not set to ignore errors (see Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.ContinueOnException),
// an exception is thrown unless the error is handled (filtered out) by the method
// specified in Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.ErrorFilter
// (if specified). The method may, for example, consume the error by reporting it
// in a log, so that the enumeration continues as in the case of Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.ContinueOnException
// option but the user will be informed about errors.
//
global_watch_stats.Inc();
DirectoryEnumerationFilters filter = new DirectoryEnumerationFilters();
filter.ErrorFilter = DecideIfErrorDuringDirScan;
filter.InclusionFilter = DecideIfIncludeDuringDirScan;
filter.RecursionFilter = DecideIfRecurseDuringDirScan;
// Note: don't use the CancellationToken, just throw an exception in the InclusionFilter when it's time to abort the scan.
//filter.CancellationToken = null;
IEnumerable <string> filenames_in_folder = Directory.EnumerateFiles(configured_folder_to_watch,
DirectoryEnumerationOptions.Files |
DirectoryEnumerationOptions.BasicSearch |
//DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.LargeCache |
DirectoryEnumerationOptions.Recursive,
filter);
// SearchOption.AllDirectories);
Logging.Debug特("Directory.EnumerateFiles took {0} ms", clk.ElapsedMilliseconds);
// Do NOT count files which are already present in our library/DB,
// despite the fact that those also *do* take time and effort to check
// in the code above.
//
// The issue here is that when we would import files A,B,C,D,E,F,G,H,I,J,K,
// we would do so in tiny batches, resulting in a rescan after each batch
// where the already processed files will be included in the set, but must
// be filtered out as 'already in there' in the code above.
// Iff we had counted *all* files we inspect from the Watch Directory,
// we would never make it batch the first batch as then our count limit
// would trigger already for every round through here!
List <string> filenames_that_are_new = new List <string>();
foreach (string filename in filenames_in_folder)
{
Logging.Info("FolderWatcher: {0} of {1} files have been processed/inspected (total {2} scanned, {3} skipped, {4} ignored)", watch_stats.processed_file_count, watch_stats.processing_file_count, watch_stats.scanned_file_count, watch_stats.skipped_file_count, watch_stats.scanned_file_count - watch_stats.skipped_file_count - watch_stats.processing_file_count);
try
{
// check the file once again: it MAY have disappeared while we were slowly scanning the remainder of the dirtree.
FileSystemEntryInfo info = File.GetFileSystemEntryInfo(filename);
watch_stats.processing_file_count++;
Logging.Info("FolderWatcher is importing {0}", filename);
filenames_that_are_new.Add(filename);
}
catch (Exception ex)
{
Logging.Error(ex, "Folder Watcher: skipping file {0} due to file I/O error {1}", filename, ex.Message);
}
}
Logging.Debug特("Directory.EnumerateFiles took {0} ms", clk.ElapsedMilliseconds);
// Create the import records
List <FilenameWithMetadataImport> filename_with_metadata_imports = new List <FilenameWithMetadataImport>();
foreach (var filename in filenames_that_are_new)
{
filename_with_metadata_imports.Add(new FilenameWithMetadataImport
{
filename = filename,
tags = new HashSet <string>(tags)
});
#if false
// delay until the PDF has actually been processed completely!
//
// Add this file to the list of processed files...
folder_watcher_manager.RememberProcessedFile(filename);
#endif
}
// Get the library to import all these new files
if (filename_with_metadata_imports.Count > 0)
{
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_SYNCHRONOUS(LibraryRef, true, filename_with_metadata_imports.ToArray());
// TODO: refactor the ImportingIntoLibrary class
}
watch_stats.processed_file_count = watch_stats.processing_file_count;
Logging.Info("FolderWatcher: {0} of {1} files have been processed/inspected (total {2} scanned, {3} skipped, {4} ignored)", watch_stats.processed_file_count, watch_stats.processing_file_count, watch_stats.scanned_file_count, watch_stats.skipped_file_count, watch_stats.scanned_file_count - watch_stats.skipped_file_count - watch_stats.processing_file_count);
if (watch_stats.index_processing_clock.ElapsedMilliseconds >= FolderWatcher.MAX_SECONDS_PER_ITERATION)
{
Logging.Info("FolderWatcher: Taking a nap due to MAX_SECONDS_PER_ITERATION: {0} seconds consumed, {1} threads pending", watch_stats.index_processing_clock.ElapsedMilliseconds / 1E3, SafeThreadPool.QueuedThreadCount);
watch_stats.daemon.Sleep(SECONDS_TO_RELAX_PER_ITERATION);
watch_stats.index_processing_clock.Restart();
}
Logging.Debug("FolderWatcher End-Of-Round ({0} ms)", clk.ElapsedMilliseconds);
}
Logging.Debug("FolderWatcher END");
}
/// <summary>
/// The daemon code calls this occasionally to poke it into action to do work
/// </summary>
/// <param name="daemon"></param>
public void TaskDaemonEntryPoint(Daemon daemon)
{
// We don't want to start watching files until the library is loaded...
if (!library.LibraryIsLoaded)
{
Logging.Info("Library is not yet loaded, so waiting before watching...");
// Indicate that the library may still not have been changed...
folder_contents_has_changed = true;
return;
}
// Update our fole system watcher if necessary
CheckIfFolderNameHasChanged();
// If the current folder is blank, do nothing
if (String.IsNullOrEmpty(folder_to_watch))
{
return;
}
// If the folder does not exist, do nothing
if (!Directory.Exists(folder_to_watch))
{
return;
}
// If the folder or its contents has not changed since the last time, do nothing
if (!folder_contents_has_changed)
{
return;
}
// Mark that we are now processing the folder
folder_contents_has_changed = false;
// If we get this far then there might be some work to do in the folder...
string[] filenames_in_folder = Directory.GetFiles(previous_folder_to_watch, "*.pdf", SearchOption.AllDirectories);
List <PDFDocument> pdf_documents_already_in_library = library.PDFDocuments;
List <string> filenames_that_are_new = new List <string>();
foreach (string filename in filenames_in_folder)
{
// If we already have this file in the "cache since we started", skip it
if (folder_watcher_manager.HaveProcessedFile(filename))
{
//Logging.Info("FolderWatcher is skipping {0} as it has already been processed", filename);
continue;
}
// If we already have this file in the "pdf file locations", skip it
bool is_already_in_library = false;
foreach (PDFDocument pdf_document in pdf_documents_already_in_library)
{
if (pdf_document.DownloadLocation == filename)
{
is_already_in_library = true;
break;
}
}
if (is_already_in_library)
{
// Add this file to the list of processed files...
folder_watcher_manager.RememberProcessedFile(filename);
continue;
}
// Check that the file is not still locked - if it is, mark that the folder is still "changed" and come back later..
if (IsFileLocked(filename))
{
Logging.Info("Watched folder contains file '{0}' which is locked, so coming back later...", filename);
folder_contents_has_changed = true;
continue;
}
Logging.Info("FolderWatcher is importing {0}", filename);
filenames_that_are_new.Add(filename);
// Add this file to the list of processed files...
folder_watcher_manager.RememberProcessedFile(filename);
}
// Create the import records
List <ImportingIntoLibrary.FilenameWithMetadataImport> filename_with_metadata_imports = new List <ImportingIntoLibrary.FilenameWithMetadataImport>();
foreach (var filename in filenames_that_are_new)
{
filename_with_metadata_imports.Add(new ImportingIntoLibrary.FilenameWithMetadataImport {
filename = filename, tags = new List <string>(this.tags)
});
}
// Get the library to import all these new files
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_ASYNCHRONOUS(library, true, true, filename_with_metadata_imports.ToArray());
}
