/// <summary>
/// Initiates the MediaItemSaveBlock
/// </summary>
/// <remarks>
/// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
/// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
/// therefore does not need a BoundedCapacity.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 500 },
new ExecutionDataflowBlockOptions { CancellationToken = ct },
BLOCK_NAME, false, parentImportJobController)
{
}
public ImportJobInformation(ImportJobInformation other)
{
_jobType = other.JobType;
_basePath = other.BasePath;
_metadataExtractorIds = new HashSet <Guid>(other.MetadataExtractorIds);
_includeSubDirectories = other.IncludeSubDirectories;
_state = other.State;
}
/// <summary>
/// Initiates the DirectorySaveBlock
/// </summary>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions { CancellationToken = ct },
new ExecutionDataflowBlockOptions { CancellationToken = ct },
new ExecutionDataflowBlockOptions { CancellationToken = ct },
BLOCK_NAME, true, parentImportJobController)
{
}
public ImportJobInformation(ImportJobInformation other)
{
_jobType = other.JobType;
_basePath = other.BasePath;
_metadataExtractorIds = new HashSet<Guid>(other.MetadataExtractorIds);
_includeSubDirectories = other.IncludeSubDirectories;
_state = other.State;
}
/// <summary>
/// Initiates the MetadataExtractorBlock
/// </summary>
/// <remarks>
/// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
/// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param>
public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
: base(importJobInformation,
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100 },
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController)
{
_forceQuickMode = forceQuickMode;
_mostRecentMiaCreationDate = new Lazy<Task<DateTime>>(GetMostRecentMiaCreationDate);
}
public override bool Equals(object obj)
{
if (!(obj is ImportJobInformation))
{
return(false);
}
ImportJobInformation other = (ImportJobInformation)obj;
return(_basePath == other._basePath && _jobType == other._jobType &&
_includeSubDirectories == other._includeSubDirectories &&
_state == other._state);
}
public ImporterWorkerAction(ActionType actionType, ImportJobInformation importJobInformation)
{
if (actionType == ActionType.Startup)
throw new ArgumentException("ActionType.Startup must not relate to an ImportJobInformation");
if (actionType == ActionType.Activate)
throw new ArgumentException("ActionType.Activate must not relate to an ImportJobInformation and requires an IMediaBrowsing and an IImportResultHandler");
if (actionType == ActionType.Suspend)
throw new ArgumentException("ActionType.Suspend must not relate to an ImportJobInformation");
if (actionType == ActionType.Shutdown)
throw new ArgumentException("ActionType.Shutdown must not relate to an ImportJobInformation");
_actionType = actionType;
_importJobInformation = importJobInformation;
_mediaBrowsingCallback = null;
_importResultHandler = null;
_tcs = new TaskCompletionSource<object>();
}
public ImportJobController(ImportJobNewGen importJob, int importJobNumber, ImporterWorkerNewGen parentImporterWorker)
{
_importJobInformation = importJob.ImportJobInformation;
_importJobNumber = importJobNumber;
_parentImporterWorker = parentImporterWorker;
_numberOfLastPendingImportResource = 0;
_numberOfDisposedPendingImportResources = 0;
_notifyProgress = true;
_pendingImportResources = new ConcurrentDictionary<ResourcePath, PendingImportResourceNewGen>();
_importJobControllerCompletion = new TaskCompletionSource<object>();
_firstBlockHasFinished = new TaskCompletionSource<object>();
_cts = new CancellationTokenSource();
_parentImporterWorker.NotifyProgress(true);
_dataflowBlocks = new List<ImporterWorkerDataflowBlockBase>();
SetupDataflowBlocks(importJob.PendingImportResources);
_dataflowBlocks.ForEach(block => block.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted));
Task.WhenAll(_dataflowBlocks.Select(block => block.Completion)).ContinueWith(OnFinished);
}
public ImportJob(ImportJobType jobType, ResourcePath basePath, IEnumerable<Guid> metadataExtractorIds,
bool includeSubDirectories)
{
_jobInfo = new ImportJobInformation(jobType, basePath, new List<Guid>(metadataExtractorIds), includeSubDirectories);
}
public ImportJobNewGen(ImportJobInformation importJobInformation, List<PendingImportResourceNewGen> pendingImportResources)
{
ImportJobInformation = importJobInformation;
PendingImportResources = pendingImportResources;
}
/// <summary>
/// Tries to remove an ImportJobController from <see cref="_importJobControllers"/>
/// </summary>
/// <param name="importJobInformation">
/// <see cref="ImportJobInformation"/> describing the <see cref="ImportJobController"/> to be removed
/// </param>
/// <returns>true if removal was successful, otherwise false</returns>
internal bool TryUnregisterImportJobController(ImportJobInformation importJobInformation)
{
ImportJobController removedController;
return _importJobControllers.TryRemove(importJobInformation, out removedController);
}
private void DoCancelImport(ImportJobInformation? importJobInformation)
{
if (_status == Status.Shutdown)
{
ServiceRegistration.Get<ILogger>().Error("ImporterWorker: Cancelation of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'");
return;
}
if (importJobInformation == null)
{
// Cancel all ImportJobs
foreach (var kvp in _importJobControllers)
kvp.Value.Cancel();
Task.WhenAll(_importJobControllers.Values.Select(i => i.Completion)).Wait();
foreach (var kvp in _importJobControllers)
kvp.Value.Dispose();
}
else
{
// Cancel only the ImportJobs for the specified path and all its child paths
var jobsToBeCanceled = new HashSet<ImportJobController>();
foreach (var kvp in _importJobControllers)
if (importJobInformation.Value.BasePath.IsSameOrParentOf(kvp.Key.BasePath))
{
kvp.Value.Cancel();
jobsToBeCanceled.Add(kvp.Value);
}
Task.WhenAll(jobsToBeCanceled.Select(controller => controller.Completion)).Wait();
foreach (var controller in jobsToBeCanceled)
controller.Dispose();
}
}
private void DoScheduleImport(ImportJobInformation importJobInformation)
{
if (_status == Status.Shutdown)
{
ServiceRegistration.Get<ILogger>().Error("ImporterWorker: Scheduling of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'");
return;
}
// For now we always set this to active to make it look like the old ImporterWorker
// ToDo: Remove this and all usages of ImportJobInformation.State
importJobInformation.State = ImportJobState.Active;
// if the ImportJob to be scheduled is the same as or contains an
// already running ImportJob, cancel the already running ImportJob
// and schedule this one
var jobsToBeCancelled = new HashSet<ImportJobController>();
foreach (var kvp in _importJobControllers)
if (importJobInformation >= kvp.Key)
{
ServiceRegistration.Get<ILogger>().Info("ImporterWorker: {0} is contained in or the same as the ImportJob which is currently being scheduled. Canceling {1}", kvp.Value, kvp.Value);
kvp.Value.Cancel();
jobsToBeCancelled.Add(kvp.Value);
}
// We need to wait here until the canceled ImportJobs are removed from _importJobControllers
// otherwise we run into trouble when the ImportJobs equal each other because then they
// have the same key in _importJobControllers.
Task.WhenAll(jobsToBeCancelled.Select(controller => controller.Completion)).Wait();
foreach (var controller in jobsToBeCancelled)
controller.Dispose();
var importJobController = new ImportJobController(new ImportJobNewGen(importJobInformation, null), Interlocked.Increment(ref _numberOfLastImportJob), this);
_importJobControllers[importJobInformation] = importJobController;
ServiceRegistration.Get<ILogger>().Info("ImporterWorker: Scheduled {0} ({1}) (Path ='{2}', ImportJobType='{3}', IncludeSubdirectories='{4}')", importJobController, _status, importJobInformation.BasePath, importJobInformation.JobType, importJobInformation.IncludeSubDirectories);
ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, importJobInformation.BasePath, importJobInformation.JobType);
if (_status == Status.Activated)
importJobController.Activate(_mediaBrowsing, _importResultHandler);
}