/// <summary>
/// Initializes the DataflowBlock
/// </summary>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="inputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InputBlock"/></param>
/// <param name="innerBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InnerBlock"/></param>
/// <param name="outputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="OutputBlock"/></param>
/// <param name="blockname">Name of this DataflowBlock (must be unique in a given chain of DataflowBlocks)</param>
/// <param name="isRestorePointAfterDeserialization">
/// <c>true</c>, if after deserialiization from disk, <see cref="PendingImportResourceNewGen"/>s are restored to
/// this block. If <c>false</c>, they are restored to the last passed DataflowBlock having this parameter set to <c>true</c>
/// </param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
protected ImporterWorkerDataflowBlockBase(ImportJobInformation importJobInformation, ExecutionDataflowBlockOptions inputBlockOptions, ExecutionDataflowBlockOptions innerBlockOptions, ExecutionDataflowBlockOptions outputBlockOptions, String blockname, bool isRestorePointAfterDeserialization, ImportJobController parentImportJobController, CancellationToken ct)
{
_blockName = blockname;
_isRestorePointAfterDeserialization = isRestorePointAfterDeserialization;
_ct = ct;
ImportJobInformation = importJobInformation;
InputBlockOptions = inputBlockOptions;
InnerBlockOptions = innerBlockOptions;
OutputBlockOptions = outputBlockOptions;
ParentImportJobController = parentImportJobController;
_stopWatch = new Stopwatch();
_tcs = new TaskCompletionSource <object>();
Activated = new AsyncManualResetEvent(InnerBlockOptions.CancellationToken);
InputBlock = new TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => InputBlockMethod(p), InputBlockOptions);
OutputBlock = new TransformManyBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => OutputBlockMethod(p), OutputBlockOptions);
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
InnerBlock = CreateInnerBlock();
InnerBlock.LinkTo(OutputBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
InputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
InnerBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
OutputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(InputBlock.Completion, InnerBlock.Completion, OutputBlock.Completion).ContinueWith(OnAllBlocksFinished);
}
/// <summary>
/// Initiates the DirectoryUnfoldBlock
/// </summary>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public DirectoryUnfoldBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, true, parentImportJobController, ct)
{
}
/// <summary>
/// Initiates the DirectorySaveBlock
/// </summary>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, true, parentImportJobController)
{
}
/// <summary>
/// Initiates the MediaItemSaveBlock
/// </summary>
/// <remarks>
/// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
/// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
/// therefore does not need a BoundedCapacity.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 500
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, false, parentImportJobController)
{
}
private void DoScheduleImport(ImportJobInformation importJobInformation)
{
if (_status == Status.Shutdown)
{
ServiceRegistration.Get <ILogger>().Error("ImporterWorker: Scheduling of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'");
return;
}
// For now we always set this to active to make it look like the old ImporterWorker
// ToDo: Remove this and all usages of ImportJobInformation.State
importJobInformation.State = ImportJobState.Active;
// if the ImportJob to be scheduled is the same as or contains an
// already running ImportJob, cancel the already running ImportJob
// and schedule this one
var jobsToBeCancelled = new HashSet <ImportJobController>();
foreach (var kvp in _importJobControllers)
{
if (importJobInformation >= kvp.Key)
{
ServiceRegistration.Get <ILogger>().Info("ImporterWorker: {0} is contained in or the same as the ImportJob which is currently being scheduled. Canceling {1}", kvp.Value, kvp.Value);
kvp.Value.Cancel();
jobsToBeCancelled.Add(kvp.Value);
}
}
// We need to wait here until the canceled ImportJobs are removed from _importJobControllers
// otherwise we run into trouble when the ImportJobs equal each other because then they
// have the same key in _importJobControllers.
Task.WhenAll(jobsToBeCancelled.Select(controller => controller.Completion)).Wait();
foreach (var controller in jobsToBeCancelled)
{
controller.Dispose();
}
//Set updated media items to changed
_mediaBrowsing?.MarkUpdatableMediaItems();
var importJobController = new ImportJobController(new ImportJobNewGen(importJobInformation, null), Interlocked.Increment(ref _numberOfLastImportJob), this);
_importJobControllers[importJobInformation] = importJobController;
ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Scheduled {0} ({1}) (Path ='{2}', ImportJobType='{3}', IncludeSubdirectories='{4}')", importJobController, _status, importJobInformation.BasePath, importJobInformation.JobType, importJobInformation.IncludeSubDirectories);
ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, importJobInformation.BasePath, importJobInformation.JobType);
if (_status == Status.Activated)
{
importJobController.Activate(_mediaBrowsing, _importResultHandler);
}
}
/// <summary>
/// Initiates the MetadataExtractorBlock
/// </summary>
/// <remarks>
/// The preceding MediaItemLoadBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
BLOCK_NAME, true, parentImportJobController, ct)
{
_forceQuickMode = forceQuickMode;
}
/// <summary>
/// Initiates the MediaItemLoadBlock
/// </summary>
/// <remarks>
/// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MediaItemLoadBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 500
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
BLOCK_NAME, false, parentImportJobController, ct)
{
_mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate);
}
/// <summary>
/// Initiates the MetadataExtractorBlock
/// </summary>
/// <remarks>
/// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
/// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param>
public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController)
{
_forceQuickMode = forceQuickMode;
_mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate);
}
/// <summary>
/// Initiates the RelationshipExtractorBlock
/// </summary>
/// <remarks>
/// The preceding MediaItemSaveBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this RelationshipExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 50,
/// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
/// therefore does not need a BoundedCapacity.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public RelationshipExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 50
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, true, parentImportJobController, ct)
{
_relationshipCache = new LRURelationshipCache();
_cacheSync = new SemaphoreSlim(1, 1);
_processedMediaItemIds = new ConcurrentDictionary <Guid, byte>();
}
public ImportJobController(ImportJobNewGen importJob, int importJobNumber, ImporterWorkerNewGen parentImporterWorker)
{
_importJobInformation = importJob.ImportJobInformation;
_importJobNumber = importJobNumber;
_parentImporterWorker = parentImporterWorker;
_numberOfLastPendingImportResource = 0;
_numberOfDisposedPendingImportResources = 0;
_notifyProgress = true;
_pendingImportResources = new ConcurrentDictionary <ResourcePath, PendingImportResourceNewGen>();
_importJobControllerCompletion = new TaskCompletionSource <object>();
_firstBlockHasFinished = new TaskCompletionSource <object>();
_cts = new CancellationTokenSource();
_parentImporterWorker.NotifyProgress(true);
_dataflowBlocks = new List <ImporterWorkerDataflowBlockBase>();
SetupDataflowBlocks(importJob.PendingImportResources);
_dataflowBlocks.ForEach(block => block.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted));
Task.WhenAll(_dataflowBlocks.Select(block => block.Completion)).ContinueWith(OnFinished);
}
public ImporterWorkerAction(ActionType actionType, ImportJobInformation importJobInformation)
{
if (actionType == ActionType.Startup)
{
throw new ArgumentException("ActionType.Startup must not relate to an ImportJobInformation");
}
if (actionType == ActionType.Activate)
{
throw new ArgumentException("ActionType.Activate must not relate to an ImportJobInformation and requires an IMediaBrowsing and an IImportResultHandler");
}
if (actionType == ActionType.Suspend)
{
throw new ArgumentException("ActionType.Suspend must not relate to an ImportJobInformation");
}
if (actionType == ActionType.Shutdown)
{
throw new ArgumentException("ActionType.Shutdown must not relate to an ImportJobInformation");
}
_actionType = actionType;
_importJobInformation = importJobInformation;
_mediaBrowsingCallback = null;
_importResultHandler = null;
_tcs = new TaskCompletionSource <object>();
}
protected async Task <Guid> UpdateMediaItem(Guid parentDirectoryId, ResourcePath path, Guid mediaItemId, IEnumerable <MediaItemAspect> updatedAspects, ImportJobInformation jobInfo, bool isRefresh)
{
while (true)
{
try
{
await Activated.WaitAsync().ConfigureAwait(false);
// ReSharper disable PossibleMultipleEnumeration
return(await _importResultHandler.UpdateMediaItemAsync(parentDirectoryId, path, mediaItemId, updatedAspects, isRefresh, jobInfo.BasePath).ConfigureAwait(false));
// ReSharper restore PossibleMultipleEnumeration
}
catch (DisconnectedException)
{
await MedialibraryDisconnectedSuspend();
}
}
}
/// <summary>
/// Tries to remove an ImportJobController from <see cref="_importJobControllers"/>
/// </summary>
/// <param name="importJobInformation">
/// <see cref="ImportJobInformation"/> describing the <see cref="ImportJobController"/> to be removed
/// </param>
/// <returns>true if removal was successful, otherwise false</returns>
internal bool TryUnregisterImportJobController(ImportJobInformation importJobInformation)
{
ImportJobController removedController;
return(_importJobControllers.TryRemove(importJobInformation, out removedController));
}
public ImportJob(ImportJobType jobType, ResourcePath basePath, IEnumerable <Guid> metadataExtractorIds,
bool includeSubDirectories)
{
_jobInfo = new ImportJobInformation(jobType, basePath, new List <Guid>(metadataExtractorIds), includeSubDirectories);
}
protected async Task <Guid> UpdateMediaItem(Guid parentDirectoryId, ResourcePath path, IEnumerable <MediaItemAspect> updatedAspects, ImportJobInformation jobInfo, bool isRefresh, CancellationToken cancelToken)
{
while (true)
{
try
{
await Activated.WaitAsync();
// ReSharper disable PossibleMultipleEnumeration
return(_importResultHandler.UpdateMediaItem(parentDirectoryId, path, updatedAspects, isRefresh, jobInfo.BasePath, cancelToken));
// ReSharper restore PossibleMultipleEnumeration
}
catch (DisconnectedException)
{
ServiceRegistration.Get <ILogger>().Info("ImporterWorker.{0}.{1}: MediaLibrary disconnected. Requesting suspension...", ParentImportJobController, _blockName);
ParentImportJobController.ParentImporterWorker.RequestAction(new ImporterWorkerAction(ImporterWorkerAction.ActionType.Suspend)).Wait();
}
}
}
public ImportJobNewGen(ImportJobInformation importJobInformation, List <PendingImportResourceNewGen> pendingImportResources)
{
ImportJobInformation = importJobInformation;
PendingImportResources = pendingImportResources;
}
