/// <summary> /// Initializes the DataflowBlock /// </summary> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="inputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InputBlock"/></param> /// <param name="innerBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InnerBlock"/></param> /// <param name="outputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="OutputBlock"/></param> /// <param name="blockname">Name of this DataflowBlock (must be unique in a given chain of DataflowBlocks)</param> /// <param name="isRestorePointAfterDeserialization"> /// <c>true</c>, if after deserialiization from disk, <see cref="PendingImportResourceNewGen"/>s are restored to /// this block. If <c>false</c>, they are restored to the last passed DataflowBlock having this parameter set to <c>true</c> /// </param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> protected ImporterWorkerDataflowBlockBase(ImportJobInformation importJobInformation, ExecutionDataflowBlockOptions inputBlockOptions, ExecutionDataflowBlockOptions innerBlockOptions, ExecutionDataflowBlockOptions outputBlockOptions, String blockname, bool isRestorePointAfterDeserialization, ImportJobController parentImportJobController, CancellationToken ct) { _blockName = blockname; _isRestorePointAfterDeserialization = isRestorePointAfterDeserialization; _ct = ct; ImportJobInformation = importJobInformation; InputBlockOptions = inputBlockOptions; InnerBlockOptions = innerBlockOptions; OutputBlockOptions = outputBlockOptions; ParentImportJobController = parentImportJobController; _stopWatch = new Stopwatch(); _tcs = new TaskCompletionSource <object>(); Activated = new AsyncManualResetEvent(InnerBlockOptions.CancellationToken); InputBlock = new TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => InputBlockMethod(p), InputBlockOptions); OutputBlock = new TransformManyBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => OutputBlockMethod(p), OutputBlockOptions); // ReSharper disable once DoNotCallOverridableMethodsInConstructor InnerBlock = CreateInnerBlock(); InnerBlock.LinkTo(OutputBlock, new DataflowLinkOptions { PropagateCompletion = true }); InputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted); InnerBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted); OutputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted); Task.WhenAll(InputBlock.Completion, InnerBlock.Completion, OutputBlock.Completion).ContinueWith(OnAllBlocksFinished); }
/// <summary> /// Initiates the DirectoryUnfoldBlock /// </summary> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public DirectoryUnfoldBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, BLOCK_NAME, true, parentImportJobController, ct) { }
/// <summary> /// Initiates the DirectorySaveBlock /// </summary> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, BLOCK_NAME, true, parentImportJobController) { }
/// <summary> /// Initiates the MediaItemSaveBlock /// </summary> /// <remarks> /// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500, /// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and /// therefore does not need a BoundedCapacity. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 500 }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, BLOCK_NAME, false, parentImportJobController) { }
private void DoScheduleImport(ImportJobInformation importJobInformation) { if (_status == Status.Shutdown) { ServiceRegistration.Get <ILogger>().Error("ImporterWorker: Scheduling of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'"); return; } // For now we always set this to active to make it look like the old ImporterWorker // ToDo: Remove this and all usages of ImportJobInformation.State importJobInformation.State = ImportJobState.Active; // if the ImportJob to be scheduled is the same as or contains an // already running ImportJob, cancel the already running ImportJob // and schedule this one var jobsToBeCancelled = new HashSet <ImportJobController>(); foreach (var kvp in _importJobControllers) { if (importJobInformation >= kvp.Key) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker: {0} is contained in or the same as the ImportJob which is currently being scheduled. Canceling {1}", kvp.Value, kvp.Value); kvp.Value.Cancel(); jobsToBeCancelled.Add(kvp.Value); } } // We need to wait here until the canceled ImportJobs are removed from _importJobControllers // otherwise we run into trouble when the ImportJobs equal each other because then they // have the same key in _importJobControllers. Task.WhenAll(jobsToBeCancelled.Select(controller => controller.Completion)).Wait(); foreach (var controller in jobsToBeCancelled) { controller.Dispose(); } //Set updated media items to changed _mediaBrowsing?.MarkUpdatableMediaItems(); var importJobController = new ImportJobController(new ImportJobNewGen(importJobInformation, null), Interlocked.Increment(ref _numberOfLastImportJob), this); _importJobControllers[importJobInformation] = importJobController; ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Scheduled {0} ({1}) (Path ='{2}', ImportJobType='{3}', IncludeSubdirectories='{4}')", importJobController, _status, importJobInformation.BasePath, importJobInformation.JobType, importJobInformation.IncludeSubDirectories); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, importJobInformation.BasePath, importJobInformation.JobType); if (_status == Status.Activated) { importJobController.Activate(_mediaBrowsing, _importResultHandler); } }
/// <summary> /// Initiates the MetadataExtractorBlock /// </summary> /// <remarks> /// The preceding MediaItemLoadBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100, /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to /// set the BoundedCapacity of the OutputBlock to 1. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, BLOCK_NAME, true, parentImportJobController, ct) { _forceQuickMode = forceQuickMode; }
/// <summary> /// Initiates the MediaItemLoadBlock /// </summary> /// <remarks> /// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500, /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to /// set the BoundedCapacity of the OutputBlock to 1. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public MediaItemLoadBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 500 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, BLOCK_NAME, false, parentImportJobController, ct) { _mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate); }
/// <summary> /// Initiates the MetadataExtractorBlock /// </summary> /// <remarks> /// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100, /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to /// set the BoundedCapacity of the OutputBlock to 1. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> /// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param> public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController) { _forceQuickMode = forceQuickMode; _mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate); }
/// <summary> /// Initiates the RelationshipExtractorBlock /// </summary> /// <remarks> /// The preceding MediaItemSaveBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this RelationshipExtractorBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 50, /// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and /// therefore does not need a BoundedCapacity. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public RelationshipExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 50 }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, BLOCK_NAME, true, parentImportJobController, ct) { _relationshipCache = new LRURelationshipCache(); _cacheSync = new SemaphoreSlim(1, 1); _processedMediaItemIds = new ConcurrentDictionary <Guid, byte>(); }
public ImportJobController(ImportJobNewGen importJob, int importJobNumber, ImporterWorkerNewGen parentImporterWorker) { _importJobInformation = importJob.ImportJobInformation; _importJobNumber = importJobNumber; _parentImporterWorker = parentImporterWorker; _numberOfLastPendingImportResource = 0; _numberOfDisposedPendingImportResources = 0; _notifyProgress = true; _pendingImportResources = new ConcurrentDictionary <ResourcePath, PendingImportResourceNewGen>(); _importJobControllerCompletion = new TaskCompletionSource <object>(); _firstBlockHasFinished = new TaskCompletionSource <object>(); _cts = new CancellationTokenSource(); _parentImporterWorker.NotifyProgress(true); _dataflowBlocks = new List <ImporterWorkerDataflowBlockBase>(); SetupDataflowBlocks(importJob.PendingImportResources); _dataflowBlocks.ForEach(block => block.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted)); Task.WhenAll(_dataflowBlocks.Select(block => block.Completion)).ContinueWith(OnFinished); }
public ImporterWorkerAction(ActionType actionType, ImportJobInformation importJobInformation) { if (actionType == ActionType.Startup) { throw new ArgumentException("ActionType.Startup must not relate to an ImportJobInformation"); } if (actionType == ActionType.Activate) { throw new ArgumentException("ActionType.Activate must not relate to an ImportJobInformation and requires an IMediaBrowsing and an IImportResultHandler"); } if (actionType == ActionType.Suspend) { throw new ArgumentException("ActionType.Suspend must not relate to an ImportJobInformation"); } if (actionType == ActionType.Shutdown) { throw new ArgumentException("ActionType.Shutdown must not relate to an ImportJobInformation"); } _actionType = actionType; _importJobInformation = importJobInformation; _mediaBrowsingCallback = null; _importResultHandler = null; _tcs = new TaskCompletionSource <object>(); }
protected async Task <Guid> UpdateMediaItem(Guid parentDirectoryId, ResourcePath path, Guid mediaItemId, IEnumerable <MediaItemAspect> updatedAspects, ImportJobInformation jobInfo, bool isRefresh) { while (true) { try { await Activated.WaitAsync().ConfigureAwait(false); // ReSharper disable PossibleMultipleEnumeration return(await _importResultHandler.UpdateMediaItemAsync(parentDirectoryId, path, mediaItemId, updatedAspects, isRefresh, jobInfo.BasePath).ConfigureAwait(false)); // ReSharper restore PossibleMultipleEnumeration } catch (DisconnectedException) { await MedialibraryDisconnectedSuspend(); } } }
/// <summary> /// Tries to remove an ImportJobController from <see cref="_importJobControllers"/> /// </summary> /// <param name="importJobInformation"> /// <see cref="ImportJobInformation"/> describing the <see cref="ImportJobController"/> to be removed /// </param> /// <returns>true if removal was successful, otherwise false</returns> internal bool TryUnregisterImportJobController(ImportJobInformation importJobInformation) { ImportJobController removedController; return(_importJobControllers.TryRemove(importJobInformation, out removedController)); }
public ImportJob(ImportJobType jobType, ResourcePath basePath, IEnumerable <Guid> metadataExtractorIds, bool includeSubDirectories) { _jobInfo = new ImportJobInformation(jobType, basePath, new List <Guid>(metadataExtractorIds), includeSubDirectories); }
protected async Task <Guid> UpdateMediaItem(Guid parentDirectoryId, ResourcePath path, IEnumerable <MediaItemAspect> updatedAspects, ImportJobInformation jobInfo, bool isRefresh, CancellationToken cancelToken) { while (true) { try { await Activated.WaitAsync(); // ReSharper disable PossibleMultipleEnumeration return(_importResultHandler.UpdateMediaItem(parentDirectoryId, path, updatedAspects, isRefresh, jobInfo.BasePath, cancelToken)); // ReSharper restore PossibleMultipleEnumeration } catch (DisconnectedException) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker.{0}.{1}: MediaLibrary disconnected. Requesting suspension...", ParentImportJobController, _blockName); ParentImportJobController.ParentImporterWorker.RequestAction(new ImporterWorkerAction(ImporterWorkerAction.ActionType.Suspend)).Wait(); } } }
public ImportJobNewGen(ImportJobInformation importJobInformation, List <PendingImportResourceNewGen> pendingImportResources) { ImportJobInformation = importJobInformation; PendingImportResources = pendingImportResources; }