/// <summary> /// Initiates the MediaItemSaveBlock /// </summary> /// <remarks> /// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500, /// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and /// therefore does not need a BoundedCapacity. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 500 }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, BLOCK_NAME, false, parentImportJobController) { }
public ImportJobInformation(ImportJobInformation other) { _jobType = other.JobType; _basePath = other.BasePath; _metadataExtractorIds = new HashSet <Guid>(other.MetadataExtractorIds); _includeSubDirectories = other.IncludeSubDirectories; _state = other.State; }
/// <summary> /// Initiates the DirectorySaveBlock /// </summary> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, new ExecutionDataflowBlockOptions { CancellationToken = ct }, BLOCK_NAME, true, parentImportJobController) { }
public ImportJobInformation(ImportJobInformation other) { _jobType = other.JobType; _basePath = other.BasePath; _metadataExtractorIds = new HashSet<Guid>(other.MetadataExtractorIds); _includeSubDirectories = other.IncludeSubDirectories; _state = other.State; }
/// <summary> /// Initiates the MetadataExtractorBlock /// </summary> /// <remarks> /// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100, /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to /// set the BoundedCapacity of the OutputBlock to 1. /// </remarks> /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param> /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param> /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param> /// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param> public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode) : base(importJobInformation, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100 }, new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 }, forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController) { _forceQuickMode = forceQuickMode; _mostRecentMiaCreationDate = new Lazy<Task<DateTime>>(GetMostRecentMiaCreationDate); }
public override bool Equals(object obj) { if (!(obj is ImportJobInformation)) { return(false); } ImportJobInformation other = (ImportJobInformation)obj; return(_basePath == other._basePath && _jobType == other._jobType && _includeSubDirectories == other._includeSubDirectories && _state == other._state); }
public ImporterWorkerAction(ActionType actionType, ImportJobInformation importJobInformation) { if (actionType == ActionType.Startup) throw new ArgumentException("ActionType.Startup must not relate to an ImportJobInformation"); if (actionType == ActionType.Activate) throw new ArgumentException("ActionType.Activate must not relate to an ImportJobInformation and requires an IMediaBrowsing and an IImportResultHandler"); if (actionType == ActionType.Suspend) throw new ArgumentException("ActionType.Suspend must not relate to an ImportJobInformation"); if (actionType == ActionType.Shutdown) throw new ArgumentException("ActionType.Shutdown must not relate to an ImportJobInformation"); _actionType = actionType; _importJobInformation = importJobInformation; _mediaBrowsingCallback = null; _importResultHandler = null; _tcs = new TaskCompletionSource<object>(); }
public ImportJobController(ImportJobNewGen importJob, int importJobNumber, ImporterWorkerNewGen parentImporterWorker) { _importJobInformation = importJob.ImportJobInformation; _importJobNumber = importJobNumber; _parentImporterWorker = parentImporterWorker; _numberOfLastPendingImportResource = 0; _numberOfDisposedPendingImportResources = 0; _notifyProgress = true; _pendingImportResources = new ConcurrentDictionary<ResourcePath, PendingImportResourceNewGen>(); _importJobControllerCompletion = new TaskCompletionSource<object>(); _firstBlockHasFinished = new TaskCompletionSource<object>(); _cts = new CancellationTokenSource(); _parentImporterWorker.NotifyProgress(true); _dataflowBlocks = new List<ImporterWorkerDataflowBlockBase>(); SetupDataflowBlocks(importJob.PendingImportResources); _dataflowBlocks.ForEach(block => block.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted)); Task.WhenAll(_dataflowBlocks.Select(block => block.Completion)).ContinueWith(OnFinished); }
public ImportJob(ImportJobType jobType, ResourcePath basePath, IEnumerable<Guid> metadataExtractorIds, bool includeSubDirectories) { _jobInfo = new ImportJobInformation(jobType, basePath, new List<Guid>(metadataExtractorIds), includeSubDirectories); }
public ImportJobNewGen(ImportJobInformation importJobInformation, List<PendingImportResourceNewGen> pendingImportResources) { ImportJobInformation = importJobInformation; PendingImportResources = pendingImportResources; }
/// <summary> /// Tries to remove an ImportJobController from <see cref="_importJobControllers"/> /// </summary> /// <param name="importJobInformation"> /// <see cref="ImportJobInformation"/> describing the <see cref="ImportJobController"/> to be removed /// </param> /// <returns>true if removal was successful, otherwise false</returns> internal bool TryUnregisterImportJobController(ImportJobInformation importJobInformation) { ImportJobController removedController; return _importJobControllers.TryRemove(importJobInformation, out removedController); }
private void DoCancelImport(ImportJobInformation? importJobInformation) { if (_status == Status.Shutdown) { ServiceRegistration.Get<ILogger>().Error("ImporterWorker: Cancelation of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'"); return; } if (importJobInformation == null) { // Cancel all ImportJobs foreach (var kvp in _importJobControllers) kvp.Value.Cancel(); Task.WhenAll(_importJobControllers.Values.Select(i => i.Completion)).Wait(); foreach (var kvp in _importJobControllers) kvp.Value.Dispose(); } else { // Cancel only the ImportJobs for the specified path and all its child paths var jobsToBeCanceled = new HashSet<ImportJobController>(); foreach (var kvp in _importJobControllers) if (importJobInformation.Value.BasePath.IsSameOrParentOf(kvp.Key.BasePath)) { kvp.Value.Cancel(); jobsToBeCanceled.Add(kvp.Value); } Task.WhenAll(jobsToBeCanceled.Select(controller => controller.Completion)).Wait(); foreach (var controller in jobsToBeCanceled) controller.Dispose(); } }
private void DoScheduleImport(ImportJobInformation importJobInformation) { if (_status == Status.Shutdown) { ServiceRegistration.Get<ILogger>().Error("ImporterWorker: Scheduling of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'"); return; } // For now we always set this to active to make it look like the old ImporterWorker // ToDo: Remove this and all usages of ImportJobInformation.State importJobInformation.State = ImportJobState.Active; // if the ImportJob to be scheduled is the same as or contains an // already running ImportJob, cancel the already running ImportJob // and schedule this one var jobsToBeCancelled = new HashSet<ImportJobController>(); foreach (var kvp in _importJobControllers) if (importJobInformation >= kvp.Key) { ServiceRegistration.Get<ILogger>().Info("ImporterWorker: {0} is contained in or the same as the ImportJob which is currently being scheduled. Canceling {1}", kvp.Value, kvp.Value); kvp.Value.Cancel(); jobsToBeCancelled.Add(kvp.Value); } // We need to wait here until the canceled ImportJobs are removed from _importJobControllers // otherwise we run into trouble when the ImportJobs equal each other because then they // have the same key in _importJobControllers. Task.WhenAll(jobsToBeCancelled.Select(controller => controller.Completion)).Wait(); foreach (var controller in jobsToBeCancelled) controller.Dispose(); var importJobController = new ImportJobController(new ImportJobNewGen(importJobInformation, null), Interlocked.Increment(ref _numberOfLastImportJob), this); _importJobControllers[importJobInformation] = importJobController; ServiceRegistration.Get<ILogger>().Info("ImporterWorker: Scheduled {0} ({1}) (Path ='{2}', ImportJobType='{3}', IncludeSubdirectories='{4}')", importJobController, _status, importJobInformation.BasePath, importJobInformation.JobType, importJobInformation.IncludeSubDirectories); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, importJobInformation.BasePath, importJobInformation.JobType); if (_status == Status.Activated) importJobController.Activate(_mediaBrowsing, _importResultHandler); }