public void ScheduleRefresh(ResourcePath path, IEnumerable <string> mediaCategories, bool includeSubDirectories) { ICollection <ImportJob> importJobs; lock (_syncObj) importJobs = new List <ImportJob>(_importJobs); ICollection <ImportJob> removeImportJobs = new List <ImportJob>(); foreach (ImportJob checkJob in importJobs) { if (checkJob.BasePath.IsSameOrParentOf(path)) { // The new job is included in an already existing job - re-schedule existing job path = checkJob.BasePath; checkJob.Cancel(); removeImportJobs.Add(checkJob); } if (path.IsParentOf(checkJob.BasePath)) { // The new job will include the old job checkJob.Cancel(); removeImportJobs.Add(checkJob); } } lock (_syncObj) foreach (ImportJob removeJob in removeImportJobs) { _importJobs.Remove(removeJob); } ICollection <Guid> metadataExtractorIds = GetMetadataExtractorIdsForMediaCategories(mediaCategories); ImportJob job = new ImportJob(ImportJobType.Refresh, path, metadataExtractorIds, includeSubDirectories); EnqueueImportJob(job); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, path, ImportJobType.Refresh); }
public void CancelJobsForPath(ResourcePath path) { ICollection <ImportJob> importJobs; lock (_syncObj) importJobs = new List <ImportJob>(_importJobs); ICollection <ImportJob> cancelImportJobs = new List <ImportJob>(); foreach (ImportJob job in importJobs) { if (path.IsSameOrParentOf(job.BasePath)) { job.Cancel(); cancelImportJobs.Add(job); } } lock (_syncObj) foreach (ImportJob job in cancelImportJobs) { if (_importJobs.Remove(job)) { ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduleCanceled, path); } } }
/// <summary> /// Imports the resource with the given <paramref name="mediaItemAccessor"/>. /// </summary> /// <remarks> /// This method will be called for file resources as well as for directory resources because some metadata extractors /// extract their metadata from directories. /// </remarks> /// <param name="mediaItemAccessor">File or directory resource to be imported.</param> /// <param name="parentDirectoryId">Media item id of the parent directory, if present, else <see cref="Guid.Empty"/>.</param> /// <param name="metadataExtractors">Collection of metadata extractors to apply to the given resoure.</param> /// <param name="resultHandler">Callback to notify the import results.</param> /// <param name="mediaAccessor">Convenience reference to the media accessor.</param> /// <returns><c>true</c>, if metadata could be extracted from the given <paramref name="mediaItemAccessor"/>, else /// <c>false</c>.</returns> protected bool ImportResource(ImportJob importJob, IResourceAccessor mediaItemAccessor, Guid parentDirectoryId, ICollection <IMetadataExtractor> metadataExtractors, IImportResultHandler resultHandler, IMediaAccessor mediaAccessor) { const bool forceQuickMode = false; // Allow extractions with probably longer runtime. ResourcePath path = mediaItemAccessor.CanonicalLocalResourcePath; ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportStatus, path); IDictionary <Guid, IList <MediaItemAspect> > aspects = mediaAccessor.ExtractMetadataAsync(mediaItemAccessor, metadataExtractors, forceQuickMode).Result; if (aspects == null) { // No metadata could be extracted return(false); } using (CancellationTokenSource cancelToken = new CancellationTokenSource()) { try { resultHandler.UpdateMediaItemAsync(parentDirectoryId, path, MediaItemAspect.GetAspects(aspects), importJob.JobType == ImportJobType.Refresh, importJob.BasePath); resultHandler.DeleteUnderPathAsync(path); } catch { cancelToken.Cancel(); throw; } } return(true); }
private void OnMessageReceived(AsynchronousMessageQueue queue, SystemMessage message) { if (message.ChannelName == SystemMessaging.CHANNEL) { SystemMessaging.MessageType messageType = (SystemMessaging.MessageType)message.MessageType; switch (messageType) { case SystemMessaging.MessageType.SystemStateChanged: SystemState newState = (SystemState)message.MessageData[SystemMessaging.NEW_STATE]; if (newState == SystemState.ShuttingDown) { IsSuspended = true; } break; } } if (message.ChannelName == TaskSchedulerMessaging.CHANNEL) { TaskSchedulerMessaging.MessageType messageType = (TaskSchedulerMessaging.MessageType)message.MessageType; switch (messageType) { case TaskSchedulerMessaging.MessageType.DUE: Task dueTask = (Task)message.MessageData[TaskSchedulerMessaging.TASK]; if (dueTask.ID == _importerTaskId) { // Forward a new message which will be handled by MediaLibrary (it knows the shares and configuration), then it will // schedule the local shares. ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.RefreshLocalShares); } break; } } }
/// <summary> /// Activates the ImportJob /// </summary> /// <param name="mediaBrowsingCallback"></param> /// <param name="importResultHandler"></param> public void Activate(IMediaBrowsing mediaBrowsingCallback, IImportResultHandler importResultHandler) { // To avoid peaks on system startup we start one Block every 100ms. // Currently we also need this because the MediaAccessor is not threadsafe on startup // see here: http://forum.team-mediaportal.com/threads/mediaaccessor-not-thread-safe.125132/ // ToDo: Make MediaAccessor threadsafe on startup foreach (var block in _dataflowBlocks) { block.Activate(mediaBrowsingCallback, importResultHandler); Task.Delay(100).Wait(); } ServiceRegistration.Get <ILogger>().Info("ImporterWorker.{0}: Activated", this); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportStarted, _importJobInformation.BasePath); }
private void DoScheduleImport(ImportJobInformation importJobInformation) { if (_status == Status.Shutdown) { ServiceRegistration.Get <ILogger>().Error("ImporterWorker: Scheduling of an ImportJob was requested although status was neither 'Activated' nor 'Suspended' but 'Shutdown'"); return; } // For now we always set this to active to make it look like the old ImporterWorker // ToDo: Remove this and all usages of ImportJobInformation.State importJobInformation.State = ImportJobState.Active; // if the ImportJob to be scheduled is the same as or contains an // already running ImportJob, cancel the already running ImportJob // and schedule this one var jobsToBeCancelled = new HashSet <ImportJobController>(); foreach (var kvp in _importJobControllers) { if (importJobInformation >= kvp.Key) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker: {0} is contained in or the same as the ImportJob which is currently being scheduled. Canceling {1}", kvp.Value, kvp.Value); kvp.Value.Cancel(); jobsToBeCancelled.Add(kvp.Value); } } // We need to wait here until the canceled ImportJobs are removed from _importJobControllers // otherwise we run into trouble when the ImportJobs equal each other because then they // have the same key in _importJobControllers. Task.WhenAll(jobsToBeCancelled.Select(controller => controller.Completion)).Wait(); foreach (var controller in jobsToBeCancelled) { controller.Dispose(); } //Set updated media items to changed _mediaBrowsing?.MarkUpdatableMediaItems(); var importJobController = new ImportJobController(new ImportJobNewGen(importJobInformation, null), Interlocked.Increment(ref _numberOfLastImportJob), this); _importJobControllers[importJobInformation] = importJobController; ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Scheduled {0} ({1}) (Path ='{2}', ImportJobType='{3}', IncludeSubdirectories='{4}')", importJobController, _status, importJobInformation.BasePath, importJobInformation.JobType, importJobInformation.IncludeSubDirectories); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, importJobInformation.BasePath, importJobInformation.JobType); if (_status == Status.Activated) { importJobController.Activate(_mediaBrowsing, _importResultHandler); } }
/// <summary> /// Imports the resource with the given <paramref name="mediaItemAccessor"/>. /// </summary> /// <remarks> /// This method will be called for file resources as well as for directory resources because some metadata extractors /// extract their metadata from directories. /// </remarks> /// <param name="mediaItemAccessor">File or directory resource to be imported.</param> /// <param name="parentDirectoryId">Media item id of the parent directory, if present, else <see cref="Guid.Empty"/>.</param> /// <param name="metadataExtractors">Collection of metadata extractors to apply to the given resoure.</param> /// <param name="resultHandler">Callback to notify the import results.</param> /// <param name="mediaAccessor">Convenience reference to the media accessor.</param> /// <returns><c>true</c>, if metadata could be extracted from the given <paramref name="mediaItemAccessor"/>, else /// <c>false</c>.</returns> protected bool ImportResource(IResourceAccessor mediaItemAccessor, Guid parentDirectoryId, ICollection <IMetadataExtractor> metadataExtractors, IImportResultHandler resultHandler, IMediaAccessor mediaAccessor) { const bool forceQuickMode = false; // Allow extractions with probably longer runtime. ResourcePath path = mediaItemAccessor.CanonicalLocalResourcePath; ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportStatus, path); IDictionary <Guid, MediaItemAspect> aspects = mediaAccessor.ExtractMetadata(mediaItemAccessor, metadataExtractors, forceQuickMode); if (aspects == null) { // No metadata could be extracted return(false); } resultHandler.UpdateMediaItem(parentDirectoryId, path, aspects.Values); resultHandler.DeleteUnderPath(path); return(true); }
private void OnFinished(Task previousTask) { // Do not notify about progress anymore for every disposed PendingImportResource _notifyProgress = false; if (_pendingImportResources.Count > 0) { // The ImportJob has finished, but we have PendingImportJobResources left that have not been disposed. // This should only happen when the ImportJob finishes in cancelled oder faulted state. When the ImportJob // ran to completion, the DataflowBlocks should have disposed all the PendingImportResources. if (!previousTask.IsCanceled && !previousTask.IsFaulted) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: The ImportJob ran to completion but there are {1} undisposed PendingImportResources left. Disposing them now...", this, _pendingImportResources.Count); } } if (previousTask.IsFaulted) { ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Error while processing", previousTask.Exception, this); } else if (previousTask.IsCanceled) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker.{0}: Canceled", this); } else { ServiceRegistration.Get <ILogger>().Info("ImporterWorker.{0}: Completed", this); } // When this ImportJob was cancelled it needs to be disposed manually. if (!previousTask.IsCanceled) { Dispose(); } ImporterWorkerMessaging.SendImportMessage(previousTask.IsCanceled ? ImporterWorkerMessaging.MessageType.ImportScheduleCanceled : ImporterWorkerMessaging.MessageType.ImportCompleted, _importJobInformation.BasePath); _parentImporterWorker.NotifyProgress(true); // If this ImportJob faulted or was cancelled we can't do anything but log it (which we do above). // Therefore the Completion Task of this ImportJobController always returns 'RunToCompletion' to // avoid exceptions being thrown when this Task is awaited. _importJobControllerCompletion.SetResult(null); }
public void ScheduleImport(ResourcePath path, IEnumerable <string> mediaCategories, bool includeSubDirectories) { ICollection <ImportJob> importJobs; lock (_syncObj) importJobs = new List <ImportJob>(_importJobs); if (importJobs.Any(checkJob => checkJob.JobType == ImportJobType.Import && checkJob.BasePath.IsSameOrParentOf(path))) { // Path is already being scheduled as Import job // => the new job is already included in an already existing job return; } CancelJobsForPath(path); ICollection <Guid> metadataExtractorIds = GetMetadataExtractorIdsForMediaCategories(mediaCategories); ImportJob job = new ImportJob(ImportJobType.Import, path, metadataExtractorIds, includeSubDirectories); EnqueueImportJob(job); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportScheduled, path, ImportJobType.Import); }
/// <summary> /// Executes the given <paramref name="importJob"/>. /// </summary> /// <remarks> /// This method automatically terminates if it encounters that this importer worker was suspended or that the /// given <paramref name="importJob"/> was cancelled. /// </remarks> /// <param name="importJob">Import job to be executed. The state variables of this parameter will be updated by /// this method.</param> protected void Process(ImportJob importJob) { ImportJobState state = importJob.State; if (state == ImportJobState.Finished || state == ImportJobState.Cancelled || state == ImportJobState.Erroneous) { return; } // Preparation IMediaAccessor mediaAccessor = ServiceRegistration.Get <IMediaAccessor>(); IImportResultHandler resultHandler; IMediaBrowsing mediaBrowsing; lock (_syncObj) { resultHandler = _importResultHandler; mediaBrowsing = _mediaBrowsingCallback; } if (mediaBrowsing == null || resultHandler == null) { // Can be the case if this importer worker was asynchronously suspended return; } try { try { ICollection <IMetadataExtractor> metadataExtractors = new List <IMetadataExtractor>(); foreach (Guid metadataExtractorId in importJob.MetadataExtractorIds) { IMetadataExtractor extractor; if (!mediaAccessor.LocalMetadataExtractors.TryGetValue(metadataExtractorId, out extractor)) { continue; } metadataExtractors.Add(extractor); } // Prepare import if (state == ImportJobState.Scheduled) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Starting import job '{0}'", importJob); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportStarted, importJob.BasePath); IResourceAccessor ra; if (!importJob.BasePath.TryCreateLocalResourceAccessor(out ra)) { throw new ArgumentException(string.Format("Unable to access resource path '{0}'", importJob.BasePath)); } using (ra) { IFileSystemResourceAccessor fsra = ra as IFileSystemResourceAccessor; if (fsra != null) { // Prepare complex import process importJob.PendingResources.Add(new PendingImportResource(Guid.Empty, (IFileSystemResourceAccessor)fsra.Clone())); importJob.State = ImportJobState.Active; } else { // Simple resource import ImportSingleFile(importJob, ra, metadataExtractors, mediaBrowsing, resultHandler, mediaAccessor); lock (importJob.SyncObj) if (importJob.State == ImportJobState.Active) { importJob.State = ImportJobState.Finished; } return; } } } else { ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Resuming import job '{0}' ({1} items pending)", importJob, importJob.PendingResources.Count); } // Actual import process while (importJob.HasPendingResources) { Thread.Sleep(0); CheckImportStillRunning(importJob.State); PendingImportResource pendingImportResource; lock (importJob.SyncObj) pendingImportResource = importJob.PendingResources.FirstOrDefault(); if (pendingImportResource.IsValid) { IFileSystemResourceAccessor fsra = pendingImportResource.ResourceAccessor; int numPending = importJob.PendingResources.Count; string moreResources = numPending > 1 ? string.Format(" ({0} more resources pending)", numPending) : string.Empty; ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Importing '{0}'{1}", fsra.ResourcePathName, moreResources); if (fsra.IsFile && fsra.Exists) { ImportResource(importJob, fsra, pendingImportResource.ParentDirectory, metadataExtractors, resultHandler, mediaAccessor); } else if (!fsra.IsFile) { CheckImportStillRunning(importJob.State); Guid?currentDirectoryId = ImportDirectory(importJob, pendingImportResource.ParentDirectory, fsra, metadataExtractors, mediaBrowsing, resultHandler, mediaAccessor); CheckImportStillRunning(importJob.State); if (currentDirectoryId.HasValue && importJob.IncludeSubDirectories) { // Add subdirectories in front of work queue lock (importJob.SyncObj) { ICollection <IFileSystemResourceAccessor> directories = FileSystemResourceNavigator.GetChildDirectories(fsra, false); if (directories != null) { foreach (IFileSystemResourceAccessor childDirectory in directories) { importJob.PendingResources.Insert(0, new PendingImportResource(currentDirectoryId.Value, childDirectory)); } } } } } else { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker: Cannot import resource '{0}': It's neither a file nor a directory", fsra.CanonicalLocalResourcePath.Serialize()); } } lock (importJob.SyncObj) importJob.PendingResources.Remove(pendingImportResource); pendingImportResource.Dispose(); } lock (importJob.SyncObj) if (importJob.State == ImportJobState.Active) { importJob.State = ImportJobState.Finished; } ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Finished import job '{0}'", importJob); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportCompleted, importJob.BasePath); } catch (Exception e) { CheckSuspended(e); // Throw ImportAbortException if suspended - will skip warning and tagging job as erroneous ServiceRegistration.Get <ILogger>().Warn("ImporterWorker: Problem processing '{0}'", e, importJob); ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportCompleted, importJob.BasePath); importJob.State = ImportJobState.Erroneous; } } catch (ImportSuspendedException) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Suspending import job '{0}' ({1} items pending - will be continued next time)", importJob, importJob.PendingResources.Count); } catch (ImportAbortException) { ServiceRegistration.Get <ILogger>().Info("ImporterWorker: Aborting import job '{0}' ({1} items pending)", importJob, importJob.PendingResources.Count); } }
/// <summary> /// Imports or refreshes the directory with the specified <paramref name="directoryAccessor"/>. Sub directories will not /// be processed in this method. /// </summary> /// <param name="importJob">The import job being processed.</param> /// <param name="parentDirectoryId">Media item id of the parent directory, if present, else <see cref="Guid.Empty"/>.</param> /// <param name="directoryAccessor">Resource accessor for the directory to import.</param> /// <param name="metadataExtractors">Metadata extractors to apply on the resources.</param> /// <param name="mediaBrowsing">Callback interface to the media library for the refresh import type.</param> /// <param name="resultHandler">Callback to notify the import result.</param> /// <param name="mediaAccessor">Convenience reference to the media accessor.</param> /// <returns>Id of the directory's media item or <c>null</c>, if the given <paramref name="directoryAccessor"/> /// was imported as a media item or if an error occured. If <c>null</c> is returned, the directory import should be /// considered to be finished.</returns> protected Guid?ImportDirectory(ImportJob importJob, Guid parentDirectoryId, IFileSystemResourceAccessor directoryAccessor, ICollection <IMetadataExtractor> metadataExtractors, IMediaBrowsing mediaBrowsing, IImportResultHandler resultHandler, IMediaAccessor mediaAccessor) { ResourcePath currentDirectoryPath = directoryAccessor.CanonicalLocalResourcePath; try { ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportStatus, currentDirectoryPath); if (ImportResource(importJob, directoryAccessor, parentDirectoryId, metadataExtractors, resultHandler, mediaAccessor)) { // The directory could be imported as a media item. // If the directory itself was identified as a normal media item, don't import its children. // Necessary for DVD directories, for example. return(null); } Guid directoryId = GetOrAddDirectory(importJob, directoryAccessor, parentDirectoryId, mediaBrowsing, resultHandler); IDictionary <string, MediaItem> path2Item = new Dictionary <string, MediaItem>(); if (importJob.JobType == ImportJobType.Refresh) { foreach (MediaItem mediaItem in mediaBrowsing.BrowseAsync(directoryId, IMPORTER_PROVIDER_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION, null, false).Result) { IList <MultipleMediaItemAspect> providerResourceAspects; if (MediaItemAspect.TryGetAspects(mediaItem.Aspects, ProviderResourceAspect.Metadata, out providerResourceAspects)) { path2Item[providerResourceAspects[0].GetAttributeValue <string>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)] = mediaItem; } } } CheckImportStillRunning(importJob.State); ICollection <IFileSystemResourceAccessor> files = FileSystemResourceNavigator.GetFiles(directoryAccessor, false); if (files != null) { foreach (IFileSystemResourceAccessor fileAccessor in files) { using (fileAccessor) { // Add & update files ResourcePath currentFilePath = fileAccessor.CanonicalLocalResourcePath; string serializedFilePath = currentFilePath.Serialize(); try { SingleMediaItemAspect importerAspect; MediaItem mediaItem; if (importJob.JobType == ImportJobType.Refresh && path2Item.TryGetValue(serializedFilePath, out mediaItem) && MediaItemAspect.TryGetAspect(mediaItem.Aspects, ImporterAspect.Metadata, out importerAspect) && importerAspect.GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE) > fileAccessor.LastChanged) { // We can skip this file; it was imported after the last change time of the item path2Item.Remove(serializedFilePath); continue; } if (ImportResource(importJob, fileAccessor, directoryId, metadataExtractors, resultHandler, mediaAccessor)) { path2Item.Remove(serializedFilePath); } } catch (Exception e) { CheckSuspended(e); // Throw ImportAbortException if suspended - will skip warning and tagging job as erroneous ServiceRegistration.Get <ILogger>().Warn("ImporterWorker: Problem while importing resource '{0}'", e, serializedFilePath); importJob.State = ImportJobState.Erroneous; } CheckImportStillRunning(importJob.State); } } } if (importJob.JobType == ImportJobType.Refresh) { // Remove remaining (= non-present) files foreach (string pathStr in path2Item.Keys) { ResourcePath path = ResourcePath.Deserialize(pathStr); try { IResourceAccessor ra; if (!path.TryCreateLocalResourceAccessor(out ra)) { throw new IllegalCallException("Unable to access resource path '{0}'", importJob.BasePath); } using (ra) { IFileSystemResourceAccessor fsra = ra as IFileSystemResourceAccessor; if (fsra == null || !fsra.IsFile) { // Don't touch directories because they will be imported in a different call of ImportDirectory continue; } } } catch (IllegalCallException) { // This happens if the resource doesn't exist any more - we also catch missing directories here } // Delete all remaining items resultHandler.DeleteMediaItemAsync(path); CheckImportStillRunning(importJob.State); } } return(directoryId); } catch (ImportSuspendedException) { throw; } catch (ImportAbortException) { throw; } catch (UnauthorizedAccessException e) { // If the access to the file or folder was denied, simply continue with the others ServiceRegistration.Get <ILogger>().Warn("ImporterWorker: Problem accessing resource '{0}', continueing with one", e, currentDirectoryPath); } catch (Exception e) { CheckSuspended(e); // Throw ImportAbortException if suspended - will skip warning and tagging job as erroneous ServiceRegistration.Get <ILogger>().Warn("ImporterWorker: Problem while importing directory '{0}'", e, currentDirectoryPath); importJob.State = ImportJobState.Erroneous; } return(null); }
private void SendProgressNotificationMessage() { ImporterWorkerMessaging.SendImportMessage(ImporterWorkerMessaging.MessageType.ImportProgress, _importJobControllers.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Progress)); }