/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Saves all MediaItems with their MIAs to the Database /// - In case of SingleResources in RefreshImports it also deletes all MediaItems below the current one in the database /// (necessary if in a previous import this MediaItem was saved to the Database as a directory instead of a SingleResource) /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { // ReSharper disable once PossibleInvalidOperationException await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, importResource.Aspects.Values); if (ImportJobInformation.JobType == ImportJobType.Refresh) { if (importResource.IsSingleResource) { await DeleteUnderPath(importResource.PendingResourcePath); } } importResource.IsValid = false; return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - It does nothing but drop the current <see cref="PendingImportResourceNewGen"/> if /// (a) this is a RefreshImport, /// (b) the DateOfLastImport is more current than the LastChanged date of the resource; /// the DateOfLastImport is only set for files because for directories we don't know to what filesystem date we /// should compare (the LastChanged date of the directory may be unchanged although fils below have changed), and /// (c) the DateOfLastImport is more current than the most recent creation date of all MIAs, wich are imported in this /// ImportJob. This ensures that if a new MIA is added to the system that should be imported for this ImportJob, the /// Import is actually performed although this is a RefreshImport and the MediaItems have not changed since the last Import. /// - For all other resources it calls all the aplicable MDEs /// - It then also drops the respective <see cref="PendingImportResourceNewGen"/> if nothing could be extracted; /// This is currently in particular the case for /// - directories that are not SingleResources (as we don't have a MDE that extracts metadata for such directories, yet /// - file types that non of the applicable MDEs can handle /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { importResource.Aspects = await ExtractMetadata(importResource.ResourceAccessor, importResource.ExistingAspects, _forceQuickMode).ConfigureAwait(false); if (importResource.Aspects == null) { importResource.Aspects = importResource.ExistingAspects; } if (importResource.Aspects == null) { importResource.IsValid = false; } return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Saves all MediaItems with their MIAs to the Database /// - In case of SingleResources in RefreshImports it also deletes all MediaItems below the current one in the database /// (necessary if in a previous import this MediaItem was saved to the Database as a directory instead of a SingleResource) /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { // ReSharper disable once PossibleInvalidOperationException if (importResource.MediaItemId.HasValue) { importResource.MediaItemId = await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, importResource.MediaItemId.Value, MediaItemAspect.GetAspects(importResource.Aspects), ImportJobInformation, true).ConfigureAwait(false); } else { importResource.MediaItemId = await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, MediaItemAspect.GetAspects(importResource.Aspects), ImportJobInformation, false).ConfigureAwait(false); } if (ImportJobInformation.JobType == ImportJobType.Refresh) { if (importResource.IsSingleResource && importResource.Aspects.ContainsKey(DirectoryAspect.ASPECT_ID)) { await DeleteUnderPath(importResource.PendingResourcePath).ConfigureAwait(false); } } return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.Aspects.Clear(); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Deletes no longer existing subdirectories from the MediaLibrary /// </summary> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/>representing the currently processed directory</param> /// <param name="subDirectories">Existing subdirectories of the currently processed directory</param> /// <returns></returns> private async Task DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(PendingImportResourceNewGen importResource, IEnumerable <IFileSystemResourceAccessor> subDirectories) { var mediaItem = await LoadLocalItem(importResource.PendingResourcePath, EMPTY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION).ConfigureAwait(false); // If the currently processed directory does not yet exist in the MediaLibrary, // there is no need to check for existing subdirectories in the MediaLibrary. if (mediaItem == null) { return; } var directoryId = mediaItem.MediaItemId; // Get the subdirectories stored in the MediaLibrary for the currently procesed directory // TODO: Rework this var subDirectoryResourcePathsInMediaLibrary = (await Browse(directoryId, PROVIDERRESOURCE_DIRECTORY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION).ConfigureAwait(false)).Select(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID][0].GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH))).ToList(); // If there are no subdirectories stored in the MediaLibrary, there is no need to delete anything if (!subDirectoryResourcePathsInMediaLibrary.Any()) { return; } // Find out which subdirectories are stored in the MediaLibrary that do not exist anymore // in the filesystem and delete them (including all subdirectories and subitems) var subDirectoryResourcePathsInFileSystem = subDirectories.Select(ra => ra.CanonicalLocalResourcePath); var noLongerExistingSubdirectoryResourcePaths = subDirectoryResourcePathsInMediaLibrary.Except(subDirectoryResourcePathsInFileSystem).ToList(); foreach (var noLongerExistingSubdirectoryResourcePath in noLongerExistingSubdirectoryResourcePaths) { await DeleteMediaItem(noLongerExistingSubdirectoryResourcePath).ConfigureAwait(false); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// Extracts relationships for the media item and adds them to the media library. /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { //Check if we can extract relations for this import resource if (await ValidateImportResource(importResource).ConfigureAwait(false)) { //Try to cache the resource as a matching item might get extracted later, e.g. the SeriesEpisodeExtractor //might extract a matching episode and we should avoid processing the item again. await CacheImportResource(importResource).ConfigureAwait(false); await ExtractRelationships(importResource.ResourceAccessor, importResource.MediaItemId.Value, importResource.Aspects).ConfigureAwait(false); } importResource.IsValid = false; importResource.Aspects?.Clear(); importResource.ExistingAspects?.Clear(); return(importResource); } catch (OperationCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Checks, in case of RefreshImports, whether a refresh of a given directory is necessary /// </summary> /// <param name="importResource">PendingImportResource of the directory to be checked</param> /// <returns></returns> private async Task <bool> IsRefreshNeeded(PendingImportResourceNewGen importResource) { var directoryPath = importResource.ResourceAccessor.CanonicalLocalResourcePath; var directoryItem = await LoadLocalItem(directoryPath, EMPTY_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION); if (directoryItem != null) { MediaItemAspect directoryAspect; if (!directoryItem.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect)) { // This is the case if the parentResourcePath was formerly imported as a single resource. // We cannot reuse it and it is necessary to delete this old MediaItem. await DeleteMediaItem(directoryPath); directoryItem = null; } else { // This directory is already correctly stored in the MediaLibrary. No need to store it again, // we just cache its ID for potential subdirectories to be stored during this refresh. _parentDirectoryIds[directoryPath] = directoryItem.MediaItemId; importResource.MediaItemId = directoryItem.MediaItemId; } } return(directoryItem == null); }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Checks whether the current resource is a SingleResource. /// - If this is not the case (i.e. there are subitems under this resource) /// - it posts all the subdirectories under this resource to this DataflowBlock /// - and in case of a RefreshImport, it deletes all subdirectories of the current resource in the MediaLibrary /// that do not exist anymore. /// - If there are no more resources to be processed after this resource, it completes this DataflowBlock /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource) { try { //ToDo: Replace this with a call to IsSingleResource once this method is implemented in the MetadataExtractors if (!importResource.ResourceAccessor.IsFile && await ExtractMetadata(importResource.ResourceAccessor, true) == null) { importResource.IsSingleResource = false; } if (!importResource.IsSingleResource && ImportJobInformation.IncludeSubDirectories) { var subDirectories = FileSystemResourceNavigator.GetChildDirectories(importResource.ResourceAccessor, false) ?? new HashSet <IFileSystemResourceAccessor>(); // This may throw an exception in case of cancellation and therefore needs to be done before // posting the subdirectories to the InputBufferBlock to avoid duplicate ImportResources when // reactivating this block after it has been deserialized from disk. if (ImportJobInformation.JobType == ImportJobType.Refresh) { await DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(importResource, subDirectories); } foreach (var subDirectory in subDirectories) { this.Post(new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, subDirectory, ToString(), ParentImportJobController)); } } return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } finally { var inputBlock = (TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>)InputBlock; if (inputBlock.InputCount == 0 && inputBlock.OutputCount == 0) { ParentImportJobController.FirstBlockHasFinished(); } } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - SingleResources are just passed to the next DataflowBlock /// - If it's not a SingleResource /// - it finds all the files in the current directory, /// - in case of a RefreshImport /// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem, /// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns> /// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/> /// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory /// </returns> private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessChanges(PendingImportResourceNewGen importResource) { var result = new HashSet <PendingImportResourceNewGen> { importResource }; try { if (ImportJobInformation.JobType == ImportJobType.Refresh) { // ReSharper disable once PossibleInvalidOperationException IEnumerable <MediaItem> mediaItems = await GetUpdatableMediaItems(PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, null); if (mediaItems != null) { foreach (MediaItem mi in mediaItems) { IList <MultipleMediaItemAspect> providerAspects = null; if (MediaItemAspect.TryGetAspects(mi.Aspects, ProviderResourceAspect.Metadata, out providerAspects)) { ResourcePath path = ResourcePath.Deserialize(providerAspects[0].GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)); Guid? directoryId = providerAspects[0].GetAttributeValue <Guid?>(ProviderResourceAspect.ATTR_PARENT_DIRECTORY_ID); IResourceAccessor ra; if (path.TryCreateLocalResourceAccessor(out ra) && ra is IFileSystemResourceAccessor) { IFileSystemResourceAccessor f = ra as IFileSystemResourceAccessor; string dirPath = ResourcePathHelper.GetDirectoryName(ra.Path); ResourcePath dirRa = ResourcePath.BuildBaseProviderPath(ra.ParentProvider.Metadata.ResourceProviderId, dirPath); PendingImportResourceNewGen pir = new PendingImportResourceNewGen(dirRa, f, ToString(), ParentImportJobController, directoryId, mi.MediaItemId); pir.DateOfLastImport = DateTime.MinValue; //Force update result.Add(pir); } } } } } return(result); } catch (TaskCanceledException) { return(result); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(result); } }
/// <summary> /// Caches the aspects of the specified <see cref="PendingImportResourceNewGen"/>. /// </summary> /// <param name="importResource">The <see cref="PendingImportResourceNewGen"/> containing the aspects to cache.</param> /// <returns>A Task that completes when the item has been cached.</returns> protected async Task <bool> CacheImportResource(PendingImportResourceNewGen importResource) { MediaItem item = new MediaItem(importResource.MediaItemId.Value, importResource.Aspects); bool result = false; await _cacheSync.WaitAsync().ConfigureAwait(false); try { foreach (IRelationshipRoleExtractor roleExtractor in GetLinkedRoleExtractors(importResource.Aspects)) { result |= _relationshipCache.TryAddItem(item, roleExtractor); } } finally { _cacheSync.Release(); } return(result); }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Sets the ParentDirectoryId of the current <see cref="PendingImportResourceNewGen"/> /// - If the current resource is not a SingleResource (which are saved by the <see cref="MediaItemSaveBlock"/>) /// - it saves the directory MediaItem to the MediaLibrary /// - for RereshImports it is only saved if it is not yet in the MediaLibrary. /// - If it has been saved, it sets the MediaItemId of the current <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource) { try { importResource.ParentDirectoryId = await GetParentDirectoryId(importResource.ParentDirectory); // Directories that are single resources (such as DVD directories) are not saved in this DataflowBlock // We just pass them to the next DataflowBlock. if (!importResource.IsSingleResource) { // We only save to the MediaLibrary if // (a) this is a FirstTimeImport (i.e. not a RefreshImport), or // (b) this is a RefreshImport and the respective directory MediaItem is not yet in the MediaLibrary if (ImportJobInformation.JobType == ImportJobType.Import || await IsRefreshNeeded(importResource)) { if (importResource.ParentDirectoryId == null) { // If we cannot determine the parent directory ID we have an error case and // cannot save this directory MediaItem importResource.IsValid = false; return(importResource); } Guid newId = await AddDirectory(importResource.ResourceAccessor, importResource.ParentDirectoryId.Value); importResource.MediaItemId = newId; _parentDirectoryIds[importResource.PendingResourcePath] = newId; } } return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - It does nothing but drop the current <see cref="PendingImportResourceNewGen"/> if /// (a) this is a RefreshImport, /// (b) the DateOfLastImport is more current than the LastChanged date of the resource; /// the DateOfLastImport is only set for files because for directories we don't know to what filesystem date we /// should compare (the LastChanged date of the directory may be unchanged although fils below have changed), and /// (c) the DateOfLastImport is more current than the most recent creation date of all MIAs, wich are imported in this /// ImportJob. This ensures that if a new MIA is added to the system that should be imported for this ImportJob, the /// Import is actually performed although this is a RefreshImport and the MediaItems have not changed since the last Import. /// - For all other resources it calls all the aplicable MDEs /// - It then also drops the respective <see cref="PendingImportResourceNewGen"/> if nothing could be extracted; /// This is currently in particular the case for /// - directories that are not SingleResources (as we don't have a MDE that extracts metadata for such directories, yet /// - file types that non of the applicable MDEs can handle /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { if (ImportJobInformation.JobType == ImportJobType.Refresh) { // Do not import again, if the file or directory wasn't changed since the last import // and there were no new relevant MIAs added since then. // ToDo: We should only omit MDEs that get their data from the file or directory itself. All others should be called anyway. if (importResource.DateOfLastImport > importResource.ResourceAccessor.LastChanged && importResource.DateOfLastImport > await _mostRecentMiaCreationDate.Value) { importResource.IsValid = false; return(importResource); } } importResource.Aspects = await ExtractMetadata(importResource.ResourceAccessor, _forceQuickMode); if (importResource.Aspects == null) { importResource.IsValid = false; } return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Determines whether the specified <paramref name="importResource"/> has a valid media item id, and in the case /// of an import resource restored from disk attempts to load its aspects from the media library. /// </summary> /// <param name="importResource">The <see cref="PendingImportResourceNewGen"/> to validate.</param> /// <returns>True if the import resource has a valid media item id and aspects.</returns> private async Task <bool> ValidateImportResource(PendingImportResourceNewGen importResource) { //It's possible for the media item id to be empty, particularly in the case of subtitles where no mergable media item //was found in the database. Don't process the item if that is the case if (!importResource.MediaItemId.HasValue || importResource.MediaItemId.Value == Guid.Empty || !_processedMediaItemIds.TryAdd(importResource.MediaItemId.Value, DUMMY_DICTIONARY_VALUE)) { return(false); } //Aspects will only be null if this import resource was restored from disk if (importResource.Aspects != null) { return(true); } //No aspects, this resource was restored from disk, try and restore the aspects from the media library. //Throttle the number of concurrent database connections to avoid a spike during startup. await _loadItemThrottle.WaitAsync(_ct).ConfigureAwait(false); try { //Try and restore the aspects MediaItem loadItem = await LoadLocalItem(importResource.MediaItemId.Value, null, await GetAllManagedMediaItemAspectTypes().ConfigureAwait(false)).ConfigureAwait(false); if (loadItem == null) { return(false); } importResource.Aspects = loadItem.Aspects; return(true); } finally { _loadItemThrottle.Release(); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - SingleResources are just passed to the next DataflowBlock /// - If it's not a SingleResource /// - it finds all the files in the current directory, /// - in case of a RefreshImport /// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem, /// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns> /// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/> /// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory /// </returns> private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessDirectory(PendingImportResourceNewGen importResource) { var result = new HashSet <PendingImportResourceNewGen> { importResource }; try { ICollection <IFileSystemResourceAccessor> files; ICollection <IFileSystemResourceAccessor> stubFiles = new HashSet <IFileSystemResourceAccessor>(); IDictionary <ResourcePath, DateTime> path2LastImportDate = new Dictionary <ResourcePath, DateTime>(); IDictionary <ResourcePath, Guid> path2MediaItem = new Dictionary <ResourcePath, Guid>(); IEnumerable <MediaItem> mediaItems = null; if (importResource.IsSingleResource) { files = new HashSet <IFileSystemResourceAccessor> { importResource.ResourceAccessor }; MediaItem mi = await LoadLocalItem(importResource.PendingResourcePath, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, null).ConfigureAwait(false); if (mi != null) { mediaItems = new List <MediaItem>(new MediaItem[] { mi }); importResource.MediaItemId = mi.MediaItemId; } } else { files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false, false) ?? new HashSet <IFileSystemResourceAccessor>(); SingleMediaItemAspect directoryAspect; // ReSharper disable once PossibleInvalidOperationException // TODO: Rework this mediaItems = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION).ConfigureAwait(false)) .Where(mi => !MediaItemAspect.TryGetAspect(mi.Aspects, DirectoryAspect.Metadata, out directoryAspect)); } if (mediaItems != null) { foreach (MediaItem mi in mediaItems) { //Check metadata and files: // 1. Last import date is lower than file change date => Refresh needed // 2. Media item ID is empty => Reimport/import needed // 3. Media item is dirty => Reimport/import needed IList <MultipleMediaItemAspect> providerAspects = null; if (MediaItemAspect.TryGetAspects(mi.Aspects, ProviderResourceAspect.Metadata, out providerAspects)) { foreach (var pra in providerAspects) { bool isStub = pra.GetAttributeValue <int>(ProviderResourceAspect.ATTR_TYPE) == ProviderResourceAspect.TYPE_STUB; ResourcePath path = ResourcePath.Deserialize(pra.GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)); if (!path2LastImportDate.ContainsKey(path) && importResource.PendingResourcePath.IsSameOrParentOf(path)) { //If last refresh is equal to added date, it has never been through the refresh cycle, so set low last change date //All media items must be added because the paths are later used to delete no longer existing media items var lastImportDate = mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE); if (mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <bool>(ImporterAspect.ATTR_DIRTY)) //If it is dirty, refresh is needed { path2LastImportDate.Add(path, DateTime.MinValue); } else { path2LastImportDate.Add(path, lastImportDate); } } if (!importResource.IsSingleResource && !isStub && !path2MediaItem.ContainsKey(path)) { path2MediaItem.Add(path, mi.MediaItemId); } // Stub items need their media item id because the do no have a unique path // So add them now as long as the needed info is known if (isStub) { IFileSystemResourceAccessor file = null; try { IResourceAccessor ra; if (path.TryCreateLocalResourceAccessor(out ra)) { file = ra as IFileSystemResourceAccessor; } } catch { } // Only add it if it still exists if (files.Where(f => file != null && f.CanonicalLocalResourcePath == file.CanonicalLocalResourcePath).Any()) { stubFiles.Add(file); DateTime dateTime; PendingImportResourceNewGen pir = new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, file, ToString(), ParentImportJobController, importResource.MediaItemId, mi.MediaItemId, true); pir.ExistingAspects = mi.Aspects; if (ImportJobInformation.JobType == ImportJobType.Refresh) { if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime)) { pir.DateOfLastImport = dateTime; } } result.Add(pir); } } } } } await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys).ConfigureAwait(false); } //Add new stub items foreach (var file in files.Where(f => !path2LastImportDate.Keys.Contains(f.CanonicalLocalResourcePath))) { if (await IsStubResource(file).ConfigureAwait(false)) { stubFiles.Add(file); DateTime dateTime; var stubAspects = await ExtractStubItems(file).ConfigureAwait(false); if (stubAspects != null) { foreach (var aspects in stubAspects) { PendingImportResourceNewGen pir = new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, file, ToString(), ParentImportJobController, importResource.MediaItemId, null, true); pir.ExistingAspects = aspects; if (ImportJobInformation.JobType == ImportJobType.Refresh) { if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime)) { pir.DateOfLastImport = dateTime; } } result.Add(pir); } } } } //Remove stub files from files collection so they don't get added again foreach (IFileSystemResourceAccessor file in stubFiles) { IFileSystemResourceAccessor stub = files.Where(f => f.CanonicalLocalResourcePath == file.CanonicalLocalResourcePath).FirstOrDefault(); if (stub != null) { files.Remove(stub); } } //Add importers for files if any if (!importResource.IsSingleResource) { if (ImportJobInformation.JobType == ImportJobType.Import) { //Only import new files so only add non existing paths result.UnionWith(files.Where(f => !path2LastImportDate.Keys.Contains(f.CanonicalLocalResourcePath)). Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId))); } else { result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId, path2MediaItem.ContainsKey(f.CanonicalLocalResourcePath) ? path2MediaItem[f.CanonicalLocalResourcePath] : (Guid?)null))); } // We set the directory resource as invalid because directories have no need for further processing importResource.IsValid = false; } // If this is a RefreshImport and we found files of the current directory in the MediaLibrary, // store the DateOfLastImport in the PendingImportResource if (ImportJobInformation.JobType == ImportJobType.Refresh) { DateTime dateTime; if (path2LastImportDate != null) { foreach (var pir in result) { if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime)) { pir.DateOfLastImport = dateTime; } } } } return(result); } catch (TaskCanceledException) { return(result); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(result); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Saves all MediaItems with their MIAs to the Database /// - In case of SingleResources in RefreshImports it also deletes all MediaItems below the current one in the database /// (necessary if in a previous import this MediaItem was saved to the Database as a directory instead of a SingleResource) /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task<PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { // ReSharper disable once PossibleInvalidOperationException await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, importResource.Aspects.Values); if (ImportJobInformation.JobType == ImportJobType.Refresh) if(importResource.IsSingleResource) await DeleteUnderPath(importResource.PendingResourcePath); importResource.IsValid = false; return importResource; } catch (TaskCanceledException) { return importResource; } catch (Exception ex) { ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return importResource; } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Loads MediaItems with their MIAs from the Database for RefreshImports if last update was more than a day ago. /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { if (ImportJobInformation.JobType == ImportJobType.Refresh) { // Do not import again, if the file or directory wasn't changed since the last import // and there were no new relevant MIAs added since then. // ToDo: We should only omit MDEs that get their data from the file or directory itself. All others should be called anyway. if (importResource.DateOfLastImport > importResource.ResourceAccessor.LastChanged && importResource.DateOfLastImport > await _mostRecentMiaCreationDate.Value.ConfigureAwait(false) && importResource.MediaItemId.HasValue) { importResource.IsValid = false; return(importResource); } //if (importResource.DateOfLastImport == DateTime.MinValue) // ServiceRegistration.Get<ILogger>().Info("File {0} force import", importResource.PendingResourcePath); //else if (importResource.DateOfLastImport < importResource.ResourceAccessor.LastChanged) // ServiceRegistration.Get<ILogger>().Info("File {0} changed after import {1} < {2}", importResource.PendingResourcePath, importResource.DateOfLastImport, importResource.ResourceAccessor.LastChanged); //else if (importResource.DateOfLastImport < await _mostRecentMiaCreationDate.Value) // ServiceRegistration.Get<ILogger>().Info("File {0} changed before aspect change", importResource.PendingResourcePath); //else if (!importResource.MediaItemId.HasValue) // ServiceRegistration.Get<ILogger>().Info("File {0} needs import", importResource.PendingResourcePath); ICollection <Guid> aspects = await GetAllManagedMediaItemAspectTypes().ConfigureAwait(false); List <Guid> optionalAspects = new List <Guid>(aspects); if (optionalAspects.Contains(ProviderResourceAspect.ASPECT_ID)) { optionalAspects.Remove(ProviderResourceAspect.ASPECT_ID); } List <Guid> requiredAspects = new List <Guid>(); requiredAspects.Add(ProviderResourceAspect.ASPECT_ID); // ReSharper disable once PossibleInvalidOperationException MediaItem mediaItem = null; if (importResource.MediaItemId.HasValue) { mediaItem = await LoadLocalItem(importResource.MediaItemId.Value, requiredAspects.AsEnumerable(), optionalAspects.AsEnumerable()).ConfigureAwait(false); } else if (!importResource.IsStubResource) { mediaItem = await LoadLocalItem(importResource.PendingResourcePath, requiredAspects.AsEnumerable(), optionalAspects.AsEnumerable()).ConfigureAwait(false); } if (mediaItem != null) { SingleMediaItemAspect directoryAspect; if (MediaItemAspect.TryGetAspect(mediaItem.Aspects, DirectoryAspect.Metadata, out directoryAspect)) { importResource.IsValid = false; return(importResource); } importResource.ExistingAspects = mediaItem.Aspects; importResource.MediaItemId = mediaItem.MediaItemId; } } return(importResource); } catch (TaskCanceledException) { return(importResource); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(importResource); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Checks whether the current resource is a SingleResource. /// - If this is not the case (i.e. there are subitems under this resource) /// - it posts all the subdirectories under this resource to this DataflowBlock /// - and in case of a RefreshImport, it deletes all subdirectories of the current resource in the MediaLibrary /// that do not exist anymore. /// - If there are no more resources to be processed after this resource, it completes this DataflowBlock /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task<PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource) { try { //ToDo: Replace this with a call to IsSingleResource once this method is implemented in the MetadataExtractors if (!importResource.ResourceAccessor.IsFile && await ExtractMetadata(importResource.ResourceAccessor, true) == null) importResource.IsSingleResource = false; if (!importResource.IsSingleResource && ImportJobInformation.IncludeSubDirectories) { var subDirectories = FileSystemResourceNavigator.GetChildDirectories(importResource.ResourceAccessor, false) ?? new HashSet<IFileSystemResourceAccessor>(); // This may throw an exception in case of cancellation and therefore needs to be done before // posting the subdirectories to the InputBufferBlock to avoid duplicate ImportResources when // reactivating this block after it has been deserialized from disk. if (ImportJobInformation.JobType == ImportJobType.Refresh) await DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(importResource, subDirectories); foreach (var subDirectory in subDirectories) this.Post(new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, subDirectory, ToString(), ParentImportJobController)); } return importResource; } catch (TaskCanceledException) { return importResource; } catch (Exception ex) { ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return importResource; } finally { var inputBlock = (TransformBlock<PendingImportResourceNewGen, PendingImportResourceNewGen>)InputBlock; if (inputBlock.InputCount == 0 && inputBlock.OutputCount == 0) ParentImportJobController.FirstBlockHasFinished(); } }
/// <summary> /// Checks, in case of RefreshImports, whether a refresh of a given directory is necessary /// </summary> /// <param name="importResource">PendingImportResource of the directory to be checked</param> /// <returns></returns> private async Task<bool> IsRefreshNeeded(PendingImportResourceNewGen importResource) { var directoryPath = importResource.ResourceAccessor.CanonicalLocalResourcePath; var directoryItem = await LoadLocalItem(directoryPath, EMPTY_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION); if (directoryItem != null) { MediaItemAspect directoryAspect; if (!directoryItem.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect)) { // This is the case if the parentResourcePath was formerly imported as a single resource. // We cannot reuse it and it is necessary to delete this old MediaItem. await DeleteMediaItem(directoryPath); directoryItem = null; } else { // This directory is already correctly stored in the MediaLibrary. No need to store it again, // we just cache its ID for potential subdirectories to be stored during this refresh. _parentDirectoryIds[directoryPath] = directoryItem.MediaItemId; importResource.MediaItemId = directoryItem.MediaItemId; } } return (directoryItem == null); }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - It does nothing but drop the current <see cref="PendingImportResourceNewGen"/> if /// (a) this is a RefreshImport, /// (b) the DateOfLastImport is more current than the LastChanged date of the resource; /// the DateOfLastImport is only set for files because for directories we don't know to what filesystem date we /// should compare (the LastChanged date of the directory may be unchanged although fils below have changed), and /// (c) the DateOfLastImport is more current than the most recent creation date of all MIAs, wich are imported in this /// ImportJob. This ensures that if a new MIA is added to the system that should be imported for this ImportJob, the /// Import is actually performed although this is a RefreshImport and the MediaItems have not changed since the last Import. /// - For all other resources it calls all the aplicable MDEs /// - It then also drops the respective <see cref="PendingImportResourceNewGen"/> if nothing could be extracted; /// This is currently in particular the case for /// - directories that are not SingleResources (as we don't have a MDE that extracts metadata for such directories, yet /// - file types that non of the applicable MDEs can handle /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task<PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource) { try { if (ImportJobInformation.JobType == ImportJobType.Refresh) { // Do not import again, if the file or directory wasn't changed since the last import // and there were no new relevant MIAs added since then. // ToDo: We should only omit MDEs that get their data from the file or directory itself. All others should be called anyway. if (importResource.DateOfLastImport > importResource.ResourceAccessor.LastChanged && importResource.DateOfLastImport > await _mostRecentMiaCreationDate.Value) { importResource.IsValid = false; return importResource; } } importResource.Aspects = await ExtractMetadata(importResource.ResourceAccessor, _forceQuickMode); if (importResource.Aspects == null) importResource.IsValid = false; return importResource; } catch (TaskCanceledException) { return importResource; } catch (Exception ex) { ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return importResource; } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - SingleResources are just passed to the next DataflowBlock /// - If it's not a SingleResource /// - it finds all the files in the current directory, /// - in case of a RefreshImport /// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem, /// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns> /// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/> /// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory /// </returns> private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessDirectory(PendingImportResourceNewGen importResource) { var result = new HashSet <PendingImportResourceNewGen> { importResource }; try { if (importResource.IsSingleResource) { return(result); } // FileSystemResourceNavigator.GetFiles also returns files that were identified as virtual directories by // FileSystemResourceNavigator.GetChildDirectories (such as zip-files). // ToDo: Clarify if this is a bug var files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false) ?? new HashSet <IFileSystemResourceAccessor>(); IDictionary <ResourcePath, DateTime> path2LastImportDate = null; if (ImportJobInformation.JobType == ImportJobType.Refresh) { MediaItemAspect directoryAspect; // ReSharper disable once PossibleInvalidOperationException path2LastImportDate = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION)) .Where(mi => !mi.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect)) .ToDictionary(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID].GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)), mi => mi[ImporterAspect.ASPECT_ID].GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE)); await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys); } result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId))); // If this is a RefreshImport and we found files of the current directory in the MediaLibrary, // store the DateOfLastImport in the PendingImportResource DateTime dateTime; if (path2LastImportDate != null) { foreach (var pir in result) { if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime)) { pir.DateOfLastImport = dateTime; } } } return(result); } catch (TaskCanceledException) { return(result); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(result); } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - SingleResources are just passed to the next DataflowBlock /// - If it's not a SingleResource /// - it finds all the files in the current directory, /// - in case of a RefreshImport /// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem, /// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns> /// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/> /// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory /// </returns> private async Task<IEnumerable<PendingImportResourceNewGen>> ProcessDirectory(PendingImportResourceNewGen importResource) { var result = new HashSet<PendingImportResourceNewGen> { importResource }; try { if (importResource.IsSingleResource) return result; // FileSystemResourceNavigator.GetFiles also returns files that were identified as virtual directories by // FileSystemResourceNavigator.GetChildDirectories (such as zip-files). // ToDo: Clarify if this is a bug var files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false) ?? new HashSet<IFileSystemResourceAccessor>(); IDictionary<ResourcePath, DateTime> path2LastImportDate = null; if (ImportJobInformation.JobType == ImportJobType.Refresh) { MediaItemAspect directoryAspect; // ReSharper disable once PossibleInvalidOperationException path2LastImportDate = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION)) .Where(mi => !mi.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect)) .ToDictionary(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID].GetAttributeValue<String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)), mi => mi[ImporterAspect.ASPECT_ID].GetAttributeValue<DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE)); await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys); } result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId))); // If this is a RefreshImport and we found files of the current directory in the MediaLibrary, // store the DateOfLastImport in the PendingImportResource DateTime dateTime; if (path2LastImportDate != null) foreach (var pir in result) if(path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime)) pir.DateOfLastImport = dateTime; return result; } catch (TaskCanceledException) { return result; } catch (Exception ex) { ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return result; } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - Sets the ParentDirectoryId of the current <see cref="PendingImportResourceNewGen"/> /// - If the current resource is not a SingleResource (which are saved by the <see cref="MediaItemSaveBlock"/>) /// - it saves the directory MediaItem to the MediaLibrary /// - for RereshImports it is only saved if it is not yet in the MediaLibrary. /// - If it has been saved, it sets the MediaItemId of the current <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns> private async Task<PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource) { try { importResource.ParentDirectoryId = await GetParentDirectoryId(importResource.ParentDirectory); // Directories that are single resources (such as DVD directories) are not saved in this DataflowBlock // We just pass them to the next DataflowBlock. if (!importResource.IsSingleResource) { // We only save to the MediaLibrary if // (a) this is a FirstTimeImport (i.e. not a RefreshImport), or // (b) this is a RefreshImport and the respective directory MediaItem is not yet in the MediaLibrary if (ImportJobInformation.JobType == ImportJobType.Import || await IsRefreshNeeded(importResource)) { if (importResource.ParentDirectoryId == null) { // If we cannot determine the parent directory ID we have an error case and // cannot save this directory MediaItem importResource.IsValid = false; return importResource; } Guid newId = await AddDirectory(importResource.ResourceAccessor, importResource.ParentDirectoryId.Value); importResource.MediaItemId = newId; _parentDirectoryIds[importResource.PendingResourcePath] = newId; } } return importResource; } catch (TaskCanceledException) { return importResource; } catch (Exception ex) { ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return importResource; } }
/// <summary> /// Main process method for the InnerBlock /// </summary> /// <remarks> /// - SingleResources are just passed to the next DataflowBlock /// - If it's not a SingleResource /// - it finds all the files in the current directory, /// - in case of a RefreshImport /// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem, /// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/> /// </remarks> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param> /// <returns> /// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/> /// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory /// </returns> private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessDirectory(PendingImportResourceNewGen importResource) { var result = new HashSet <PendingImportResourceNewGen> { importResource }; try { if (importResource.IsSingleResource) { return(result); } // FileSystemResourceNavigator.GetFiles also returns files that were identified as virtual directories by // FileSystemResourceNavigator.GetChildDirectories (such as zip-files). // ToDo: Clarify if this is a bug var files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false) ?? new HashSet <IFileSystemResourceAccessor>(); IDictionary <ResourcePath, DateTime> path2LastImportDate = null; IDictionary <ResourcePath, Guid> path2MediaItem = null; SingleMediaItemAspect directoryAspect; // ReSharper disable once PossibleInvalidOperationException // TODO: Rework this IEnumerable <MediaItem> mediaItems = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION)) .Where(mi => !MediaItemAspect.TryGetAspect(mi.Aspects, DirectoryAspect.Metadata, out directoryAspect)); if (mediaItems != null) { path2LastImportDate = new Dictionary <ResourcePath, DateTime>(); path2MediaItem = new Dictionary <ResourcePath, Guid>(); foreach (MediaItem mi in mediaItems) { //Check metadata and files: // 1. Last import date is lower than file change date => Refresh needed // 2. Media item ID is empty => Reimport/import needed // 3. Media item is dirty => Reimport/import needed IList <MultipleMediaItemAspect> providerAspects = null; if (MediaItemAspect.TryGetAspects(mi.Aspects, ProviderResourceAspect.Metadata, out providerAspects)) { foreach (var pra in providerAspects) { ResourcePath path = ResourcePath.Deserialize(pra.GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)); if (!path2LastImportDate.ContainsKey(path)) { //If last refresh is equal to added date, it has never been through the refresh cycle, so set low last change date //All media items must be added because the paths are later used to delete no longer existing media items var lastImportDate = mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE); var addedDate = mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <DateTime>(ImporterAspect.ATTR_DATEADDED); if ((lastImportDate - addedDate).TotalSeconds <= 5) { path2LastImportDate.Add(path, DateTime.MinValue); } else { path2LastImportDate.Add(path, lastImportDate); } } if (!path2MediaItem.ContainsKey(path)) { //If it is dirty, leave media item ID empty if (mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <bool>(ImporterAspect.ATTR_DIRTY)) { continue; } path2MediaItem.Add(path, mi.MediaItemId); } } } } await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys); } if (ImportJobInformation.JobType == ImportJobType.Import) { //Only import new files so only add non existing paths result.UnionWith(files.Where(f => !path2LastImportDate.Keys.Contains(f.CanonicalLocalResourcePath)). Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId))); } else { result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId, path2MediaItem.ContainsKey(f.CanonicalLocalResourcePath) ? path2MediaItem[f.CanonicalLocalResourcePath] : (Guid?)null))); // If this is a RefreshImport and we found files of the current directory in the MediaLibrary, // store the DateOfLastImport in the PendingImportResource DateTime dateTime; if (path2LastImportDate != null) { foreach (var pir in result) { if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime)) { pir.DateOfLastImport = dateTime; } } } } return(result); } catch (TaskCanceledException) { return(result); } catch (Exception ex) { ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource); importResource.IsValid = false; return(result); } }
/// <summary> /// Deletes no longer existing subdirectories from the MediaLibrary /// </summary> /// <param name="importResource"><see cref="PendingImportResourceNewGen"/>representing the currently processed directory</param> /// <param name="subDirectories">Existing subdirectories of the currently processed directory</param> /// <returns></returns> private async Task DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(PendingImportResourceNewGen importResource, IEnumerable<IFileSystemResourceAccessor> subDirectories) { var mediaItem = await LoadLocalItem(importResource.PendingResourcePath, EMPTY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION); // If the currently processed directory does not yet exist in the MediaLibrary, // there is no need to check for existing subdirectories in the MediaLibrary. if (mediaItem == null) return; var directoryId = mediaItem.MediaItemId; // Get the subdirectories stored in the MediaLibrary for the currently procesed directory var subDirectoryResourcePathsInMediaLibrary = (await Browse(directoryId, PROVIDERRESOURCE_DIRECTORY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION)).Select(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID].GetAttributeValue<String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH))).ToList(); // If there are no subdirectories stored in the MediaLibrary, there is no need to delete anything if (!subDirectoryResourcePathsInMediaLibrary.Any()) return; // Find out which subdirectories are stored in the MediaLibrary that do not exist anymore // in the filesystem and delete them (including all subdirectories and subitems) var subDirectoryResourcePathsInFileSystem = subDirectories.Select(ra => ra.CanonicalLocalResourcePath); var noLongerExistingSubdirectoryResourcePaths = subDirectoryResourcePathsInMediaLibrary.Except(subDirectoryResourcePathsInFileSystem).ToList(); foreach (var noLongerExistingSubdirectoryResourcePath in noLongerExistingSubdirectoryResourcePaths) await DeleteMediaItem(noLongerExistingSubdirectoryResourcePath); }