/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Saves all MediaItems with their MIAs to the Database
/// - In case of SingleResources in RefreshImports it also deletes all MediaItems below the current one in the database
/// (necessary if in a previous import this MediaItem was saved to the Database as a directory instead of a SingleResource)
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
// ReSharper disable once PossibleInvalidOperationException
await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, importResource.Aspects.Values);
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
if (importResource.IsSingleResource)
{
await DeleteUnderPath(importResource.PendingResourcePath);
}
}
importResource.IsValid = false;
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - It does nothing but drop the current <see cref="PendingImportResourceNewGen"/> if
/// (a) this is a RefreshImport,
/// (b) the DateOfLastImport is more current than the LastChanged date of the resource;
/// the DateOfLastImport is only set for files because for directories we don't know to what filesystem date we
/// should compare (the LastChanged date of the directory may be unchanged although fils below have changed), and
/// (c) the DateOfLastImport is more current than the most recent creation date of all MIAs, wich are imported in this
/// ImportJob. This ensures that if a new MIA is added to the system that should be imported for this ImportJob, the
/// Import is actually performed although this is a RefreshImport and the MediaItems have not changed since the last Import.
/// - For all other resources it calls all the aplicable MDEs
/// - It then also drops the respective <see cref="PendingImportResourceNewGen"/> if nothing could be extracted;
/// This is currently in particular the case for
/// - directories that are not SingleResources (as we don't have a MDE that extracts metadata for such directories, yet
/// - file types that non of the applicable MDEs can handle
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
importResource.Aspects = await ExtractMetadata(importResource.ResourceAccessor, importResource.ExistingAspects, _forceQuickMode).ConfigureAwait(false);
if (importResource.Aspects == null)
{
importResource.Aspects = importResource.ExistingAspects;
}
if (importResource.Aspects == null)
{
importResource.IsValid = false;
}
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Saves all MediaItems with their MIAs to the Database
/// - In case of SingleResources in RefreshImports it also deletes all MediaItems below the current one in the database
/// (necessary if in a previous import this MediaItem was saved to the Database as a directory instead of a SingleResource)
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
// ReSharper disable once PossibleInvalidOperationException
if (importResource.MediaItemId.HasValue)
{
importResource.MediaItemId = await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, importResource.MediaItemId.Value, MediaItemAspect.GetAspects(importResource.Aspects), ImportJobInformation, true).ConfigureAwait(false);
}
else
{
importResource.MediaItemId = await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, MediaItemAspect.GetAspects(importResource.Aspects), ImportJobInformation, false).ConfigureAwait(false);
}
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
if (importResource.IsSingleResource && importResource.Aspects.ContainsKey(DirectoryAspect.ASPECT_ID))
{
await DeleteUnderPath(importResource.PendingResourcePath).ConfigureAwait(false);
}
}
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.Aspects.Clear();
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Deletes no longer existing subdirectories from the MediaLibrary
/// </summary>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/>representing the currently processed directory</param>
/// <param name="subDirectories">Existing subdirectories of the currently processed directory</param>
/// <returns></returns>
private async Task DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(PendingImportResourceNewGen importResource, IEnumerable <IFileSystemResourceAccessor> subDirectories)
{
var mediaItem = await LoadLocalItem(importResource.PendingResourcePath, EMPTY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION).ConfigureAwait(false);
// If the currently processed directory does not yet exist in the MediaLibrary,
// there is no need to check for existing subdirectories in the MediaLibrary.
if (mediaItem == null)
{
return;
}
var directoryId = mediaItem.MediaItemId;
// Get the subdirectories stored in the MediaLibrary for the currently procesed directory
// TODO: Rework this
var subDirectoryResourcePathsInMediaLibrary = (await Browse(directoryId, PROVIDERRESOURCE_DIRECTORY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION).ConfigureAwait(false)).Select(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID][0].GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH))).ToList();
// If there are no subdirectories stored in the MediaLibrary, there is no need to delete anything
if (!subDirectoryResourcePathsInMediaLibrary.Any())
{
return;
}
// Find out which subdirectories are stored in the MediaLibrary that do not exist anymore
// in the filesystem and delete them (including all subdirectories and subitems)
var subDirectoryResourcePathsInFileSystem = subDirectories.Select(ra => ra.CanonicalLocalResourcePath);
var noLongerExistingSubdirectoryResourcePaths = subDirectoryResourcePathsInMediaLibrary.Except(subDirectoryResourcePathsInFileSystem).ToList();
foreach (var noLongerExistingSubdirectoryResourcePath in noLongerExistingSubdirectoryResourcePaths)
{
await DeleteMediaItem(noLongerExistingSubdirectoryResourcePath).ConfigureAwait(false);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// Extracts relationships for the media item and adds them to the media library.
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
//Check if we can extract relations for this import resource
if (await ValidateImportResource(importResource).ConfigureAwait(false))
{
//Try to cache the resource as a matching item might get extracted later, e.g. the SeriesEpisodeExtractor
//might extract a matching episode and we should avoid processing the item again.
await CacheImportResource(importResource).ConfigureAwait(false);
await ExtractRelationships(importResource.ResourceAccessor, importResource.MediaItemId.Value, importResource.Aspects).ConfigureAwait(false);
}
importResource.IsValid = false;
importResource.Aspects?.Clear();
importResource.ExistingAspects?.Clear();
return(importResource);
}
catch (OperationCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Checks, in case of RefreshImports, whether a refresh of a given directory is necessary
/// </summary>
/// <param name="importResource">PendingImportResource of the directory to be checked</param>
/// <returns></returns>
private async Task <bool> IsRefreshNeeded(PendingImportResourceNewGen importResource)
{
var directoryPath = importResource.ResourceAccessor.CanonicalLocalResourcePath;
var directoryItem = await LoadLocalItem(directoryPath, EMPTY_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION);
if (directoryItem != null)
{
MediaItemAspect directoryAspect;
if (!directoryItem.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect))
{
// This is the case if the parentResourcePath was formerly imported as a single resource.
// We cannot reuse it and it is necessary to delete this old MediaItem.
await DeleteMediaItem(directoryPath);
directoryItem = null;
}
else
{
// This directory is already correctly stored in the MediaLibrary. No need to store it again,
// we just cache its ID for potential subdirectories to be stored during this refresh.
_parentDirectoryIds[directoryPath] = directoryItem.MediaItemId;
importResource.MediaItemId = directoryItem.MediaItemId;
}
}
return(directoryItem == null);
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Checks whether the current resource is a SingleResource.
/// - If this is not the case (i.e. there are subitems under this resource)
/// - it posts all the subdirectories under this resource to this DataflowBlock
/// - and in case of a RefreshImport, it deletes all subdirectories of the current resource in the MediaLibrary
/// that do not exist anymore.
/// - If there are no more resources to be processed after this resource, it completes this DataflowBlock
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource)
{
try
{
//ToDo: Replace this with a call to IsSingleResource once this method is implemented in the MetadataExtractors
if (!importResource.ResourceAccessor.IsFile && await ExtractMetadata(importResource.ResourceAccessor, true) == null)
{
importResource.IsSingleResource = false;
}
if (!importResource.IsSingleResource && ImportJobInformation.IncludeSubDirectories)
{
var subDirectories = FileSystemResourceNavigator.GetChildDirectories(importResource.ResourceAccessor, false) ?? new HashSet <IFileSystemResourceAccessor>();
// This may throw an exception in case of cancellation and therefore needs to be done before
// posting the subdirectories to the InputBufferBlock to avoid duplicate ImportResources when
// reactivating this block after it has been deserialized from disk.
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
await DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(importResource, subDirectories);
}
foreach (var subDirectory in subDirectories)
{
this.Post(new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, subDirectory, ToString(), ParentImportJobController));
}
}
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
finally
{
var inputBlock = (TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>)InputBlock;
if (inputBlock.InputCount == 0 && inputBlock.OutputCount == 0)
{
ParentImportJobController.FirstBlockHasFinished();
}
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - SingleResources are just passed to the next DataflowBlock
/// - If it's not a SingleResource
/// - it finds all the files in the current directory,
/// - in case of a RefreshImport
/// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem,
/// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns>
/// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/>
/// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory
/// </returns>
private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessChanges(PendingImportResourceNewGen importResource)
{
var result = new HashSet <PendingImportResourceNewGen> {
importResource
};
try
{
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
// ReSharper disable once PossibleInvalidOperationException
IEnumerable <MediaItem> mediaItems = await GetUpdatableMediaItems(PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, null);
if (mediaItems != null)
{
foreach (MediaItem mi in mediaItems)
{
IList <MultipleMediaItemAspect> providerAspects = null;
if (MediaItemAspect.TryGetAspects(mi.Aspects, ProviderResourceAspect.Metadata, out providerAspects))
{
ResourcePath path = ResourcePath.Deserialize(providerAspects[0].GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH));
Guid? directoryId = providerAspects[0].GetAttributeValue <Guid?>(ProviderResourceAspect.ATTR_PARENT_DIRECTORY_ID);
IResourceAccessor ra;
if (path.TryCreateLocalResourceAccessor(out ra) && ra is IFileSystemResourceAccessor)
{
IFileSystemResourceAccessor f = ra as IFileSystemResourceAccessor;
string dirPath = ResourcePathHelper.GetDirectoryName(ra.Path);
ResourcePath dirRa = ResourcePath.BuildBaseProviderPath(ra.ParentProvider.Metadata.ResourceProviderId, dirPath);
PendingImportResourceNewGen pir = new PendingImportResourceNewGen(dirRa, f, ToString(), ParentImportJobController, directoryId, mi.MediaItemId);
pir.DateOfLastImport = DateTime.MinValue; //Force update
result.Add(pir);
}
}
}
}
}
return(result);
}
catch (TaskCanceledException)
{
return(result);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(result);
}
}
/// <summary>
/// Caches the aspects of the specified <see cref="PendingImportResourceNewGen"/>.
/// </summary>
/// <param name="importResource">The <see cref="PendingImportResourceNewGen"/> containing the aspects to cache.</param>
/// <returns>A Task that completes when the item has been cached.</returns>
protected async Task <bool> CacheImportResource(PendingImportResourceNewGen importResource)
{
MediaItem item = new MediaItem(importResource.MediaItemId.Value, importResource.Aspects);
bool result = false;
await _cacheSync.WaitAsync().ConfigureAwait(false);
try
{
foreach (IRelationshipRoleExtractor roleExtractor in GetLinkedRoleExtractors(importResource.Aspects))
{
result |= _relationshipCache.TryAddItem(item, roleExtractor);
}
}
finally
{
_cacheSync.Release();
}
return(result);
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Sets the ParentDirectoryId of the current <see cref="PendingImportResourceNewGen"/>
/// - If the current resource is not a SingleResource (which are saved by the <see cref="MediaItemSaveBlock"/>)
/// - it saves the directory MediaItem to the MediaLibrary
/// - for RereshImports it is only saved if it is not yet in the MediaLibrary.
/// - If it has been saved, it sets the MediaItemId of the current <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource)
{
try
{
importResource.ParentDirectoryId = await GetParentDirectoryId(importResource.ParentDirectory);
// Directories that are single resources (such as DVD directories) are not saved in this DataflowBlock
// We just pass them to the next DataflowBlock.
if (!importResource.IsSingleResource)
{
// We only save to the MediaLibrary if
// (a) this is a FirstTimeImport (i.e. not a RefreshImport), or
// (b) this is a RefreshImport and the respective directory MediaItem is not yet in the MediaLibrary
if (ImportJobInformation.JobType == ImportJobType.Import || await IsRefreshNeeded(importResource))
{
if (importResource.ParentDirectoryId == null)
{
// If we cannot determine the parent directory ID we have an error case and
// cannot save this directory MediaItem
importResource.IsValid = false;
return(importResource);
}
Guid newId = await AddDirectory(importResource.ResourceAccessor, importResource.ParentDirectoryId.Value);
importResource.MediaItemId = newId;
_parentDirectoryIds[importResource.PendingResourcePath] = newId;
}
}
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - It does nothing but drop the current <see cref="PendingImportResourceNewGen"/> if
/// (a) this is a RefreshImport,
/// (b) the DateOfLastImport is more current than the LastChanged date of the resource;
/// the DateOfLastImport is only set for files because for directories we don't know to what filesystem date we
/// should compare (the LastChanged date of the directory may be unchanged although fils below have changed), and
/// (c) the DateOfLastImport is more current than the most recent creation date of all MIAs, wich are imported in this
/// ImportJob. This ensures that if a new MIA is added to the system that should be imported for this ImportJob, the
/// Import is actually performed although this is a RefreshImport and the MediaItems have not changed since the last Import.
/// - For all other resources it calls all the aplicable MDEs
/// - It then also drops the respective <see cref="PendingImportResourceNewGen"/> if nothing could be extracted;
/// This is currently in particular the case for
/// - directories that are not SingleResources (as we don't have a MDE that extracts metadata for such directories, yet
/// - file types that non of the applicable MDEs can handle
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
// Do not import again, if the file or directory wasn't changed since the last import
// and there were no new relevant MIAs added since then.
// ToDo: We should only omit MDEs that get their data from the file or directory itself. All others should be called anyway.
if (importResource.DateOfLastImport > importResource.ResourceAccessor.LastChanged &&
importResource.DateOfLastImport > await _mostRecentMiaCreationDate.Value)
{
importResource.IsValid = false;
return(importResource);
}
}
importResource.Aspects = await ExtractMetadata(importResource.ResourceAccessor, _forceQuickMode);
if (importResource.Aspects == null)
{
importResource.IsValid = false;
}
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Determines whether the specified <paramref name="importResource"/> has a valid media item id, and in the case
/// of an import resource restored from disk attempts to load its aspects from the media library.
/// </summary>
/// <param name="importResource">The <see cref="PendingImportResourceNewGen"/> to validate.</param>
/// <returns>True if the import resource has a valid media item id and aspects.</returns>
private async Task <bool> ValidateImportResource(PendingImportResourceNewGen importResource)
{
//It's possible for the media item id to be empty, particularly in the case of subtitles where no mergable media item
//was found in the database. Don't process the item if that is the case
if (!importResource.MediaItemId.HasValue || importResource.MediaItemId.Value == Guid.Empty ||
!_processedMediaItemIds.TryAdd(importResource.MediaItemId.Value, DUMMY_DICTIONARY_VALUE))
{
return(false);
}
//Aspects will only be null if this import resource was restored from disk
if (importResource.Aspects != null)
{
return(true);
}
//No aspects, this resource was restored from disk, try and restore the aspects from the media library.
//Throttle the number of concurrent database connections to avoid a spike during startup.
await _loadItemThrottle.WaitAsync(_ct).ConfigureAwait(false);
try
{
//Try and restore the aspects
MediaItem loadItem = await LoadLocalItem(importResource.MediaItemId.Value, null, await GetAllManagedMediaItemAspectTypes().ConfigureAwait(false)).ConfigureAwait(false);
if (loadItem == null)
{
return(false);
}
importResource.Aspects = loadItem.Aspects;
return(true);
}
finally
{
_loadItemThrottle.Release();
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - SingleResources are just passed to the next DataflowBlock
/// - If it's not a SingleResource
/// - it finds all the files in the current directory,
/// - in case of a RefreshImport
/// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem,
/// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns>
/// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/>
/// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory
/// </returns>
private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessDirectory(PendingImportResourceNewGen importResource)
{
var result = new HashSet <PendingImportResourceNewGen> {
importResource
};
try
{
ICollection <IFileSystemResourceAccessor> files;
ICollection <IFileSystemResourceAccessor> stubFiles = new HashSet <IFileSystemResourceAccessor>();
IDictionary <ResourcePath, DateTime> path2LastImportDate = new Dictionary <ResourcePath, DateTime>();
IDictionary <ResourcePath, Guid> path2MediaItem = new Dictionary <ResourcePath, Guid>();
IEnumerable <MediaItem> mediaItems = null;
if (importResource.IsSingleResource)
{
files = new HashSet <IFileSystemResourceAccessor> {
importResource.ResourceAccessor
};
MediaItem mi = await LoadLocalItem(importResource.PendingResourcePath, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, null).ConfigureAwait(false);
if (mi != null)
{
mediaItems = new List <MediaItem>(new MediaItem[] { mi });
importResource.MediaItemId = mi.MediaItemId;
}
}
else
{
files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false, false) ?? new HashSet <IFileSystemResourceAccessor>();
SingleMediaItemAspect directoryAspect;
// ReSharper disable once PossibleInvalidOperationException
// TODO: Rework this
mediaItems = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION).ConfigureAwait(false))
.Where(mi => !MediaItemAspect.TryGetAspect(mi.Aspects, DirectoryAspect.Metadata, out directoryAspect));
}
if (mediaItems != null)
{
foreach (MediaItem mi in mediaItems)
{
//Check metadata and files:
// 1. Last import date is lower than file change date => Refresh needed
// 2. Media item ID is empty => Reimport/import needed
// 3. Media item is dirty => Reimport/import needed
IList <MultipleMediaItemAspect> providerAspects = null;
if (MediaItemAspect.TryGetAspects(mi.Aspects, ProviderResourceAspect.Metadata, out providerAspects))
{
foreach (var pra in providerAspects)
{
bool isStub = pra.GetAttributeValue <int>(ProviderResourceAspect.ATTR_TYPE) == ProviderResourceAspect.TYPE_STUB;
ResourcePath path = ResourcePath.Deserialize(pra.GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH));
if (!path2LastImportDate.ContainsKey(path) && importResource.PendingResourcePath.IsSameOrParentOf(path))
{
//If last refresh is equal to added date, it has never been through the refresh cycle, so set low last change date
//All media items must be added because the paths are later used to delete no longer existing media items
var lastImportDate = mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE);
if (mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <bool>(ImporterAspect.ATTR_DIRTY)) //If it is dirty, refresh is needed
{
path2LastImportDate.Add(path, DateTime.MinValue);
}
else
{
path2LastImportDate.Add(path, lastImportDate);
}
}
if (!importResource.IsSingleResource && !isStub && !path2MediaItem.ContainsKey(path))
{
path2MediaItem.Add(path, mi.MediaItemId);
}
// Stub items need their media item id because the do no have a unique path
// So add them now as long as the needed info is known
if (isStub)
{
IFileSystemResourceAccessor file = null;
try
{
IResourceAccessor ra;
if (path.TryCreateLocalResourceAccessor(out ra))
{
file = ra as IFileSystemResourceAccessor;
}
}
catch { }
// Only add it if it still exists
if (files.Where(f => file != null && f.CanonicalLocalResourcePath == file.CanonicalLocalResourcePath).Any())
{
stubFiles.Add(file);
DateTime dateTime;
PendingImportResourceNewGen pir = new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, file, ToString(),
ParentImportJobController, importResource.MediaItemId, mi.MediaItemId, true);
pir.ExistingAspects = mi.Aspects;
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime))
{
pir.DateOfLastImport = dateTime;
}
}
result.Add(pir);
}
}
}
}
}
await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys).ConfigureAwait(false);
}
//Add new stub items
foreach (var file in files.Where(f => !path2LastImportDate.Keys.Contains(f.CanonicalLocalResourcePath)))
{
if (await IsStubResource(file).ConfigureAwait(false))
{
stubFiles.Add(file);
DateTime dateTime;
var stubAspects = await ExtractStubItems(file).ConfigureAwait(false);
if (stubAspects != null)
{
foreach (var aspects in stubAspects)
{
PendingImportResourceNewGen pir = new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, file, ToString(),
ParentImportJobController, importResource.MediaItemId, null, true);
pir.ExistingAspects = aspects;
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime))
{
pir.DateOfLastImport = dateTime;
}
}
result.Add(pir);
}
}
}
}
//Remove stub files from files collection so they don't get added again
foreach (IFileSystemResourceAccessor file in stubFiles)
{
IFileSystemResourceAccessor stub = files.Where(f => f.CanonicalLocalResourcePath == file.CanonicalLocalResourcePath).FirstOrDefault();
if (stub != null)
{
files.Remove(stub);
}
}
//Add importers for files if any
if (!importResource.IsSingleResource)
{
if (ImportJobInformation.JobType == ImportJobType.Import)
{
//Only import new files so only add non existing paths
result.UnionWith(files.Where(f => !path2LastImportDate.Keys.Contains(f.CanonicalLocalResourcePath)).
Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId)));
}
else
{
result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId,
path2MediaItem.ContainsKey(f.CanonicalLocalResourcePath) ? path2MediaItem[f.CanonicalLocalResourcePath] : (Guid?)null)));
}
// We set the directory resource as invalid because directories have no need for further processing
importResource.IsValid = false;
}
// If this is a RefreshImport and we found files of the current directory in the MediaLibrary,
// store the DateOfLastImport in the PendingImportResource
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
DateTime dateTime;
if (path2LastImportDate != null)
{
foreach (var pir in result)
{
if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime))
{
pir.DateOfLastImport = dateTime;
}
}
}
}
return(result);
}
catch (TaskCanceledException)
{
return(result);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(result);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Saves all MediaItems with their MIAs to the Database
/// - In case of SingleResources in RefreshImports it also deletes all MediaItems below the current one in the database
/// (necessary if in a previous import this MediaItem was saved to the Database as a directory instead of a SingleResource)
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task<PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
// ReSharper disable once PossibleInvalidOperationException
await UpdateMediaItem(importResource.ParentDirectoryId.Value, importResource.PendingResourcePath, importResource.Aspects.Values);
if (ImportJobInformation.JobType == ImportJobType.Refresh)
if(importResource.IsSingleResource)
await DeleteUnderPath(importResource.PendingResourcePath);
importResource.IsValid = false;
return importResource;
}
catch (TaskCanceledException)
{
return importResource;
}
catch (Exception ex)
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return importResource;
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Loads MediaItems with their MIAs from the Database for RefreshImports if last update was more than a day ago.
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task <PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
// Do not import again, if the file or directory wasn't changed since the last import
// and there were no new relevant MIAs added since then.
// ToDo: We should only omit MDEs that get their data from the file or directory itself. All others should be called anyway.
if (importResource.DateOfLastImport > importResource.ResourceAccessor.LastChanged &&
importResource.DateOfLastImport > await _mostRecentMiaCreationDate.Value.ConfigureAwait(false) &&
importResource.MediaItemId.HasValue)
{
importResource.IsValid = false;
return(importResource);
}
//if (importResource.DateOfLastImport == DateTime.MinValue)
// ServiceRegistration.Get<ILogger>().Info("File {0} force import", importResource.PendingResourcePath);
//else if (importResource.DateOfLastImport < importResource.ResourceAccessor.LastChanged)
// ServiceRegistration.Get<ILogger>().Info("File {0} changed after import {1} < {2}", importResource.PendingResourcePath, importResource.DateOfLastImport, importResource.ResourceAccessor.LastChanged);
//else if (importResource.DateOfLastImport < await _mostRecentMiaCreationDate.Value)
// ServiceRegistration.Get<ILogger>().Info("File {0} changed before aspect change", importResource.PendingResourcePath);
//else if (!importResource.MediaItemId.HasValue)
// ServiceRegistration.Get<ILogger>().Info("File {0} needs import", importResource.PendingResourcePath);
ICollection <Guid> aspects = await GetAllManagedMediaItemAspectTypes().ConfigureAwait(false);
List <Guid> optionalAspects = new List <Guid>(aspects);
if (optionalAspects.Contains(ProviderResourceAspect.ASPECT_ID))
{
optionalAspects.Remove(ProviderResourceAspect.ASPECT_ID);
}
List <Guid> requiredAspects = new List <Guid>();
requiredAspects.Add(ProviderResourceAspect.ASPECT_ID);
// ReSharper disable once PossibleInvalidOperationException
MediaItem mediaItem = null;
if (importResource.MediaItemId.HasValue)
{
mediaItem = await LoadLocalItem(importResource.MediaItemId.Value, requiredAspects.AsEnumerable(), optionalAspects.AsEnumerable()).ConfigureAwait(false);
}
else if (!importResource.IsStubResource)
{
mediaItem = await LoadLocalItem(importResource.PendingResourcePath, requiredAspects.AsEnumerable(), optionalAspects.AsEnumerable()).ConfigureAwait(false);
}
if (mediaItem != null)
{
SingleMediaItemAspect directoryAspect;
if (MediaItemAspect.TryGetAspect(mediaItem.Aspects, DirectoryAspect.Metadata, out directoryAspect))
{
importResource.IsValid = false;
return(importResource);
}
importResource.ExistingAspects = mediaItem.Aspects;
importResource.MediaItemId = mediaItem.MediaItemId;
}
}
return(importResource);
}
catch (TaskCanceledException)
{
return(importResource);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(importResource);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Checks whether the current resource is a SingleResource.
/// - If this is not the case (i.e. there are subitems under this resource)
/// - it posts all the subdirectories under this resource to this DataflowBlock
/// - and in case of a RefreshImport, it deletes all subdirectories of the current resource in the MediaLibrary
/// that do not exist anymore.
/// - If there are no more resources to be processed after this resource, it completes this DataflowBlock
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task<PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource)
{
try
{
//ToDo: Replace this with a call to IsSingleResource once this method is implemented in the MetadataExtractors
if (!importResource.ResourceAccessor.IsFile && await ExtractMetadata(importResource.ResourceAccessor, true) == null)
importResource.IsSingleResource = false;
if (!importResource.IsSingleResource && ImportJobInformation.IncludeSubDirectories)
{
var subDirectories = FileSystemResourceNavigator.GetChildDirectories(importResource.ResourceAccessor, false) ?? new HashSet<IFileSystemResourceAccessor>();
// This may throw an exception in case of cancellation and therefore needs to be done before
// posting the subdirectories to the InputBufferBlock to avoid duplicate ImportResources when
// reactivating this block after it has been deserialized from disk.
if (ImportJobInformation.JobType == ImportJobType.Refresh)
await DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(importResource, subDirectories);
foreach (var subDirectory in subDirectories)
this.Post(new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, subDirectory, ToString(), ParentImportJobController));
}
return importResource;
}
catch (TaskCanceledException)
{
return importResource;
}
catch (Exception ex)
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return importResource;
}
finally
{
var inputBlock = (TransformBlock<PendingImportResourceNewGen, PendingImportResourceNewGen>)InputBlock;
if (inputBlock.InputCount == 0 && inputBlock.OutputCount == 0)
ParentImportJobController.FirstBlockHasFinished();
}
}
/// <summary>
/// Checks, in case of RefreshImports, whether a refresh of a given directory is necessary
/// </summary>
/// <param name="importResource">PendingImportResource of the directory to be checked</param>
/// <returns></returns>
private async Task<bool> IsRefreshNeeded(PendingImportResourceNewGen importResource)
{
var directoryPath = importResource.ResourceAccessor.CanonicalLocalResourcePath;
var directoryItem = await LoadLocalItem(directoryPath, EMPTY_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION);
if (directoryItem != null)
{
MediaItemAspect directoryAspect;
if (!directoryItem.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect))
{
// This is the case if the parentResourcePath was formerly imported as a single resource.
// We cannot reuse it and it is necessary to delete this old MediaItem.
await DeleteMediaItem(directoryPath);
directoryItem = null;
}
else
{
// This directory is already correctly stored in the MediaLibrary. No need to store it again,
// we just cache its ID for potential subdirectories to be stored during this refresh.
_parentDirectoryIds[directoryPath] = directoryItem.MediaItemId;
importResource.MediaItemId = directoryItem.MediaItemId;
}
}
return (directoryItem == null);
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - It does nothing but drop the current <see cref="PendingImportResourceNewGen"/> if
/// (a) this is a RefreshImport,
/// (b) the DateOfLastImport is more current than the LastChanged date of the resource;
/// the DateOfLastImport is only set for files because for directories we don't know to what filesystem date we
/// should compare (the LastChanged date of the directory may be unchanged although fils below have changed), and
/// (c) the DateOfLastImport is more current than the most recent creation date of all MIAs, wich are imported in this
/// ImportJob. This ensures that if a new MIA is added to the system that should be imported for this ImportJob, the
/// Import is actually performed although this is a RefreshImport and the MediaItems have not changed since the last Import.
/// - For all other resources it calls all the aplicable MDEs
/// - It then also drops the respective <see cref="PendingImportResourceNewGen"/> if nothing could be extracted;
/// This is currently in particular the case for
/// - directories that are not SingleResources (as we don't have a MDE that extracts metadata for such directories, yet
/// - file types that non of the applicable MDEs can handle
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task<PendingImportResourceNewGen> ProcessMediaItem(PendingImportResourceNewGen importResource)
{
try
{
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
// Do not import again, if the file or directory wasn't changed since the last import
// and there were no new relevant MIAs added since then.
// ToDo: We should only omit MDEs that get their data from the file or directory itself. All others should be called anyway.
if (importResource.DateOfLastImport > importResource.ResourceAccessor.LastChanged &&
importResource.DateOfLastImport > await _mostRecentMiaCreationDate.Value)
{
importResource.IsValid = false;
return importResource;
}
}
importResource.Aspects = await ExtractMetadata(importResource.ResourceAccessor, _forceQuickMode);
if (importResource.Aspects == null)
importResource.IsValid = false;
return importResource;
}
catch (TaskCanceledException)
{
return importResource;
}
catch (Exception ex)
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return importResource;
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - SingleResources are just passed to the next DataflowBlock
/// - If it's not a SingleResource
/// - it finds all the files in the current directory,
/// - in case of a RefreshImport
/// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem,
/// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns>
/// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/>
/// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory
/// </returns>
private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessDirectory(PendingImportResourceNewGen importResource)
{
var result = new HashSet <PendingImportResourceNewGen> {
importResource
};
try
{
if (importResource.IsSingleResource)
{
return(result);
}
// FileSystemResourceNavigator.GetFiles also returns files that were identified as virtual directories by
// FileSystemResourceNavigator.GetChildDirectories (such as zip-files).
// ToDo: Clarify if this is a bug
var files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false) ?? new HashSet <IFileSystemResourceAccessor>();
IDictionary <ResourcePath, DateTime> path2LastImportDate = null;
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
MediaItemAspect directoryAspect;
// ReSharper disable once PossibleInvalidOperationException
path2LastImportDate = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION))
.Where(mi => !mi.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect))
.ToDictionary(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID].GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)), mi => mi[ImporterAspect.ASPECT_ID].GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE));
await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys);
}
result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId)));
// If this is a RefreshImport and we found files of the current directory in the MediaLibrary,
// store the DateOfLastImport in the PendingImportResource
DateTime dateTime;
if (path2LastImportDate != null)
{
foreach (var pir in result)
{
if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime))
{
pir.DateOfLastImport = dateTime;
}
}
}
return(result);
}
catch (TaskCanceledException)
{
return(result);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(result);
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - SingleResources are just passed to the next DataflowBlock
/// - If it's not a SingleResource
/// - it finds all the files in the current directory,
/// - in case of a RefreshImport
/// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem,
/// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns>
/// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/>
/// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory
/// </returns>
private async Task<IEnumerable<PendingImportResourceNewGen>> ProcessDirectory(PendingImportResourceNewGen importResource)
{
var result = new HashSet<PendingImportResourceNewGen> { importResource };
try
{
if (importResource.IsSingleResource)
return result;
// FileSystemResourceNavigator.GetFiles also returns files that were identified as virtual directories by
// FileSystemResourceNavigator.GetChildDirectories (such as zip-files).
// ToDo: Clarify if this is a bug
var files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false) ?? new HashSet<IFileSystemResourceAccessor>();
IDictionary<ResourcePath, DateTime> path2LastImportDate = null;
if (ImportJobInformation.JobType == ImportJobType.Refresh)
{
MediaItemAspect directoryAspect;
// ReSharper disable once PossibleInvalidOperationException
path2LastImportDate = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION))
.Where(mi => !mi.Aspects.TryGetValue(DirectoryAspect.ASPECT_ID, out directoryAspect))
.ToDictionary(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID].GetAttributeValue<String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH)), mi => mi[ImporterAspect.ASPECT_ID].GetAttributeValue<DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE));
await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys);
}
result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId)));
// If this is a RefreshImport and we found files of the current directory in the MediaLibrary,
// store the DateOfLastImport in the PendingImportResource
DateTime dateTime;
if (path2LastImportDate != null)
foreach (var pir in result)
if(path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime))
pir.DateOfLastImport = dateTime;
return result;
}
catch (TaskCanceledException)
{
return result;
}
catch (Exception ex)
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return result;
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - Sets the ParentDirectoryId of the current <see cref="PendingImportResourceNewGen"/>
/// - If the current resource is not a SingleResource (which are saved by the <see cref="MediaItemSaveBlock"/>)
/// - it saves the directory MediaItem to the MediaLibrary
/// - for RereshImports it is only saved if it is not yet in the MediaLibrary.
/// - If it has been saved, it sets the MediaItemId of the current <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns><see cref="PendingImportResourceNewGen"/> after processing</returns>
private async Task<PendingImportResourceNewGen> ProcessDirectory(PendingImportResourceNewGen importResource)
{
try
{
importResource.ParentDirectoryId = await GetParentDirectoryId(importResource.ParentDirectory);
// Directories that are single resources (such as DVD directories) are not saved in this DataflowBlock
// We just pass them to the next DataflowBlock.
if (!importResource.IsSingleResource)
{
// We only save to the MediaLibrary if
// (a) this is a FirstTimeImport (i.e. not a RefreshImport), or
// (b) this is a RefreshImport and the respective directory MediaItem is not yet in the MediaLibrary
if (ImportJobInformation.JobType == ImportJobType.Import || await IsRefreshNeeded(importResource))
{
if (importResource.ParentDirectoryId == null)
{
// If we cannot determine the parent directory ID we have an error case and
// cannot save this directory MediaItem
importResource.IsValid = false;
return importResource;
}
Guid newId = await AddDirectory(importResource.ResourceAccessor, importResource.ParentDirectoryId.Value);
importResource.MediaItemId = newId;
_parentDirectoryIds[importResource.PendingResourcePath] = newId;
}
}
return importResource;
}
catch (TaskCanceledException)
{
return importResource;
}
catch (Exception ex)
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return importResource;
}
}
/// <summary>
/// Main process method for the InnerBlock
/// </summary>
/// <remarks>
/// - SingleResources are just passed to the next DataflowBlock
/// - If it's not a SingleResource
/// - it finds all the files in the current directory,
/// - in case of a RefreshImport
/// - it deletes all the files in the MediaLibrary that do not exist anymore in the filesystem,
/// - it stores the DateOfLastImport of all the files in the <see cref="PendingImportResourceNewGen"/>
/// </remarks>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/> to be processed</param>
/// <returns>
/// a HashSet of <see cref="PendingImportResourceNewGen"/>s containing the current <see cref="PendingImportResource"/>
/// after processing as well as <see cref="PendingImportResourceNewGen"/>s for all files in the current directory
/// </returns>
private async Task <IEnumerable <PendingImportResourceNewGen> > ProcessDirectory(PendingImportResourceNewGen importResource)
{
var result = new HashSet <PendingImportResourceNewGen> {
importResource
};
try
{
if (importResource.IsSingleResource)
{
return(result);
}
// FileSystemResourceNavigator.GetFiles also returns files that were identified as virtual directories by
// FileSystemResourceNavigator.GetChildDirectories (such as zip-files).
// ToDo: Clarify if this is a bug
var files = FileSystemResourceNavigator.GetFiles(importResource.ResourceAccessor, false) ?? new HashSet <IFileSystemResourceAccessor>();
IDictionary <ResourcePath, DateTime> path2LastImportDate = null;
IDictionary <ResourcePath, Guid> path2MediaItem = null;
SingleMediaItemAspect directoryAspect;
// ReSharper disable once PossibleInvalidOperationException
// TODO: Rework this
IEnumerable <MediaItem> mediaItems = (await Browse(importResource.MediaItemId.Value, PROVIDERRESOURCE_IMPORTER_MIA_ID_ENUMERATION, DIRECTORY_MIA_ID_ENUMERATION))
.Where(mi => !MediaItemAspect.TryGetAspect(mi.Aspects, DirectoryAspect.Metadata, out directoryAspect));
if (mediaItems != null)
{
path2LastImportDate = new Dictionary <ResourcePath, DateTime>();
path2MediaItem = new Dictionary <ResourcePath, Guid>();
foreach (MediaItem mi in mediaItems)
{
//Check metadata and files:
// 1. Last import date is lower than file change date => Refresh needed
// 2. Media item ID is empty => Reimport/import needed
// 3. Media item is dirty => Reimport/import needed
IList <MultipleMediaItemAspect> providerAspects = null;
if (MediaItemAspect.TryGetAspects(mi.Aspects, ProviderResourceAspect.Metadata, out providerAspects))
{
foreach (var pra in providerAspects)
{
ResourcePath path = ResourcePath.Deserialize(pra.GetAttributeValue <String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH));
if (!path2LastImportDate.ContainsKey(path))
{
//If last refresh is equal to added date, it has never been through the refresh cycle, so set low last change date
//All media items must be added because the paths are later used to delete no longer existing media items
var lastImportDate = mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <DateTime>(ImporterAspect.ATTR_LAST_IMPORT_DATE);
var addedDate = mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <DateTime>(ImporterAspect.ATTR_DATEADDED);
if ((lastImportDate - addedDate).TotalSeconds <= 5)
{
path2LastImportDate.Add(path, DateTime.MinValue);
}
else
{
path2LastImportDate.Add(path, lastImportDate);
}
}
if (!path2MediaItem.ContainsKey(path))
{
//If it is dirty, leave media item ID empty
if (mi.Aspects[ImporterAspect.ASPECT_ID][0].GetAttributeValue <bool>(ImporterAspect.ATTR_DIRTY))
{
continue;
}
path2MediaItem.Add(path, mi.MediaItemId);
}
}
}
}
await DeleteNoLongerExistingFilesFromMediaLibrary(files, path2LastImportDate.Keys);
}
if (ImportJobInformation.JobType == ImportJobType.Import)
{
//Only import new files so only add non existing paths
result.UnionWith(files.Where(f => !path2LastImportDate.Keys.Contains(f.CanonicalLocalResourcePath)).
Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId)));
}
else
{
result.UnionWith(files.Select(f => new PendingImportResourceNewGen(importResource.ResourceAccessor.CanonicalLocalResourcePath, f, ToString(), ParentImportJobController, importResource.MediaItemId,
path2MediaItem.ContainsKey(f.CanonicalLocalResourcePath) ? path2MediaItem[f.CanonicalLocalResourcePath] : (Guid?)null)));
// If this is a RefreshImport and we found files of the current directory in the MediaLibrary,
// store the DateOfLastImport in the PendingImportResource
DateTime dateTime;
if (path2LastImportDate != null)
{
foreach (var pir in result)
{
if (path2LastImportDate.TryGetValue(pir.PendingResourcePath, out dateTime))
{
pir.DateOfLastImport = dateTime;
}
}
}
}
return(result);
}
catch (TaskCanceledException)
{
return(result);
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}.{1}: Error while processing {2}", ex, ParentImportJobController, ToString(), importResource);
importResource.IsValid = false;
return(result);
}
}
/// <summary>
/// Deletes no longer existing subdirectories from the MediaLibrary
/// </summary>
/// <param name="importResource"><see cref="PendingImportResourceNewGen"/>representing the currently processed directory</param>
/// <param name="subDirectories">Existing subdirectories of the currently processed directory</param>
/// <returns></returns>
private async Task DeleteNoLongerExistingSubdirectoriesFromMediaLibrary(PendingImportResourceNewGen importResource, IEnumerable<IFileSystemResourceAccessor> subDirectories)
{
var mediaItem = await LoadLocalItem(importResource.PendingResourcePath, EMPTY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION);
// If the currently processed directory does not yet exist in the MediaLibrary,
// there is no need to check for existing subdirectories in the MediaLibrary.
if (mediaItem == null)
return;
var directoryId = mediaItem.MediaItemId;
// Get the subdirectories stored in the MediaLibrary for the currently procesed directory
var subDirectoryResourcePathsInMediaLibrary = (await Browse(directoryId, PROVIDERRESOURCE_DIRECTORY_MIA_ID_ENUMERATION, EMPTY_MIA_ID_ENUMERATION)).Select(mi => ResourcePath.Deserialize(mi[ProviderResourceAspect.ASPECT_ID].GetAttributeValue<String>(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH))).ToList();
// If there are no subdirectories stored in the MediaLibrary, there is no need to delete anything
if (!subDirectoryResourcePathsInMediaLibrary.Any())
return;
// Find out which subdirectories are stored in the MediaLibrary that do not exist anymore
// in the filesystem and delete them (including all subdirectories and subitems)
var subDirectoryResourcePathsInFileSystem = subDirectories.Select(ra => ra.CanonicalLocalResourcePath);
var noLongerExistingSubdirectoryResourcePaths = subDirectoryResourcePathsInMediaLibrary.Except(subDirectoryResourcePathsInFileSystem).ToList();
foreach (var noLongerExistingSubdirectoryResourcePath in noLongerExistingSubdirectoryResourcePaths)
await DeleteMediaItem(noLongerExistingSubdirectoryResourcePath);
}
