public void UnregisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
{
// We need to make sure that only the PendingImportResource object that is registered can unregister itself, hence
// the ReferenceEquals condition. If we don't do that and have two different PendingImportResource objects pointing
// to the same ResourcePath (which are therefore "equal"), the registration fails as described above, but the
// deregistration succeeds. As a result, the ImportJob will not be saved to disk completely on shutdown.
PendingImportResourceNewGen registeredPendingImportResourceNewGen;
if (_pendingImportResources.TryGetValue(pendingImportResource.PendingResourcePath, out registeredPendingImportResourceNewGen))
{
if (ReferenceEquals(pendingImportResource, registeredPendingImportResourceNewGen))
{
PendingImportResourceNewGen removedPendingImportResource;
if (!_pendingImportResources.TryRemove(pendingImportResource.PendingResourcePath, out removedPendingImportResource))
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: Could not unregister {1}", this, pendingImportResource);
}
}
}
// If this ImportJobController is not completed, notify the ImporterWorker
// every 25 disposed (i.e. completed) PendingImportResources
if (_notifyProgress && Interlocked.Increment(ref _numberOfDisposedPendingImportResources) % 25 == 0)
{
_parentImporterWorker.NotifyProgress(false);
}
}
public void RegisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
{
if (!_pendingImportResources.TryAdd(pendingImportResource.PendingResourcePath, pendingImportResource))
{
// Stubs are special because they are all based on the same resource path and therefore should not
// be disposed.
if (pendingImportResource.IsValid && pendingImportResource.IsStubResource)
{
lock (_pendingStubResourcesRefCount)
{
if (!_pendingStubResourcesRefCount.ContainsKey(pendingImportResource.PendingResourcePath))
{
_pendingStubResourcesRefCount.Add(pendingImportResource.PendingResourcePath, 1);
}
else
{
_pendingStubResourcesRefCount[pendingImportResource.PendingResourcePath]++;
}
}
return;
}
// Due to the BoundedCapacity of the of FileUnfoldBlock.OutputBlock and MetadataExtractorBlock.InputBlock
// it may be the case that a directory resource has already been processed by the FileUnfoldBlock but is
// not immediately passed to the MetadataExtractorBlock.InputBlock, where its CurrentBlock property is set.
// If in this situation we suspend the ImportJob to disk and later on resume from disk, the directory resource
// is processed again by the FileUnfoldBlock although the contained file resources have already been unfolded.
// As a result, the PendingImportResource object for this file resource is created twice, which, at the creation
// of the second object, returns false when calling TryAdd above. We can safely set the IsValid property of the
// second one to false to filter it out (i.e. it will be disposed in the respective OutputBlock).
pendingImportResource.IsValid = false;
}
}
/// <summary>
/// Instantiates all the necessary DataflowBlocks for the given ImportJob
/// </summary>
/// <remarks>
/// We first have to distinguish between two cases here:
/// - BasePath points to a resource for which we can only create an IResourceAccessor - not an IFilesystemResourceAccessor
/// For this case we only import that single resource and don't have to take care of directories and subdirectories
/// ToDo: This still needs to be implemented
/// - BasePath points to a resource for which we can create an IFilesystemResourceAccessor
/// Here we first check whether the resource on the given BasePath exists. If not, we do nothing.
/// If it does exist, we distinguish two cases:
/// - The ImportJob was restored from disk, in which case we push the existing PendingImportResources to the respective DataflowBlocks.
/// - The ImportJob was freshly created, in which case we push the BasePath to the first DataFlowBlock.
/// In this case there are three subcases:
/// - BasePath points to a single resource
/// - BasePath points to a directory which is not a single resource and the ImportJob does not include subdirectories
/// - BasePath points to a directory which is not a single resource and the ImportJob does include subdirectories
/// These subcases, however, are taken care of by the DataflowBlocks - not by the ImportJobController
/// </remarks>
private void SetupDataflowBlocks(IEnumerable <PendingImportResourceNewGen> pendingImportResources)
{
// If we cannot access the BasePath at all, we just log and return
IResourceAccessor ra = null;
try
{
if (!_importJobInformation.BasePath.TryCreateLocalResourceAccessor(out ra))
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: Unable to access resource path '{1}'.", this, _importJobInformation.BasePath);
return;
}
}
catch (Exception e)
{
ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Error while creating ResourceAccessor for resource path '{1}'.", e, this, _importJobInformation.BasePath);
if (ra != null)
{
ra.Dispose();
}
return;
}
try
{
// As of now we have a ResourceAccessor that needs to be disposed
using (ra)
{
// If we have a ResourceAccessor which is not an IFileSystemResourceAccessor, just import that single resource
if (!(ra is IFileSystemResourceAccessor))
{
// ToDo: Implement import of Non-IFilesSystemResourceAccessors
return;
}
// Now we are sure it is an IFileSystemResourceAccessor
var fsra = ra as IFileSystemResourceAccessor;
// If the BasePath does not exist, we do nothing. This is necessary to avoid whole shares being removed from
// the MediaLibrary when a RefreshImport is scheduled while e.g. network resources are unavailable.
// If fsra is a NetworkNeighborhoddResourceAccessor and its IsServerPath() method returns true, fsra.Exists()
// will always return true. If therefore the BasePath of this Import points to a whole server and this server
// is not available during a RefreshImport, the whole share will be deleted from the MediaLibrary.
// ToDo: Rework this in NetworkNeighborhoodResourceAccessor
if (!fsra.Exists)
{
ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: Resource '{1}' does not exists or is not available.", this, _importJobInformation.BasePath);
return;
}
// Now we are sure that we need a DataflowBlock network
// Create the blocks
_dataflowBlocks.Add(new DirectoryUnfoldBlock(_cts.Token, _importJobInformation, this));
_dataflowBlocks.Add(new DirectorySaveBlock(_cts.Token, _importJobInformation, this));
_dataflowBlocks.Add(new FileUnfoldBlock(_cts.Token, _importJobInformation, this));
_dataflowBlocks.Add(new MetadataExtractorBlock(_cts.Token, _importJobInformation, this, false));
_dataflowBlocks.Add(new MediaItemSaveBlock(_cts.Token, _importJobInformation, this));
// Link the blocks
for (int i = 0; i < _dataflowBlocks.Count - 1; i++)
{
_dataflowBlocks[i].LinkTo(_dataflowBlocks[i + 1], new DataflowLinkOptions {
PropagateCompletion = true
});
}
_dataflowBlocks[_dataflowBlocks.Count - 1].LinkTo(DataflowBlock.NullTarget <PendingImportResourceNewGen>());
// Fill the blocks
var completeFirstBlockAfterTheseTasks = new HashSet <Task>();
bool firstBlockNeedsCompletion = true;
if (pendingImportResources == null)
{
// This ImportJob was freshly created and not persisted to disk before
// Just post the BasePath as new PendingImportResource
var rootImportResource = new PendingImportResourceNewGen(null, fsra.Clone() as IFileSystemResourceAccessor, DirectoryUnfoldBlock.BLOCK_NAME, this);
_dataflowBlocks[0].Post(rootImportResource);
firstBlockNeedsCompletion = false;
}
else
{
// This ImportJob was persisted to disk before
int numberOfRestoredPendingResources = 0;
foreach (var pendingImportResource in pendingImportResources)
{
pendingImportResource.InitializeAfterDeserialization(this);
ImporterWorkerDataflowBlockBase block = _dataflowBlocks.Find(b => b.ToString() == pendingImportResource.CurrentBlock);
if (block != null)
{
completeFirstBlockAfterTheseTasks.Add(block.SendAsync(pendingImportResource, _cts.Token));
numberOfRestoredPendingResources++;
if (block == _dataflowBlocks[0])
{
firstBlockNeedsCompletion = false;
}
}
else
{
ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Could not add {1} after deserialization. DataflowBlock with name {2} does not exist.", this, pendingImportResource, pendingImportResource.CurrentBlock);
pendingImportResource.Dispose();
}
}
ServiceRegistration.Get <ILogger>().Debug("ImporterWorker.{0}: {1} PendingImportResources restored.", this, numberOfRestoredPendingResources);
}
completeFirstBlockAfterTheseTasks.Add(_firstBlockHasFinished.Task);
if (firstBlockNeedsCompletion)
{
FirstBlockHasFinished();
}
// The first DataflowBlock in the network (DirectoryUnfoldBlock) must be set to completed when
// (a) The DirectoryUnfoldBlock has signaled that it is finished (by calling FirstBlockHasFinished()) and
// (b) in case of an ImportJob that has been restored from disk, all restored PendingImportResources
// have been put into the Dataflow network
Task.WhenAll(completeFirstBlockAfterTheseTasks).ContinueWith(previousTask => _dataflowBlocks[0].Complete());
}
}
catch (Exception e)
{
ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Error while setting up DataflowBlocks for resource path '{1}.", e, this, _importJobInformation.BasePath);
}
}
/// <summary>
/// Instantiates all the necessary DataflowBlocks for the given ImportJob
/// </summary>
/// <remarks>
/// We first have to distinguish between two cases here:
/// - BasePath points to a resource for which we can only create an IResourceAccessor - not an IFilesystemResourceAccessor
/// For this case we only import that single resource and don't have to take care of directories and subdirectories
/// ToDo: This still needs to be implemented
/// - BasePath points to a resource for which we can create an IFilesystemResourceAccessor
/// Here we first check whether the resource on the given BasePath exists. If not, we do nothing.
/// If it does exist, we distinguish two cases:
/// - The ImportJob was restored from disk, in which case we push the existing PendingImportResources to the respective DataflowBlocks.
/// - The ImportJob was freshly created, in which case we push the BasePath to the first DataFlowBlock.
/// In this case there are three subcases:
/// - BasePath points to a single resource
/// - BasePath points to a directory which is not a single resource and the ImportJob does not include subdirectories
/// - BasePath points to a directory which is not a single resource and the ImportJob does include subdirectories
/// These subcases, however, are taken care of by the DataflowBlocks - not by the ImportJobController
/// </remarks>
private void SetupDataflowBlocks(IEnumerable<PendingImportResourceNewGen> pendingImportResources)
{
// If we cannot access the BasePath at all, we just log and return
IResourceAccessor ra = null;
try
{
if (!_importJobInformation.BasePath.TryCreateLocalResourceAccessor(out ra))
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}: Unable to access resource path '{1}'.", this, _importJobInformation.BasePath);
return;
}
}
catch (Exception e)
{
ServiceRegistration.Get<ILogger>().Error("ImporterWorker.{0}: Error while creating ResourceAccessor for resource path '{1}'.", e, this, _importJobInformation.BasePath);
if (ra != null)
ra.Dispose();
return;
}
try
{
// As of now we have a ResourceAccessor that needs to be disposed
using (ra)
{
// If we have a ResourceAccessor which is not an IFileSystemResourceAccessor, just import that single resource
if (!(ra is IFileSystemResourceAccessor))
{
// ToDo: Implement import of Non-IFilesSystemResourceAccessors
return;
}
// Now we are sure it is an IFileSystemResourceAccessor
var fsra = ra as IFileSystemResourceAccessor;
// If the BasePath does not exist, we do nothing. This is necessary to avoid whole shares being removed from
// the MediaLibrary when a RefreshImport is scheduled while e.g. network resources are unavailable.
// If fsra is a NetworkNeighborhoddResourceAccessor and its IsServerPath() method returns true, fsra.Exists()
// will always return true. If therefore the BasePath of this Import points to a whole server and this server
// is not available during a RefreshImport, the whole share will be deleted from the MediaLibrary.
// ToDo: Rework this in NetworkNeighborhoodResourceAccessor
if (!fsra.Exists)
{
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}: Resource '{1}' does not exists or is not available.", this, _importJobInformation.BasePath);
return;
}
// Now we are sure that we need a DataflowBlock network
// Create the blocks
_dataflowBlocks.Add(new DirectoryUnfoldBlock(_cts.Token, _importJobInformation, this));
_dataflowBlocks.Add(new DirectorySaveBlock(_cts.Token, _importJobInformation, this));
_dataflowBlocks.Add(new FileUnfoldBlock(_cts.Token, _importJobInformation, this));
_dataflowBlocks.Add(new MetadataExtractorBlock(_cts.Token, _importJobInformation, this, false));
_dataflowBlocks.Add(new MediaItemSaveBlock(_cts.Token, _importJobInformation, this));
// Link the blocks
for (int i = 0; i < _dataflowBlocks.Count - 1; i++)
_dataflowBlocks[i].LinkTo(_dataflowBlocks[i + 1], new DataflowLinkOptions { PropagateCompletion = true });
_dataflowBlocks[_dataflowBlocks.Count - 1].LinkTo(DataflowBlock.NullTarget<PendingImportResourceNewGen>());
// Fill the blocks
var completeFirstBlockAfterTheseTasks = new HashSet<Task>();
bool firstBlockNeedsCompletion = true;
if (pendingImportResources == null)
{
// This ImportJob was freshly created and not persisted to disk before
// Just post the BasePath as new PendingImportResource
var rootImportResource = new PendingImportResourceNewGen(null, fsra.Clone() as IFileSystemResourceAccessor, DirectoryUnfoldBlock.BLOCK_NAME, this);
_dataflowBlocks[0].Post(rootImportResource);
firstBlockNeedsCompletion = false;
}
else
{
// This ImportJob was persisted to disk before
int numberOfRestoredPendingResources = 0;
foreach (var pendingImportResource in pendingImportResources)
{
pendingImportResource.InitializeAfterDeserialization(this);
ImporterWorkerDataflowBlockBase block = _dataflowBlocks.Find(b => b.ToString() == pendingImportResource.CurrentBlock);
if (block != null)
{
completeFirstBlockAfterTheseTasks.Add(block.SendAsync(pendingImportResource, _cts.Token));
numberOfRestoredPendingResources++;
if (block == _dataflowBlocks[0])
firstBlockNeedsCompletion = false;
}
else
{
ServiceRegistration.Get<ILogger>().Error("ImporterWorker.{0}: Could not add {1} after deserialization. DataflowBlock with name {2} does not exist.", this, pendingImportResource, pendingImportResource.CurrentBlock);
pendingImportResource.Dispose();
}
}
ServiceRegistration.Get<ILogger>().Debug("ImporterWorker.{0}: {1} PendingImportResources restored.", this, numberOfRestoredPendingResources);
}
completeFirstBlockAfterTheseTasks.Add(_firstBlockHasFinished.Task);
if (firstBlockNeedsCompletion)
FirstBlockHasFinished();
// The first DataflowBlock in the network (DirectoryUnfoldBlock) must be set to completed when
// (a) The DirectoryUnfoldBlock has signaled that it is finished (by calling FirstBlockHasFinished()) and
// (b) in case of an ImportJob that has been restored from disk, all restored PendingImportResources
// have been put into the Dataflow network
Task.WhenAll(completeFirstBlockAfterTheseTasks).ContinueWith(previousTask => _dataflowBlocks[0].Complete());
}
}
catch (Exception e)
{
ServiceRegistration.Get<ILogger>().Error("ImporterWorker.{0}: Error while setting up DataflowBlocks for resource path '{1}.", e, this, _importJobInformation.BasePath);
}
}
public void UnregisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
{
// We need to make sure that only the PendingImportResource object that is registered can unregister itself, hence
// the ReferenceEquals condition. If we don't do that and have two different PendingImportResource objects pointing
// to the same ResourcePath (which are therefore "equal"), the registration fails as described above, but the
// deregistration succeeds. As a result, the ImportJob will not be saved to disk completely on shutdown.
PendingImportResourceNewGen registeredPendingImportResourceNewGen;
if (_pendingImportResources.TryGetValue(pendingImportResource.PendingResourcePath, out registeredPendingImportResourceNewGen))
if (ReferenceEquals(pendingImportResource, registeredPendingImportResourceNewGen))
{
PendingImportResourceNewGen removedPendingImportResource;
if (!_pendingImportResources.TryRemove(pendingImportResource.PendingResourcePath, out removedPendingImportResource))
ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}: Could not unregister {1}", this, pendingImportResource);
}
// If this ImportJobController is not completed, notify the ImporterWorker
// every 25 disposed (i.e. completed) PendingImportResources
if (_notifyProgress && Interlocked.Increment(ref _numberOfDisposedPendingImportResources) % 25 == 0)
_parentImporterWorker.NotifyProgress(false);
}
public void RegisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
{
if (!_pendingImportResources.TryAdd(pendingImportResource.PendingResourcePath, pendingImportResource))
{
// Due to the BoundedCapacity of the of FileUnfoldBlock.OutputBlock and MetadataExtractorBlock.InputBlock
// it may be the case that a directory resource has already been processed by the FileUnfoldBlock but is
// not immediately passed to the MetadataExtractorBlock.InputBlock, where its CurrentBlock property is set.
// If in this situation we suspend the ImportJob to disk and later on resume from disk, the directory resource
// is processed again by the FileUnfoldBlock although the contained file resources have already been unfolded.
// As a result, the PendingImportResource object for this file resource is created twice, which, at the creation
// of the second object, returns false when calling TryAdd above. We can safely set the IsValid property of the
// second one to false to filter it out (i.e. it will be disposed in the respective OutputBlock).
pendingImportResource.IsValid = false;
}
}