Exemple #1
0
        public void UnregisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
        {
            // We need to make sure that only the PendingImportResource object that is registered can unregister itself, hence
            // the ReferenceEquals condition. If we don't do that and have two different PendingImportResource objects pointing
            // to the same ResourcePath (which are therefore "equal"), the registration fails as described above, but the
            // deregistration succeeds. As a result, the ImportJob will not be saved to disk completely on shutdown.
            PendingImportResourceNewGen registeredPendingImportResourceNewGen;

            if (_pendingImportResources.TryGetValue(pendingImportResource.PendingResourcePath, out registeredPendingImportResourceNewGen))
            {
                if (ReferenceEquals(pendingImportResource, registeredPendingImportResourceNewGen))
                {
                    PendingImportResourceNewGen removedPendingImportResource;
                    if (!_pendingImportResources.TryRemove(pendingImportResource.PendingResourcePath, out removedPendingImportResource))
                    {
                        ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: Could not unregister {1}", this, pendingImportResource);
                    }
                }
            }

            // If this ImportJobController is not completed, notify the ImporterWorker
            // every 25 disposed (i.e. completed) PendingImportResources
            if (_notifyProgress && Interlocked.Increment(ref _numberOfDisposedPendingImportResources) % 25 == 0)
            {
                _parentImporterWorker.NotifyProgress(false);
            }
        }
        public void RegisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
        {
            if (!_pendingImportResources.TryAdd(pendingImportResource.PendingResourcePath, pendingImportResource))
            {
                // Stubs are special because they are all based on the same resource path and therefore should not
                // be disposed.
                if (pendingImportResource.IsValid && pendingImportResource.IsStubResource)
                {
                    lock (_pendingStubResourcesRefCount)
                    {
                        if (!_pendingStubResourcesRefCount.ContainsKey(pendingImportResource.PendingResourcePath))
                        {
                            _pendingStubResourcesRefCount.Add(pendingImportResource.PendingResourcePath, 1);
                        }
                        else
                        {
                            _pendingStubResourcesRefCount[pendingImportResource.PendingResourcePath]++;
                        }
                    }
                    return;
                }

                // Due to the BoundedCapacity of the of FileUnfoldBlock.OutputBlock and MetadataExtractorBlock.InputBlock
                // it may be the case that a directory resource has already been processed by the FileUnfoldBlock but is
                // not immediately passed to the MetadataExtractorBlock.InputBlock, where its CurrentBlock property is set.
                // If in this situation we suspend the ImportJob to disk and later on resume from disk, the directory resource
                // is processed again by the FileUnfoldBlock although the contained file resources have already been unfolded.
                // As a result, the PendingImportResource object for this file resource is created twice, which, at the creation
                // of the second object, returns false when calling TryAdd above. We can safely set the IsValid property of the
                // second one to false to filter it out (i.e. it will be disposed in the respective OutputBlock).
                pendingImportResource.IsValid = false;
            }
        }
Exemple #3
0
        /// <summary>
        /// Instantiates all the necessary DataflowBlocks for the given ImportJob
        /// </summary>
        /// <remarks>
        /// We first have to distinguish between two cases here:
        ///  - BasePath points to a resource for which we can only create an IResourceAccessor - not an IFilesystemResourceAccessor
        ///    For this case we only import that single resource and don't have to take care of directories and subdirectories
        ///    ToDo: This still needs to be implemented
        /// - BasePath points to a resource for which we can create an IFilesystemResourceAccessor
        ///   Here we first check whether the resource on the given BasePath exists. If not, we do nothing.
        ///   If it does exist, we distinguish two cases:
        ///   - The ImportJob was restored from disk, in which case we push the existing PendingImportResources to the respective DataflowBlocks.
        ///   - The ImportJob was freshly created, in which case we push the BasePath to the first DataFlowBlock.
        ///     In this case there are three subcases:
        ///     - BasePath points to a single resource
        ///     - BasePath points to a directory which is not a single resource and the ImportJob does not include subdirectories
        ///     - BasePath points to a directory which is not a single resource and the ImportJob does include subdirectories
        ///     These subcases, however, are taken care of by the DataflowBlocks - not by the ImportJobController
        /// </remarks>
        private void SetupDataflowBlocks(IEnumerable <PendingImportResourceNewGen> pendingImportResources)
        {
            // If we cannot access the BasePath at all, we just log and return
            IResourceAccessor ra = null;

            try
            {
                if (!_importJobInformation.BasePath.TryCreateLocalResourceAccessor(out ra))
                {
                    ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: Unable to access resource path '{1}'.", this, _importJobInformation.BasePath);
                    return;
                }
            }
            catch (Exception e)
            {
                ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Error while creating ResourceAccessor for resource path '{1}'.", e, this, _importJobInformation.BasePath);
                if (ra != null)
                {
                    ra.Dispose();
                }
                return;
            }

            try
            {
                // As of now we have a ResourceAccessor that needs to be disposed
                using (ra)
                {
                    // If we have a ResourceAccessor which is not an IFileSystemResourceAccessor, just import that single resource
                    if (!(ra is IFileSystemResourceAccessor))
                    {
                        // ToDo: Implement import of Non-IFilesSystemResourceAccessors
                        return;
                    }

                    // Now we are sure it is an IFileSystemResourceAccessor
                    var fsra = ra as IFileSystemResourceAccessor;

                    // If the BasePath does not exist, we do nothing. This is necessary to avoid whole shares being removed from
                    // the MediaLibrary when a RefreshImport is scheduled while e.g. network resources are unavailable.
                    // If fsra is a NetworkNeighborhoddResourceAccessor and its IsServerPath() method returns true, fsra.Exists()
                    // will always return true. If therefore the BasePath of this Import points to a whole server and this server
                    // is not available during a RefreshImport, the whole share will be deleted from the MediaLibrary.
                    // ToDo: Rework this in NetworkNeighborhoodResourceAccessor
                    if (!fsra.Exists)
                    {
                        ServiceRegistration.Get <ILogger>().Warn("ImporterWorker.{0}: Resource '{1}' does not exists or is not available.", this, _importJobInformation.BasePath);
                        return;
                    }

                    // Now we are sure that we need a DataflowBlock network

                    // Create the blocks
                    _dataflowBlocks.Add(new DirectoryUnfoldBlock(_cts.Token, _importJobInformation, this));
                    _dataflowBlocks.Add(new DirectorySaveBlock(_cts.Token, _importJobInformation, this));
                    _dataflowBlocks.Add(new FileUnfoldBlock(_cts.Token, _importJobInformation, this));
                    _dataflowBlocks.Add(new MetadataExtractorBlock(_cts.Token, _importJobInformation, this, false));
                    _dataflowBlocks.Add(new MediaItemSaveBlock(_cts.Token, _importJobInformation, this));

                    // Link the blocks
                    for (int i = 0; i < _dataflowBlocks.Count - 1; i++)
                    {
                        _dataflowBlocks[i].LinkTo(_dataflowBlocks[i + 1], new DataflowLinkOptions {
                            PropagateCompletion = true
                        });
                    }
                    _dataflowBlocks[_dataflowBlocks.Count - 1].LinkTo(DataflowBlock.NullTarget <PendingImportResourceNewGen>());

                    // Fill the blocks
                    var  completeFirstBlockAfterTheseTasks = new HashSet <Task>();
                    bool firstBlockNeedsCompletion         = true;
                    if (pendingImportResources == null)
                    {
                        // This ImportJob was freshly created and not persisted to disk before
                        // Just post the BasePath as new PendingImportResource
                        var rootImportResource = new PendingImportResourceNewGen(null, fsra.Clone() as IFileSystemResourceAccessor, DirectoryUnfoldBlock.BLOCK_NAME, this);
                        _dataflowBlocks[0].Post(rootImportResource);
                        firstBlockNeedsCompletion = false;
                    }
                    else
                    {
                        // This ImportJob was persisted to disk before
                        int numberOfRestoredPendingResources = 0;
                        foreach (var pendingImportResource in pendingImportResources)
                        {
                            pendingImportResource.InitializeAfterDeserialization(this);
                            ImporterWorkerDataflowBlockBase block = _dataflowBlocks.Find(b => b.ToString() == pendingImportResource.CurrentBlock);
                            if (block != null)
                            {
                                completeFirstBlockAfterTheseTasks.Add(block.SendAsync(pendingImportResource, _cts.Token));
                                numberOfRestoredPendingResources++;
                                if (block == _dataflowBlocks[0])
                                {
                                    firstBlockNeedsCompletion = false;
                                }
                            }
                            else
                            {
                                ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Could not add {1} after deserialization. DataflowBlock with name {2} does not exist.", this, pendingImportResource, pendingImportResource.CurrentBlock);
                                pendingImportResource.Dispose();
                            }
                        }
                        ServiceRegistration.Get <ILogger>().Debug("ImporterWorker.{0}: {1} PendingImportResources restored.", this, numberOfRestoredPendingResources);
                    }
                    completeFirstBlockAfterTheseTasks.Add(_firstBlockHasFinished.Task);
                    if (firstBlockNeedsCompletion)
                    {
                        FirstBlockHasFinished();
                    }

                    // The first DataflowBlock in the network (DirectoryUnfoldBlock) must be set to completed when
                    // (a) The DirectoryUnfoldBlock has signaled that it is finished (by calling FirstBlockHasFinished()) and
                    // (b) in case of an ImportJob that has been restored from disk, all restored PendingImportResources
                    //     have been put into the Dataflow network
                    Task.WhenAll(completeFirstBlockAfterTheseTasks).ContinueWith(previousTask => _dataflowBlocks[0].Complete());
                }
            }
            catch (Exception e)
            {
                ServiceRegistration.Get <ILogger>().Error("ImporterWorker.{0}: Error while setting up DataflowBlocks for resource path '{1}.", e, this, _importJobInformation.BasePath);
            }
        }
    /// <summary>
    /// Instantiates all the necessary DataflowBlocks for the given ImportJob
    /// </summary>
    /// <remarks>
    /// We first have to distinguish between two cases here:
    ///  - BasePath points to a resource for which we can only create an IResourceAccessor - not an IFilesystemResourceAccessor
    ///    For this case we only import that single resource and don't have to take care of directories and subdirectories
    ///    ToDo: This still needs to be implemented
    /// - BasePath points to a resource for which we can create an IFilesystemResourceAccessor
    ///   Here we first check whether the resource on the given BasePath exists. If not, we do nothing.
    ///   If it does exist, we distinguish two cases:
    ///   - The ImportJob was restored from disk, in which case we push the existing PendingImportResources to the respective DataflowBlocks.
    ///   - The ImportJob was freshly created, in which case we push the BasePath to the first DataFlowBlock.
    ///     In this case there are three subcases:
    ///     - BasePath points to a single resource
    ///     - BasePath points to a directory which is not a single resource and the ImportJob does not include subdirectories
    ///     - BasePath points to a directory which is not a single resource and the ImportJob does include subdirectories
    ///     These subcases, however, are taken care of by the DataflowBlocks - not by the ImportJobController
    /// </remarks>
    private void SetupDataflowBlocks(IEnumerable<PendingImportResourceNewGen> pendingImportResources)
    {
      // If we cannot access the BasePath at all, we just log and return
      IResourceAccessor ra = null;
      try
      {
        if (!_importJobInformation.BasePath.TryCreateLocalResourceAccessor(out ra))
        {
          ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}: Unable to access resource path '{1}'.", this, _importJobInformation.BasePath);
          return;
        }
      }
      catch (Exception e)
      {
        ServiceRegistration.Get<ILogger>().Error("ImporterWorker.{0}: Error while creating ResourceAccessor for resource path '{1}'.", e, this, _importJobInformation.BasePath);
        if (ra != null)
          ra.Dispose();
        return;
      }

      try
      {
        // As of now we have a ResourceAccessor that needs to be disposed
        using (ra)
        {
          // If we have a ResourceAccessor which is not an IFileSystemResourceAccessor, just import that single resource
          if (!(ra is IFileSystemResourceAccessor))
          {
            // ToDo: Implement import of Non-IFilesSystemResourceAccessors
            return;
          }

          // Now we are sure it is an IFileSystemResourceAccessor
          var fsra = ra as IFileSystemResourceAccessor;

          // If the BasePath does not exist, we do nothing. This is necessary to avoid whole shares being removed from
          // the MediaLibrary when a RefreshImport is scheduled while e.g. network resources are unavailable.
          // If fsra is a NetworkNeighborhoddResourceAccessor and its IsServerPath() method returns true, fsra.Exists()
          // will always return true. If therefore the BasePath of this Import points to a whole server and this server
          // is not available during a RefreshImport, the whole share will be deleted from the MediaLibrary.
          // ToDo: Rework this in NetworkNeighborhoodResourceAccessor
          if (!fsra.Exists)
          {
            ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}: Resource '{1}' does not exists or is not available.", this, _importJobInformation.BasePath);
            return;
          }
        
          // Now we are sure that we need a DataflowBlock network

          // Create the blocks
          _dataflowBlocks.Add(new DirectoryUnfoldBlock(_cts.Token, _importJobInformation, this));
          _dataflowBlocks.Add(new DirectorySaveBlock(_cts.Token, _importJobInformation, this));
          _dataflowBlocks.Add(new FileUnfoldBlock(_cts.Token, _importJobInformation, this));
          _dataflowBlocks.Add(new MetadataExtractorBlock(_cts.Token, _importJobInformation, this, false));
          _dataflowBlocks.Add(new MediaItemSaveBlock(_cts.Token, _importJobInformation, this));

          // Link the blocks
          for (int i = 0; i < _dataflowBlocks.Count - 1; i++)
            _dataflowBlocks[i].LinkTo(_dataflowBlocks[i + 1], new DataflowLinkOptions { PropagateCompletion = true });
          _dataflowBlocks[_dataflowBlocks.Count - 1].LinkTo(DataflowBlock.NullTarget<PendingImportResourceNewGen>());

          // Fill the blocks
          var completeFirstBlockAfterTheseTasks = new HashSet<Task>();
          bool firstBlockNeedsCompletion = true;
          if (pendingImportResources == null)
          {
            // This ImportJob was freshly created and not persisted to disk before
            // Just post the BasePath as new PendingImportResource
            var rootImportResource = new PendingImportResourceNewGen(null, fsra.Clone() as IFileSystemResourceAccessor, DirectoryUnfoldBlock.BLOCK_NAME, this);
            _dataflowBlocks[0].Post(rootImportResource);
            firstBlockNeedsCompletion = false;
          }
          else
          {
            // This ImportJob was persisted to disk before
            int numberOfRestoredPendingResources = 0;
            foreach (var pendingImportResource in pendingImportResources)
            {
              pendingImportResource.InitializeAfterDeserialization(this);
              ImporterWorkerDataflowBlockBase block = _dataflowBlocks.Find(b => b.ToString() == pendingImportResource.CurrentBlock);
              if (block != null)
              {
                completeFirstBlockAfterTheseTasks.Add(block.SendAsync(pendingImportResource, _cts.Token));
                numberOfRestoredPendingResources++;
                if (block == _dataflowBlocks[0])
                  firstBlockNeedsCompletion = false;
              }
              else
              {
                ServiceRegistration.Get<ILogger>().Error("ImporterWorker.{0}: Could not add {1} after deserialization. DataflowBlock with name {2} does not exist.", this, pendingImportResource, pendingImportResource.CurrentBlock);
                pendingImportResource.Dispose();
              }
            }
            ServiceRegistration.Get<ILogger>().Debug("ImporterWorker.{0}: {1} PendingImportResources restored.", this, numberOfRestoredPendingResources);
          }
          completeFirstBlockAfterTheseTasks.Add(_firstBlockHasFinished.Task);
          if (firstBlockNeedsCompletion)
            FirstBlockHasFinished();

          // The first DataflowBlock in the network (DirectoryUnfoldBlock) must be set to completed when
          // (a) The DirectoryUnfoldBlock has signaled that it is finished (by calling FirstBlockHasFinished()) and
          // (b) in case of an ImportJob that has been restored from disk, all restored PendingImportResources
          //     have been put into the Dataflow network
          Task.WhenAll(completeFirstBlockAfterTheseTasks).ContinueWith(previousTask => _dataflowBlocks[0].Complete());
        }
      }
      catch (Exception e)
      {
        ServiceRegistration.Get<ILogger>().Error("ImporterWorker.{0}: Error while setting up DataflowBlocks for resource path '{1}.", e, this, _importJobInformation.BasePath);
      }
    }
 public void UnregisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
 {
   // We need to make sure that only the PendingImportResource object that is registered can unregister itself, hence
   // the ReferenceEquals condition. If we don't do that and have two different PendingImportResource objects pointing
   // to the same ResourcePath (which are therefore "equal"), the registration fails as described above, but the 
   // deregistration succeeds. As a result, the ImportJob will not be saved to disk completely on shutdown.
   PendingImportResourceNewGen registeredPendingImportResourceNewGen;
   if (_pendingImportResources.TryGetValue(pendingImportResource.PendingResourcePath, out registeredPendingImportResourceNewGen))
     if (ReferenceEquals(pendingImportResource, registeredPendingImportResourceNewGen))
     {
       PendingImportResourceNewGen removedPendingImportResource;
       if (!_pendingImportResources.TryRemove(pendingImportResource.PendingResourcePath, out removedPendingImportResource))
         ServiceRegistration.Get<ILogger>().Warn("ImporterWorker.{0}: Could not unregister {1}", this, pendingImportResource);          
     }      
   
   // If this ImportJobController is not completed, notify the ImporterWorker
   // every 25 disposed (i.e. completed) PendingImportResources
   if (_notifyProgress && Interlocked.Increment(ref _numberOfDisposedPendingImportResources) % 25 == 0)
     _parentImporterWorker.NotifyProgress(false);
 }
 public void RegisterPendingImportResource(PendingImportResourceNewGen pendingImportResource)
 {
   if (!_pendingImportResources.TryAdd(pendingImportResource.PendingResourcePath, pendingImportResource))
   {
     // Due to the BoundedCapacity of the of FileUnfoldBlock.OutputBlock and MetadataExtractorBlock.InputBlock
     // it may be the case that a directory resource has already been processed by the FileUnfoldBlock but is
     // not immediately passed to the MetadataExtractorBlock.InputBlock, where its CurrentBlock property is set.
     // If in this situation we suspend the ImportJob to disk and later on resume from disk, the directory resource
     // is processed again by the FileUnfoldBlock although the contained file resources have already been unfolded.
     // As a result, the PendingImportResource object for this file resource is created twice, which, at the creation
     // of the second object, returns false when calling TryAdd above. We can safely set the IsValid property of the
     // second one to false to filter it out (i.e. it will be disposed in the respective OutputBlock).
     pendingImportResource.IsValid = false;
   }
 }