Exemple #1
0
        /// <summary>
        /// Initializes the DataflowBlock
        /// </summary>
        /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
        /// <param name="inputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InputBlock"/></param>
        /// <param name="innerBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InnerBlock"/></param>
        /// <param name="outputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="OutputBlock"/></param>
        /// <param name="blockname">Name of this DataflowBlock (must be unique in a given chain of DataflowBlocks)</param>
        /// <param name="isRestorePointAfterDeserialization">
        /// <c>true</c>, if after deserialiization from disk, <see cref="PendingImportResourceNewGen"/>s are restored to
        /// this block. If <c>false</c>, they are restored to the last passed DataflowBlock having this parameter set to <c>true</c>
        /// </param>
        /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
        protected ImporterWorkerDataflowBlockBase(ImportJobInformation importJobInformation, ExecutionDataflowBlockOptions inputBlockOptions, ExecutionDataflowBlockOptions innerBlockOptions, ExecutionDataflowBlockOptions outputBlockOptions, String blockname, bool isRestorePointAfterDeserialization, ImportJobController parentImportJobController, CancellationToken ct)
        {
            _blockName = blockname;
            _isRestorePointAfterDeserialization = isRestorePointAfterDeserialization;
            _ct = ct;
            ImportJobInformation      = importJobInformation;
            InputBlockOptions         = inputBlockOptions;
            InnerBlockOptions         = innerBlockOptions;
            OutputBlockOptions        = outputBlockOptions;
            ParentImportJobController = parentImportJobController;

            _stopWatch  = new Stopwatch();
            _tcs        = new TaskCompletionSource <object>();
            Activated   = new AsyncManualResetEvent(InnerBlockOptions.CancellationToken);
            InputBlock  = new TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => InputBlockMethod(p), InputBlockOptions);
            OutputBlock = new TransformManyBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => OutputBlockMethod(p), OutputBlockOptions);
            // ReSharper disable once DoNotCallOverridableMethodsInConstructor
            InnerBlock = CreateInnerBlock();

            InnerBlock.LinkTo(OutputBlock, new DataflowLinkOptions {
                PropagateCompletion = true
            });
            InputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
            InnerBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
            OutputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
            Task.WhenAll(InputBlock.Completion, InnerBlock.Completion, OutputBlock.Completion).ContinueWith(OnAllBlocksFinished);
        }
 /// <summary>
 /// Initiates the MediaItemSaveBlock
 /// </summary>
 /// <remarks>
 /// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
 /// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and 
 /// therefore does not need a BoundedCapacity.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
   : base(importJobInformation,
   new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
   new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 500 },
   new ExecutionDataflowBlockOptions { CancellationToken = ct },
   BLOCK_NAME, false, parentImportJobController)
 {
 }
 /// <summary>
 /// Initiates the DirectorySaveBlock
 /// </summary>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
   : base(importJobInformation,
   new ExecutionDataflowBlockOptions { CancellationToken = ct },
   new ExecutionDataflowBlockOptions { CancellationToken = ct },
   new ExecutionDataflowBlockOptions { CancellationToken = ct },
   BLOCK_NAME, true, parentImportJobController)
 {
 }
 /// <summary>
 /// Initiates the MetadataExtractorBlock
 /// </summary>
 /// <remarks>
 /// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
 /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
 /// set the BoundedCapacity of the OutputBlock to 1.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 /// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param>
 public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
   : base(importJobInformation,
   new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
   new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100 },
   new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
   forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController)
 {
   _forceQuickMode = forceQuickMode;
   _mostRecentMiaCreationDate = new Lazy<Task<DateTime>>(GetMostRecentMiaCreationDate);
 }
Exemple #5
0
 /// <summary>
 /// Initiates the MetadataExtractorBlock
 /// </summary>
 /// <remarks>
 /// The preceding MediaItemLoadBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
 /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
 /// set the BoundedCapacity of the OutputBlock to 1.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            BLOCK_NAME, true, parentImportJobController, ct)
 {
     _forceQuickMode = forceQuickMode;
 }
 /// <summary>
 /// Initiates the DirectoryUnfoldBlock
 /// </summary>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public DirectoryUnfoldBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            BLOCK_NAME, true, parentImportJobController, ct)
 {
 }
Exemple #7
0
 /// <summary>
 /// Initiates the MetadataExtractorBlock
 /// </summary>
 /// <remarks>
 /// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
 /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
 /// set the BoundedCapacity of the OutputBlock to 1.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 /// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param>
 public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController)
 {
     _forceQuickMode            = forceQuickMode;
     _mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate);
 }
Exemple #8
0
 /// <summary>
 /// Initiates the RelationshipExtractorBlock
 /// </summary>
 /// <remarks>
 /// The preceding MediaItemSaveBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this RelationshipExtractorBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 50,
 /// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
 /// therefore does not need a BoundedCapacity.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public RelationshipExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 50
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            BLOCK_NAME, true, parentImportJobController, ct)
 {
     _relationshipCache     = new LRURelationshipCache();
     _cacheSync             = new SemaphoreSlim(1, 1);
     _processedMediaItemIds = new ConcurrentDictionary <Guid, byte>();
 }
Exemple #9
0
 /// <summary>
 /// Initiates the MediaItemLoadBlock
 /// </summary>
 /// <remarks>
 /// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
 /// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
 /// set the BoundedCapacity of the OutputBlock to 1.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public MediaItemLoadBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 500
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            BLOCK_NAME, false, parentImportJobController, ct)
 {
     _mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate);
 }
 /// <summary>
 /// Initiates the DirectorySaveBlock
 /// </summary>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            BLOCK_NAME, true, parentImportJobController)
 {
 }
Exemple #11
0
 /// <summary>
 /// Initiates the MediaItemSaveBlock
 /// </summary>
 /// <remarks>
 /// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
 /// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we
 /// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
 /// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
 /// therefore does not need a BoundedCapacity.
 /// </remarks>
 /// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
 /// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
 /// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
 public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
     : base(importJobInformation,
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 1
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct, BoundedCapacity = 500
 },
            new ExecutionDataflowBlockOptions {
     CancellationToken = ct
 },
            BLOCK_NAME, false, parentImportJobController)
 {
 }