public IndexReader GetIndexReader(IDirectoryProvider provider, System.Type entity)
        {
            // one cannot access a reader for update after a writer has been accessed
            if (writers.ContainsKey(provider))
            {
                throw new AssertionFailure("Tries to read for update a index while a writer is accessed" + entity);
            }

            IndexReader reader;
            readers.TryGetValue(provider, out reader);
            if (reader != null)
            {
                return reader;
            }

            LockProvider(provider);
            dpStatistics[provider].Operations++;

            try
            {
                reader = IndexReader.Open(provider.Directory);
                readers.Add(provider, reader);
            }
            catch (IOException e)
            {
                CleanUp(new SearchException("Unable to open IndexReader for " + entity, e));
            }

            return reader;
        }
 public void Initialize(object properties, IDirectoryProvider[] providers)
 {
     directoryProvider = providers;
     if (providers.GetUpperBound(0) > 1)
     {
         throw new AssertionFailure("Using SingleDirectoryProviderSelectionStrategy with multiple DirectoryProviders");
     }
 }
 public void IdShardingStrategy()
 {
     IDirectoryProvider[] dps = new IDirectoryProvider[] {new RAMDirectoryProvider(), new RAMDirectoryProvider()};
     IdHashShardingStrategy shardingStrategy = new IdHashShardingStrategy();
     shardingStrategy.Initialize(null, dps);
     Assert.AreSame(dps[1], shardingStrategy.GetDirectoryProviderForAddition(typeof(Animal), 1, "1", null));
     Assert.AreSame(dps[0], shardingStrategy.GetDirectoryProviderForAddition(typeof(Animal), 2, "2", null));
 }
        public void Initialize(object properties, IDirectoryProvider[] providers)
        {
            if (providers == null)
            {
                throw new ArgumentNullException("providers");
            }

            this.providers = providers;
        }
Example #5
0
        /// <summary>
        /// Initializes a new instance of the <see cref="FileSystemInterface"/> class.
        /// </summary>
        /// <param name="config"></param>
        /// <param name="directoryProvider"></param>
        public FileSystemInterface(ConfigHandling config, IDirectoryProvider directoryProvider)
        {
            _config = config;
            _directoryProvider = directoryProvider;

            SearchList = new List<string> { "No", "Value" };

            //// Filling the Directory and search datatable
            this.UpdateDirectoryList(directoryProvider);
        }
        public void Initialize(IDirectoryProvider directoryProvider, IDictionary<string, string> indexProperties,
            ISearchFactoryImplementor searchFactoryImplementor)
        {
            this.directoryProvider = directoryProvider;
            string maxString;
            indexProperties.TryGetValue("optimizer.operation_limit.max", out maxString);

            if (!string.IsNullOrEmpty(maxString))
                int.TryParse(maxString, out operationMax);

            indexProperties.TryGetValue("optimizer.transaction_limit.max", out maxString);
            if (!string.IsNullOrEmpty(maxString))
                int.TryParse(maxString, out transactionMax);
        }
        public void PerformWork(PurgeAllLuceneWork work, IDirectoryProvider provider)
        {
            System.Type entity = work.EntityClass;
            if (log.IsDebugEnabled)
                log.Debug("PurgeAll Lucene index: " + entity);

            IndexReader reader = workspace.GetIndexReader(provider, entity);
            try
            {
                Term term = new Term(DocumentBuilder.CLASS_FIELDNAME, TypeHelper.LuceneTypeName(entity));
                reader.DeleteDocuments(term);
            }
            catch (Exception e)
            {
                throw new SearchException("Unable to purge all from Lucene index: " + entity, e);
            }
        }
        public void PerformWork(OptimizeLuceneWork work, IDirectoryProvider provider)
        {
            System.Type entity = work.EntityClass;
            if (log.IsDebugEnabled)
                log.Debug("Optimize Lucene index: " + entity);
            IndexWriter writer = workspace.GetIndexWriter(provider, entity, false);

            try
            {
                writer.Optimize();
                workspace.Optimize(provider);
            }
            catch (IOException e)
            {
                throw new SearchException("Unable to optimize Lucene index: " + entity, e);
            }
        }
        public IndexReader OpenReader(IDirectoryProvider[] directoryProviders)
        {
            int length = directoryProviders.Length;
            IndexReader[] readers = new IndexReader[length];
            try
            {
                for (int index = 0; index < length; index++)
                    readers[index] = IndexReader.Open(directoryProviders[index].Directory);
            }
            catch (System.IO.IOException ex)
            {
                // TODO: more contextual info
                ReaderProviderHelper.Clean(readers);
                throw new SearchException("Unable to open one of the Lucene indexes", ex);
            }

            return ReaderProviderHelper.BuildMultiReader(length, readers);
        }
Example #10
0
        public DocumentBuilder(DocumentMapping classMapping, Analyzer defaultAnalyzer, IDirectoryProvider[] directoryProviders,
                               IIndexShardingStrategy shardingStrategy)
        {
            analyzer = new ScopedAnalyzer();
            this.directoryProviders = directoryProviders;
            this.shardingStrategy = shardingStrategy;

            if (classMapping == null) throw new AssertionFailure("Unable to build a DocumemntBuilder with a null class");

            rootClassMapping = classMapping;

            Set<System.Type> processedClasses = new HashedSet<System.Type>();
            processedClasses.Add(classMapping.MappedClass);
            CollectAnalyzers(rootClassMapping, defaultAnalyzer, true, string.Empty, processedClasses);
            //processedClasses.remove( clazz ); for the sake of completness
            analyzer.GlobalAnalyzer = defaultAnalyzer;
            if (idMapping == null)
                throw new SearchException("No document id for: " + classMapping.MappedClass.Name);
        }
        public DocumentBuilder(System.Type clazz, Analyzer defaultAnalyzer, IDirectoryProvider[] directoryProviders,
            IIndexShardingStrategy shardingStrategy)
        {
            analyzer = new ScopedAnalyzer();
            beanClass = clazz;
            this.directoryProviders = directoryProviders;
            this.shardingStrategy = shardingStrategy;

            if (clazz == null) throw new AssertionFailure("Unable to build a DocumemntBuilder with a null class");

            rootPropertiesMetadata = new PropertiesMetadata();
            rootPropertiesMetadata.boost = GetBoost(clazz);
            rootPropertiesMetadata.analyzer = defaultAnalyzer;

            Set<System.Type> processedClasses = new HashedSet<System.Type>();
            processedClasses.Add(clazz);
            InitializeMembers(clazz, rootPropertiesMetadata, true, string.Empty, processedClasses);
            //processedClasses.remove( clazz ); for the sake of completness
            analyzer.GlobalAnalyzer = rootPropertiesMetadata.analyzer;
            if (idKeywordName == null)
                throw new SearchException("No document id for: " + clazz.Name);
        }
 public object GetLockObjForDirectoryProvider(IDirectoryProvider provider)
 {
     return(lockableDirectoryProviders[provider]);
 }
 public DirectoryProviders(IIndexShardingStrategy shardingStrategy, IDirectoryProvider[] providers)
 {
     this.shardingStrategy = shardingStrategy;
     this.providers = providers;
 }
Example #14
0
 public ItemCountCheck(SoftDeletedRemovalData data, IDirectoryProvider directory) : base(data, directory)
 {
 }
Example #15
0
 public DirectoryForest(IDirectoryProvider directory, DirectoryIdentity identity) : base(directory, identity)
 {
 }
Example #16
0
 public DisconnectDateCheck(SoftDeletedRemovalData data, IDirectoryProvider directory, DateTime removalCutoffDate) : base(data, directory)
 {
     this.removalCutoffDate = removalCutoffDate;
 }
Example #17
0
 protected DirectoryContainerParent(IDirectoryProvider directory, DirectoryIdentity identity) : base(directory, identity)
 {
 }
Example #18
0
        protected virtual ILoadBalanceService CreateLoadBalancerClient(DirectoryServer server, IDirectoryProvider directory)
        {
            ILoadBalanceService result;

            using (DisposeGuard disposeGuard = default(DisposeGuard))
            {
                LoadBalancerClient loadBalancerClient = LoadBalancerClient.Create(server.Fqdn, directory, this.logger);
                disposeGuard.Add <LoadBalancerClient>(loadBalancerClient);
                bool flag = true;
                ILoadBalanceService loadBalanceService = loadBalancerClient;
                if (!loadBalancerClient.ServerVersion[1])
                {
                    flag = false;
                    loadBalanceService = this.CreateCompatibilityLoadBalanceClient(server);
                }
                else if (!loadBalancerClient.ServerVersion[2])
                {
                    loadBalanceService = new SoftDeletedRemovalCapabilityDecorator(loadBalanceService, server);
                }
                if (!loadBalancerClient.ServerVersion[3])
                {
                    loadBalanceService = new ConsumerMetricsLoadBalanceCapabilityDecorator(loadBalanceService, server);
                }
                if (!loadBalancerClient.ServerVersion[5])
                {
                    loadBalanceService = new CapacitySummaryCapabilityDecorator(loadBalanceService, server, this.serviceContext);
                }
                if (flag)
                {
                    disposeGuard.Success();
                }
                result = loadBalanceService;
            }
            return(result);
        }
Example #19
0
 public SettingsProvider(ILogger <SettingsProvider> logger, IDirectoryProvider directoryProvider)
 {
     this.logger            = logger;
     this.directoryProvider = directoryProvider;
     LoadOrCreateSettings();
 }
Example #20
0
 public InjectorService(IDirectoryProvider directory, ILogger logger, MoveInjector moveInjector) : base(logger)
 {
     this.directory    = directory;
     this.moveInjector = moveInjector;
 }
Example #21
0
        protected virtual IInjectorService CreateInjectorClient(DirectoryServer server, IDirectoryProvider directory)
        {
            IInjectorService result;

            using (DisposeGuard disposeGuard = default(DisposeGuard))
            {
                InjectorClient injectorClient = InjectorClient.Create(server.Fqdn, directory, this.logger);
                disposeGuard.Add <InjectorClient>(injectorClient);
                bool             flag            = true;
                IInjectorService injectorService = injectorClient;
                if (!injectorClient.ServerVersion[1])
                {
                    flag            = false;
                    injectorService = this.CreateCompatibilityInjectorClient(server);
                }
                if (!injectorClient.ServerVersion[2])
                {
                    injectorService = new ConsumerMetricsInjectorCapabilityDecorator(injectorService);
                }
                if (flag)
                {
                    disposeGuard.Success();
                }
                result = injectorService;
            }
            return(result);
        }
Example #22
0
 public void PerformWork(DeleteLuceneWork work, IDirectoryProvider provider)
 {
     Remove(work.EntityClass, work.Id, provider);
 }
Example #23
0
 public void PerformWork(AddLuceneWork work, IDirectoryProvider provider)
 {
     Add(work.EntityClass, work.Id, work.Document, provider);
 }
Example #24
0
 public WorkWithPayload(LuceneWork work, IDirectoryProvider provider)
 {
     this.work     = work;
     this.provider = provider;
 }
 public void AddIndexingParameters(IDirectoryProvider provider, LuceneIndexingParameters indexingParameters)
 {
     dirProviderIndexingParams[provider] = indexingParameters;
 }
Example #26
0
 public CopyDirectory(FSMasterDirectoryProvider parent, string source, string destination, IDirectoryProvider directoryProvider)
 {
     this.parent            = parent;
     this.source            = source;
     this.destination       = destination;
     this.directoryProvider = directoryProvider;
 }
            public void Run()
            {
                //TODO get rid of current and use the marker file instead?
                DateTime start = DateTime.Now;
                inProgress = true;
                if (directoryProviderLock == null)
                {
                    directoryProviderLock = parent.searchFactory.GetLockableDirectoryProviders()[directoryProvider];
                    directoryProvider = null;
                }

                try
                {
                    lock (directoryProviderLock)
                    {
                        int oldIndex = parent.current;
                        int index = parent.current == 1 ? 2 : 1;
                        DirectoryInfo sourceFile = new DirectoryInfo(source);

                        DirectoryInfo destinationFile = new DirectoryInfo(Path.Combine(destination, index.ToString()));
                        //TODO make smart a parameter
                        try
                        {
                            log.Info("Copying " + sourceFile + " into " + destinationFile);
                            FileHelper.Synchronize(sourceFile, destinationFile, true);
                            parent.current = index;
                        }
                        catch (IOException e)
                        {
                            //don't change current
                            log.Error("Unable to synchronize source of " + parent.indexName, e);
                            return;
                        }
                        try
                        {
                            File.Delete(Path.Combine(destination, "current" + oldIndex));
                        }
                        catch (IOException e)
                        {
                            log.Warn("Unable to remove previous marker file from source of " + parent.indexName, e);
                        }
                        try
                        {
                            File.Create(Path.Combine(destination, "current" + index)).Dispose();
                        }
                        catch (IOException e)
                        {
                            log.Warn("Unable to create current marker in source of " + parent.indexName, e);
                        }
                    }
                }
                finally
                {
                    inProgress = false;
                }
                log.InfoFormat("Copy for {0} took {1}.", parent.indexName, (DateTime.Now - start));
            }
Example #28
0
            //[MethodImpl(MethodImplOptions.Synchronized)]
            public void Run()
            {
                // TODO get rid of current and use the marker file instead?
                DateTime start = DateTime.Now;

                inProgress = true;
                if (directoryProviderLock == null)
                {
                    directoryProviderLock = parent.searchFactory.GetLockableDirectoryProviders()[directoryProvider];
                    directoryProvider     = null;
                }

                try
                {
                    lock (directoryProviderLock)
                    {
                        int           oldIndex        = parent.current;
                        int           index           = parent.current == 1 ? 2 : 1;
                        DirectoryInfo sourceFile      = new DirectoryInfo(source);
                        DirectoryInfo destinationFile = new DirectoryInfo(Path.Combine(destination, index.ToString()));

                        // TODO make smart a parameter
                        try
                        {
                            log.Info("Copying " + sourceFile + " into " + destinationFile);
                            FileHelper.Synchronize(sourceFile, destinationFile, true);
                            parent.current = index;
                        }
                        catch (IOException e)
                        {
                            // Don't change current
                            log.Error("Unable to synchronize source of " + parent.indexName, e);
                            return;
                        }

                        try
                        {
                            File.Delete(Path.Combine(destination, "current" + oldIndex));
                        }
                        catch (IOException e)
                        {
                            log.Warn("Unable to remove previous marker file from source of " + parent.indexName, e);
                        }

                        try
                        {
                            File.Create(Path.Combine(destination, "current" + index)).Dispose();
                        }
                        catch (IOException e)
                        {
                            log.Warn("Unable to create current marker in source of " + parent.indexName, e);
                        }
                    }
                }
                finally
                {
                    inProgress = false;
                }

                log.InfoFormat("Copy for {0} took {1}.", parent.indexName, (DateTime.Now - start));
            }
 public IOptimizerStrategy GetOptimizerStrategy(IDirectoryProvider provider)
 {
     return(dirProviderOptimizerStrategy[provider]);
 }
 public object GetLockObjForDirectoryProvider(IDirectoryProvider provider)
 {
     return lockableDirectoryProviders[provider];
 }
 public OutputFilesService(IFileProvider fileProvider, IDirectoryProvider directoryProvider, IPathProvider pathProvider)
 {
     _fileProvider      = fileProvider;
     _directoryProvider = directoryProvider;
     _pathProvider      = pathProvider;
 }
 public void RegisterDirectoryProviderForLocks(IDirectoryProvider provider)
 {
     if (lockableDirectoryProviders.ContainsKey(provider) == false)
     {
         lockableDirectoryProviders.Add(provider, new object());
     }
 }
Example #33
0
 public void AddOptimizerStrategy(IDirectoryProvider provider, IOptimizerStrategy optimizerStrategy)
 {
     dirProviderOptimizerStrategy[provider] = optimizerStrategy;
 }
        private static void ConfigureOptimizerStrategy(ISearchFactoryImplementor searchFactoryImplementor, IDictionary<string, string> indexProps, IDirectoryProvider provider)
        {
            bool incremental = indexProps.ContainsKey("optimizer.operation_limit.max") ||
                               indexProps.ContainsKey("optimizer.transaction_limit.max");

            IOptimizerStrategy optimizerStrategy;
            if (incremental)
            {
                optimizerStrategy = new IncrementalOptimizerStrategy();
                optimizerStrategy.Initialize(provider, indexProps, searchFactoryImplementor);
            }
            else
            {
                optimizerStrategy = new NoOpOptimizerStrategy();
            }

            searchFactoryImplementor.AddOptimizerStrategy(provider, optimizerStrategy);
        }
Example #35
0
        public IndexReader OpenReader(IDirectoryProvider[] directoryProviders)
        {
            bool trace  = log.IsInfoEnabled;
            int  length = directoryProviders.Length;

            IndexReader[] readers = new IndexReader[length];

            if (trace)
            {
                log.Info("Opening IndexReader for directoryProviders: " + length);
            }

            for (int index = 0; index < length; index++)
            {
                IDirectoryProvider directoryProvider = directoryProviders[index];
                IndexReader        reader;
                object             directoryProviderLock = perDirectoryProviderManipulationLocks[directoryProvider];
                if (trace)
                {
                    log.Info("Opening IndexReader from " + directoryProvider.Directory);
                }
                lock (directoryProviderLock)
                {
                    activeSearchIndexReaders.TryGetValue(directoryProvider, out reader);
                }
                if (reader == null)
                {
                    if (trace)
                    {
                        log.Info("No shared IndexReader, opening a new one: " + directoryProvider.Directory);
                    }
                    reader = ReplaceActiveReader(null, directoryProviderLock, directoryProvider, readers);
                }
                else
                {
                    bool isCurrent;
                    try
                    {
                        isCurrent = reader.IsCurrent();
                    }
                    catch (IOException e)
                    {
                        throw new SearchException("Unable to read current status of Lucene IndexReader", e);
                    }

                    if (!isCurrent)
                    {
                        if (trace)
                        {
                            log.Info("Out of date shared IndexReader found, opening a new one: " +
                                     directoryProvider.Directory);
                        }
                        IndexReader outOfDateReader = reader;
                        reader = ReplaceActiveReader(outOfDateReader, directoryProviderLock, directoryProvider, readers);
                    }
                    else
                    {
                        if (trace)
                        {
                            log.Info("Valid shared IndexReader: " + directoryProvider.Directory);
                        }

                        lock (directoryProviderLock)
                        {
                            //read the latest active one, the current one could be out of date and closed already
                            //the latest active is guaranteed to be active because it's protected by the dp lock
                            reader = activeSearchIndexReaders[directoryProvider];
                            lock (semaphoreIndexReaderLock)
                            {
                                ReaderData readerData = searchIndexReaderSemaphores[reader];
                                //TODO if readerData is null????
                                readerData.Semaphore++;
                                searchIndexReaderSemaphores[reader] = readerData; //not necessary
                                if (trace)
                                {
                                    log.Info("Semaphore increased: " + readerData.Semaphore + " for " + reader);
                                }
                            }
                        }
                    }
                }
                readers[index] = reader;
            }

            return(ReaderProviderHelper.BuildMultiReader(length, readers));
        }
 public TestClassGenerator(IDirectoryProvider directoryProvider, IAnalyzerManager analyzerManager)
     : base(directoryProvider, analyzerManager)
 {
 }
 public IOptimizerStrategy GetOptimizerStrategy(IDirectoryProvider provider)
 {
     return dirProviderOptimizerStrategy[provider];
 }
Example #38
0
 public ReaderData(int semaphore, IDirectoryProvider provider)
 {
     Semaphore = semaphore;
     Provider  = provider;
 }
 public DirectoryTraverser(string directory, IDirectoryProvider directoryProvider)
 {
     this.CurrentDirectory = directory;
     this.DirectoryProvider = directoryProvider;
 }
Example #40
0
        private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock,
                                                IDirectoryProvider directoryProvider, IndexReader[] readers)
        {
            bool        trace = log.IsInfoEnabled;
            IndexReader oldReader;
            bool        closeOldReader       = false;
            bool        closeOutOfDateReader = false;
            IndexReader reader;

            /**
             * Since out of lock protection, can have multiple readers created in //
             * not worse than NotShared and limit the locking time, hence scalability
             */
            try
            {
                // TODO:RB: check if reader should be writable
                reader = IndexReader.Open(directoryProvider.Directory, false);
            }
            catch (IOException e)
            {
                throw new SearchException("Unable to open Lucene IndexReader", e);
            }
            lock (directoryProviderLock)
            {
                // Since not protected by a lock, other ones can have been added
                oldReader = activeSearchIndexReaders[directoryProvider] = reader;
                lock (semaphoreIndexReaderLock)
                {
                    searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider);
                    if (trace)
                    {
                        log.Info("Semaphore: 1 for " + reader);
                    }
                    if (outOfDateReader != null)
                    {
                        ReaderData readerData;
                        searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData);
                        if (readerData == null)
                        {
                            closeOutOfDateReader = false; //already removed by another prevous thread
                        }
                        else if (readerData.Semaphore == 0)
                        {
                            searchIndexReaderSemaphores.Remove(outOfDateReader);
                            closeOutOfDateReader = true;
                        }
                        else
                        {
                            closeOutOfDateReader = false;
                        }
                    }

                    if (oldReader != null && oldReader != outOfDateReader)
                    {
                        ReaderData readerData = searchIndexReaderSemaphores[oldReader];
                        if (readerData == null)
                        {
                            log.Warn("Semaphore should not be null");
                            closeOldReader = true; //TODO should be true or false?
                        }
                        else if (readerData.Semaphore == 0)
                        {
                            searchIndexReaderSemaphores.Remove(oldReader);
                            closeOldReader = true;
                        }
                        else
                        {
                            closeOldReader = false;
                        }
                    }
                }
            }

            if (closeOutOfDateReader)
            {
                if (trace)
                {
                    log.Info("Closing out of date IndexReader " + outOfDateReader);
                }
                try
                {
                    outOfDateReader.Close();
                }
                catch (IOException e)
                {
                    ReaderProviderHelper.Clean(readers);
                    throw new SearchException("Unable to close Lucene IndexReader", e);
                }
            }
            if (closeOldReader)
            {
                if (trace)
                {
                    log.Info("Closing old IndexReader " + oldReader);
                }
                try
                {
                    oldReader.Close();
                }
                catch (IOException e)
                {
                    ReaderProviderHelper.Clean(readers);
                    throw new SearchException("Unable to close Lucene IndexReader", e);
                }
            }
            return(reader);
        }
        private static void ConfigureIndexingParameters(ISearchFactoryImplementor searchFactoryImplementor, IDictionary<string, string> indexProps, IDirectoryProvider provider)
        {
            LuceneIndexingParameters indexingParams = new LuceneIndexingParameters();

            ConfigureProp(
                    TRANSACTION + MERGE_FACTOR,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.MergeFactor = value;
                        indexingParams.TransactionIndexParameters.MergeFactor = value;
                    });

            ConfigureProp(
                    TRANSACTION + MAX_MERGE_DOCS,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.MaxMergeDocs = value;
                        indexingParams.TransactionIndexParameters.MaxMergeDocs = value;
                    });

            ConfigureProp(
                    TRANSACTION + MAX_BUFFERED_DOCS,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.MaxBufferedDocs = value;
                        indexingParams.TransactionIndexParameters.MaxBufferedDocs = value;
                    });

            ConfigureProp(
                    TRANSACTION + RAM_BUFFER_SIZE,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.RamBufferSizeMb = value;
                        indexingParams.TransactionIndexParameters.RamBufferSizeMb = value;
                    });

            ConfigureProp(
                    TRANSACTION + TERM_INDEX_INTERVAL,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.TermIndexInterval = value;
                        indexingParams.TransactionIndexParameters.TermIndexInterval = value;
                    });

            ConfigureProp(
                    BATCH + MERGE_FACTOR,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.MergeFactor = value;
                    });

            ConfigureProp(
                    BATCH + MAX_MERGE_DOCS,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.MaxMergeDocs = value;
                    });

            ConfigureProp(
                    BATCH + MAX_BUFFERED_DOCS,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.MaxBufferedDocs = value;
                    });

            ConfigureProp(
                    BATCH + RAM_BUFFER_SIZE,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.RamBufferSizeMb = value;
                    });

            ConfigureProp(
                    BATCH + TERM_INDEX_INTERVAL,
                    indexProps,
                    delegate(int value)
                    {
                        indexingParams.BatchIndexParameters.TermIndexInterval = value;
                    });

            searchFactoryImplementor.AddIndexingParameters(provider, indexingParams);
        }
Example #42
0
 public PluginService(IPathProvider pathProvider, IReflectionProvider reflectionProvider, IDirectoryProvider directoryProvider, IAssemblyProvider assemblyProvider)
 {
     _pathProvider       = pathProvider;
     _reflectionProvider = reflectionProvider;
     _directoryProvider  = directoryProvider;
     _assemblyProvider   = assemblyProvider;
 }
        public DirectoryProviders CreateDirectoryProviders(DocumentMapping classMapping, Configuration cfg,
                                                          ISearchFactoryImplementor searchFactoryImplementor)
        {
            // Get properties
            string directoryProviderName = GetDirectoryProviderName(classMapping, cfg);
            IDictionary<string, string>[] indexProps = GetDirectoryProperties(cfg, directoryProviderName);

            // Set up the directories
            int nbrOfProviders = indexProps.Length;
            IDirectoryProvider[] providers = new IDirectoryProvider[nbrOfProviders];
            for (int index = 0; index < nbrOfProviders; index++)
            {
                string providerName = nbrOfProviders > 1
                                          ? directoryProviderName + "." + index
                                          : directoryProviderName;

                // NB Are the properties nested??
                providers[index] = CreateDirectoryProvider(providerName, indexProps[index], searchFactoryImplementor);
            }

            // Define sharding strategy
            IIndexShardingStrategy shardingStrategy;
            IDictionary<string, string> shardingProperties = new Dictionary<string, string>();

            // Any indexProperty will do, the indexProps[0] surely exists.
            foreach (KeyValuePair<string, string> entry in indexProps[0])
            {
                if (entry.Key.StartsWith(SHARDING_STRATEGY))
                {
                    shardingProperties.Add(entry);
                }
            }

            string shardingStrategyName;
            shardingProperties.TryGetValue(SHARDING_STRATEGY, out shardingStrategyName);
            if (string.IsNullOrEmpty(shardingStrategyName))
            {
                if (indexProps.Length == 1)
                {
                    shardingStrategy = new NotShardedStrategy();
                }
                else
                {
                    shardingStrategy = new IdHashShardingStrategy();
                }
            }
            else
            {
                try
                {
                    System.Type shardingStrategyClass = ReflectHelper.ClassForName(shardingStrategyName);
                    shardingStrategy = (IIndexShardingStrategy) Activator.CreateInstance(shardingStrategyClass);
                }
                catch
                {
                    // TODO: See if we can get a tigher exception trap here
                    throw new SearchException("Failed to instantiate lucene analyzer with type  " + shardingStrategyName);
                }
            }

            shardingStrategy.Initialize(shardingProperties, providers);

            return new DirectoryProviders(shardingStrategy, providers);
        }
Example #44
0
        /// <summary>
        ///     Attempts to log the user in to the FTP Server
        /// </summary>
        /// <returns></returns>
        public async Task LoginAsync()
        {
            if (IsConnected)
            {
                await LogOutAsync();
            }

            string username = Configuration.Username.IsNullOrWhiteSpace()
                ? Constants.ANONYMOUS_USER
                : Configuration.Username;

            await ControlStream.ConnectAsync();

            if (ControlStream.LocalEndPoint is System.Net.IPEndPoint)
            {
                LocalEndPoint = ((System.Net.IPEndPoint)ControlStream.LocalEndPoint).Address;
            }

            var usrResponse = await ControlStream.SendCommandAsync(new FtpCommandEnvelope
            {
                FtpCommand = FtpCommand.USER,
                Data       = username
            });

            await BailIfResponseNotAsync(usrResponse, FtpStatusCode.SendUserCommand, FtpStatusCode.SendPasswordCommand, FtpStatusCode.LoggedInProceed);

            var pwd = Configuration.Password;

            if (IsAnonymousPasswordHandle)
            {
                pwd = username != Constants.ANONYMOUS_USER ? Configuration.Password : string.Empty;
            }
            var passResponse = await ControlStream.SendCommandAsync(new FtpCommandEnvelope
            {
                FtpCommand = FtpCommand.PASS,
                Data       = pwd
            });

            await BailIfResponseNotAsync(passResponse, FtpStatusCode.LoggedInProceed);

            IsAuthenticated = true;
            UsePassive      = Configuration.UsePassive;
            if (ControlStream.IsEncrypted)
            {
                await ControlStream.SendCommandAsync(new FtpCommandEnvelope
                {
                    FtpCommand = FtpCommand.PBSZ,
                    Data       = "0"
                });

                await ControlStream.SendCommandAsync(new FtpCommandEnvelope
                {
                    FtpCommand = FtpCommand.PROT,
                    Data       = "P"
                });
            }

            Features = await DetermineFeaturesAsync();

            directoryProvider = DetermineDirectoryProvider();
            await EnableUTF8IfPossible();
            await SetTransferMode(Configuration.Mode, Configuration.ModeSecondType);

            //if (Configuration.BaseDirectory != "/")
            //{
            //    await CreateDirectoryAsync(Configuration.BaseDirectory);
            //}

            //await ChangeWorkingDirectoryAsync(Configuration.BaseDirectory);
        }
 public void Initialize(IDirectoryProvider directoryProvider, IDictionary<string, string> indexProperties, ISearchFactoryImplementor searchFactoryImplementor)
 {
 }
        private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock,
            IDirectoryProvider directoryProvider, IndexReader[] readers)
        {
            bool trace = log.IsInfoEnabled;
            IndexReader oldReader;
            bool closeOldReader = false;
            bool closeOutOfDateReader = false;
            IndexReader reader;
            /**
             * Since out of lock protection, can have multiple readers created in //
             * not worse than NotShared and limit the locking time, hence scalability
             */
            try
            {
                reader = IndexReader.Open(directoryProvider.Directory);
            }
            catch (IOException e)
            {
                throw new SearchException("Unable to open Lucene IndexReader", e);
            }
            lock (directoryProviderLock)
            {
                // Since not protected by a lock, other ones can have been added
                oldReader = activeSearchIndexReaders[directoryProvider] = reader;
                lock (semaphoreIndexReaderLock)
                {
                    searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider);
                    if (trace) log.Info("Semaphore: 1 for " + reader);
                    if (outOfDateReader != null)
                    {
                        ReaderData readerData;
                        searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData);
                        if (readerData == null)
                        {
                            closeOutOfDateReader = false; //already removed by another prevous thread
                        }
                        else if (readerData.Semaphore == 0)
                        {
                            searchIndexReaderSemaphores.Remove(outOfDateReader);
                            closeOutOfDateReader = true;
                        }
                        else
                        {
                            closeOutOfDateReader = false;
                        }
                    }

                    if (oldReader != null && oldReader != outOfDateReader)
                    {
                        ReaderData readerData = searchIndexReaderSemaphores[oldReader];
                        if (readerData == null)
                        {
                            log.Warn("Semaphore should not be null");
                            closeOldReader = true; //TODO should be true or false?
                        }
                        else if (readerData.Semaphore == 0)
                        {
                            searchIndexReaderSemaphores.Remove(oldReader);
                            closeOldReader = true;
                        }
                        else
                        {
                            closeOldReader = false;
                        }
                    }
                }
            }

            if (closeOutOfDateReader)
            {
                if (trace) log.Info("Closing out of date IndexReader " + outOfDateReader);
                try
                {
                    outOfDateReader.Close();
                }
                catch (IOException e)
                {
                    ReaderProviderHelper.Clean(readers);
                    throw new SearchException("Unable to close Lucene IndexReader", e);
                }
            }
            if (closeOldReader)
            {
                if (trace) log.Info("Closing old IndexReader " + oldReader);
                try
                {
                    oldReader.Close();
                }
                catch (IOException e)
                {
                    ReaderProviderHelper.Clean(readers);
                    throw new SearchException("Unable to close Lucene IndexReader", e);
                }
            }
            return reader;
        }
 public CopyDirectory(FSMasterDirectoryProvider parent, string source, string destination, IDirectoryProvider directoryProvider)
 {
     this.parent = parent;
     this.source = source;
     this.destination = destination;
     this.directoryProvider = directoryProvider;
 }
Example #48
0
 public IndexWriter GetIndexWriter(IDirectoryProvider provider)
 {
     return GetIndexWriter(provider, null, false);
 }
 private void LockProvider(IDirectoryProvider provider)
 {
     // Make sure to use a semaphore
     object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
     Monitor.Enter(syncLock);
     try
     {
         if (!lockedProviders.Contains(provider))
         {
             lockedProviders.Add(provider);
             dpStatistics.Add(provider, new DPStatistics());
         }
     }
     catch (Exception)
     {
         // NB This is correct here, we release the lock *only* if we have an error
         Monitor.Exit(syncLock);
         throw;
     }
 }
Example #50
0
 public void Optimize(IDirectoryProvider provider)
 {
     IOptimizerStrategy optimizerStrategy = searchFactoryImplementor.GetOptimizerStrategy(provider);
     dpStatistics[provider].OptimizationForced = true;
     optimizerStrategy.OptimizationForced();
 }
Example #51
0
 public NonConnectedMailbox(IDirectoryProvider directory, DirectoryIdentity identity, IEnumerable <IPhysicalMailbox> physicalMailboxes) : base(directory, identity, physicalMailboxes, DirectoryMailboxType.Organization)
 {
 }
Example #52
0
 public InterfaceGenerator(IDirectoryProvider directoryProvider, IAnalyzerManager analyzerManager)
     : base(directoryProvider, analyzerManager)
 {
 }
        public IndexReader OpenReader(IDirectoryProvider[] directoryProviders)
        {
            bool trace = log.IsInfoEnabled;
            int length = directoryProviders.Length;
            IndexReader[] readers = new IndexReader[length];

            if (trace) log.Info("Opening IndexReader for directoryProviders: " + length);

            for (int index = 0; index < length; index++)
            {
                IDirectoryProvider directoryProvider = directoryProviders[index];
                IndexReader reader;
                object directoryProviderLock = perDirectoryProviderManipulationLocks[directoryProvider];
                if (trace) log.Info("Opening IndexReader from " + directoryProvider.Directory);
                lock (directoryProviderLock)
                {
                    activeSearchIndexReaders.TryGetValue(directoryProvider, out reader);
                }
                if (reader == null)
                {
                    if (trace)
                        log.Info("No shared IndexReader, opening a new one: " + directoryProvider.Directory);
                    reader = ReplaceActiveReader(null, directoryProviderLock, directoryProvider, readers);
                }
                else
                {
                    bool isCurrent;
                    try
                    {
                        isCurrent = reader.IsCurrent();
                    }
                    catch (IOException e)
                    {
                        throw new SearchException("Unable to read current status of Lucene IndexReader", e);
                    }

                    if (!isCurrent)
                    {
                        if (trace)
                            log.Info("Out of date shared IndexReader found, opening a new one: " +
                                     directoryProvider.Directory);
                        IndexReader outOfDateReader = reader;
                        reader = ReplaceActiveReader(outOfDateReader, directoryProviderLock, directoryProvider, readers);
                    }
                    else
                    {
                        if (trace)
                            log.Info("Valid shared IndexReader: " + directoryProvider.Directory);

                        lock (directoryProviderLock)
                        {
                            //read the latest active one, the current one could be out of date and closed already
                            //the latest active is guaranteed to be active because it's protected by the dp lock
                            reader = activeSearchIndexReaders[directoryProvider];
                            lock (semaphoreIndexReaderLock)
                            {
                                ReaderData readerData = searchIndexReaderSemaphores[reader];
                                //TODO if readerData is null????
                                readerData.Semaphore++;
                                searchIndexReaderSemaphores[reader] = readerData; //not necessary
                                if (trace) log.Info("Semaphore increased: " + readerData.Semaphore + " for " + reader);
                            }
                        }
                    }
                }
                readers[index] = reader;
            }

            return ReaderProviderHelper.BuildMultiReader(length, readers);
        }
 public LuceneIndexingParameters GetIndexingParameters(IDirectoryProvider provider)
 {
     return(dirProviderIndexingParams[provider]);
 }
 public ReaderData(int semaphore, IDirectoryProvider provider)
 {
     Semaphore = semaphore;
     Provider = provider;
 }
 public DirectoryTraverser(string directory, IDirectoryProvider directoryProvider)
 {
     this.CurrentDirectory  = directory;
     this.DirectoryProvider = directoryProvider;
 }
Example #57
0
        /// <summary>
        /// Retrieve a read/write <see cref="IndexWriter" />
        /// </summary>
        /// <param name="provider"></param>
        /// <param name="entity"></param>
        /// <param name="modificationOperation"></param>
        /// <returns></returns>
        public IndexWriter GetIndexWriter(IDirectoryProvider provider, System.Type entity, bool modificationOperation)
        {
            // Have to close the reader before the writer is accessed.
            IndexReader reader;
            readers.TryGetValue(provider, out reader);
            if (reader != null)
            {
                try
                {
                    reader.Close();
                }
                catch (IOException ex)
                {
                    throw new SearchException("Exception while closing IndexReader", ex);
                }
                finally
                {
                    readers.Remove(provider);

                    // PH - Moved the exit lock out of the try otherwise it won't take place when we have an error closing the reader.
                    // Exit Lock added by Kailuo Wang, because the lock needs to be obtained immediately afterwards
                    object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
                    Monitor.Exit(syncLock);
                }
            }

            if (writers.ContainsKey(provider))
            {
                return writers[provider];
            }

            LockProvider(provider);

            if (modificationOperation) dpStatistics[provider].Operations++;

            try
            {
                Analyzer analyzer = entity != null
                                        ? searchFactoryImplementor.DocumentBuilders[entity].Analyzer
                                        : new SimpleAnalyzer();
                IndexWriter writer = new IndexWriter(provider.Directory, analyzer, false);

                LuceneIndexingParameters indexingParams = searchFactoryImplementor.GetIndexingParameters(provider);
                if (IsBatch)
                {
                    writer.SetMergeFactor(indexingParams.BatchMergeFactor);
                    writer.SetMaxMergeDocs(indexingParams.BatchMaxMergeDocs);
                    writer.SetMaxBufferedDocs(indexingParams.BatchMaxBufferedDocs);
                }
                else
                {
                    writer.SetMergeFactor(indexingParams.TransactionMergeFactor);
                    writer.SetMaxMergeDocs(indexingParams.TransactionMaxMergeDocs);
                    writer.SetMaxBufferedDocs(indexingParams.TransactionMaxBufferedDocs);
                }

                writers.Add(provider, writer);

                return writer;
            }
            catch (IOException ex)
            {
                CleanUp(new SearchException("Unable to open IndexWriter" + (entity != null ? " for " + entity : ""), ex));
            }

            return null;
        }
Example #58
0
 public YearMonthHistoricalFoldersTraits(IDirectoryProvider directoryProvider)
 {
     _directoryProvider = directoryProvider;
 }
Example #59
0
 private void LockProvider(IDirectoryProvider provider)
 {
     //make sure to use a semaphore
     object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
     Monitor.Enter(syncLock);
     if (!lockedProviders.Contains(provider))
     {
         lockedProviders.Add(provider);
         dpStatistics.Add(provider, new DPStatistics());
     }
 }
 public DirectoryMailbox(IDirectoryProvider directory, DirectoryIdentity identity, IEnumerable <IPhysicalMailbox> physicalMailboxes, DirectoryMailboxType mailboxType = DirectoryMailboxType.Organization) : base(directory, identity)
 {
     this.physicalMailboxes = physicalMailboxes.ToList <IPhysicalMailbox>();
     this.MailboxType       = mailboxType;
 }