public IndexReader GetIndexReader(IDirectoryProvider provider, System.Type entity)
{
// one cannot access a reader for update after a writer has been accessed
if (writers.ContainsKey(provider))
{
throw new AssertionFailure("Tries to read for update a index while a writer is accessed" + entity);
}
IndexReader reader;
readers.TryGetValue(provider, out reader);
if (reader != null)
{
return reader;
}
LockProvider(provider);
dpStatistics[provider].Operations++;
try
{
reader = IndexReader.Open(provider.Directory);
readers.Add(provider, reader);
}
catch (IOException e)
{
CleanUp(new SearchException("Unable to open IndexReader for " + entity, e));
}
return reader;
}
public void Initialize(object properties, IDirectoryProvider[] providers)
{
directoryProvider = providers;
if (providers.GetUpperBound(0) > 1)
{
throw new AssertionFailure("Using SingleDirectoryProviderSelectionStrategy with multiple DirectoryProviders");
}
}
public void IdShardingStrategy()
{
IDirectoryProvider[] dps = new IDirectoryProvider[] {new RAMDirectoryProvider(), new RAMDirectoryProvider()};
IdHashShardingStrategy shardingStrategy = new IdHashShardingStrategy();
shardingStrategy.Initialize(null, dps);
Assert.AreSame(dps[1], shardingStrategy.GetDirectoryProviderForAddition(typeof(Animal), 1, "1", null));
Assert.AreSame(dps[0], shardingStrategy.GetDirectoryProviderForAddition(typeof(Animal), 2, "2", null));
}
public void Initialize(object properties, IDirectoryProvider[] providers)
{
if (providers == null)
{
throw new ArgumentNullException("providers");
}
this.providers = providers;
}
/// <summary>
/// Initializes a new instance of the <see cref="FileSystemInterface"/> class.
/// </summary>
/// <param name="config"></param>
/// <param name="directoryProvider"></param>
public FileSystemInterface(ConfigHandling config, IDirectoryProvider directoryProvider)
{
_config = config;
_directoryProvider = directoryProvider;
SearchList = new List<string> { "No", "Value" };
//// Filling the Directory and search datatable
this.UpdateDirectoryList(directoryProvider);
}
public void Initialize(IDirectoryProvider directoryProvider, IDictionary<string, string> indexProperties,
ISearchFactoryImplementor searchFactoryImplementor)
{
this.directoryProvider = directoryProvider;
string maxString;
indexProperties.TryGetValue("optimizer.operation_limit.max", out maxString);
if (!string.IsNullOrEmpty(maxString))
int.TryParse(maxString, out operationMax);
indexProperties.TryGetValue("optimizer.transaction_limit.max", out maxString);
if (!string.IsNullOrEmpty(maxString))
int.TryParse(maxString, out transactionMax);
}
public void PerformWork(PurgeAllLuceneWork work, IDirectoryProvider provider)
{
System.Type entity = work.EntityClass;
if (log.IsDebugEnabled)
log.Debug("PurgeAll Lucene index: " + entity);
IndexReader reader = workspace.GetIndexReader(provider, entity);
try
{
Term term = new Term(DocumentBuilder.CLASS_FIELDNAME, TypeHelper.LuceneTypeName(entity));
reader.DeleteDocuments(term);
}
catch (Exception e)
{
throw new SearchException("Unable to purge all from Lucene index: " + entity, e);
}
}
public void PerformWork(OptimizeLuceneWork work, IDirectoryProvider provider)
{
System.Type entity = work.EntityClass;
if (log.IsDebugEnabled)
log.Debug("Optimize Lucene index: " + entity);
IndexWriter writer = workspace.GetIndexWriter(provider, entity, false);
try
{
writer.Optimize();
workspace.Optimize(provider);
}
catch (IOException e)
{
throw new SearchException("Unable to optimize Lucene index: " + entity, e);
}
}
public IndexReader OpenReader(IDirectoryProvider[] directoryProviders)
{
int length = directoryProviders.Length;
IndexReader[] readers = new IndexReader[length];
try
{
for (int index = 0; index < length; index++)
readers[index] = IndexReader.Open(directoryProviders[index].Directory);
}
catch (System.IO.IOException ex)
{
// TODO: more contextual info
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to open one of the Lucene indexes", ex);
}
return ReaderProviderHelper.BuildMultiReader(length, readers);
}
public DocumentBuilder(DocumentMapping classMapping, Analyzer defaultAnalyzer, IDirectoryProvider[] directoryProviders,
IIndexShardingStrategy shardingStrategy)
{
analyzer = new ScopedAnalyzer();
this.directoryProviders = directoryProviders;
this.shardingStrategy = shardingStrategy;
if (classMapping == null) throw new AssertionFailure("Unable to build a DocumemntBuilder with a null class");
rootClassMapping = classMapping;
Set<System.Type> processedClasses = new HashedSet<System.Type>();
processedClasses.Add(classMapping.MappedClass);
CollectAnalyzers(rootClassMapping, defaultAnalyzer, true, string.Empty, processedClasses);
//processedClasses.remove( clazz ); for the sake of completness
analyzer.GlobalAnalyzer = defaultAnalyzer;
if (idMapping == null)
throw new SearchException("No document id for: " + classMapping.MappedClass.Name);
}
public DocumentBuilder(System.Type clazz, Analyzer defaultAnalyzer, IDirectoryProvider[] directoryProviders,
IIndexShardingStrategy shardingStrategy)
{
analyzer = new ScopedAnalyzer();
beanClass = clazz;
this.directoryProviders = directoryProviders;
this.shardingStrategy = shardingStrategy;
if (clazz == null) throw new AssertionFailure("Unable to build a DocumemntBuilder with a null class");
rootPropertiesMetadata = new PropertiesMetadata();
rootPropertiesMetadata.boost = GetBoost(clazz);
rootPropertiesMetadata.analyzer = defaultAnalyzer;
Set<System.Type> processedClasses = new HashedSet<System.Type>();
processedClasses.Add(clazz);
InitializeMembers(clazz, rootPropertiesMetadata, true, string.Empty, processedClasses);
//processedClasses.remove( clazz ); for the sake of completness
analyzer.GlobalAnalyzer = rootPropertiesMetadata.analyzer;
if (idKeywordName == null)
throw new SearchException("No document id for: " + clazz.Name);
}
public object GetLockObjForDirectoryProvider(IDirectoryProvider provider)
{
return(lockableDirectoryProviders[provider]);
}
public DirectoryProviders(IIndexShardingStrategy shardingStrategy, IDirectoryProvider[] providers)
{
this.shardingStrategy = shardingStrategy;
this.providers = providers;
}
public ItemCountCheck(SoftDeletedRemovalData data, IDirectoryProvider directory) : base(data, directory)
{
}
public DirectoryForest(IDirectoryProvider directory, DirectoryIdentity identity) : base(directory, identity)
{
}
public DisconnectDateCheck(SoftDeletedRemovalData data, IDirectoryProvider directory, DateTime removalCutoffDate) : base(data, directory)
{
this.removalCutoffDate = removalCutoffDate;
}
protected DirectoryContainerParent(IDirectoryProvider directory, DirectoryIdentity identity) : base(directory, identity)
{
}
protected virtual ILoadBalanceService CreateLoadBalancerClient(DirectoryServer server, IDirectoryProvider directory)
{
ILoadBalanceService result;
using (DisposeGuard disposeGuard = default(DisposeGuard))
{
LoadBalancerClient loadBalancerClient = LoadBalancerClient.Create(server.Fqdn, directory, this.logger);
disposeGuard.Add <LoadBalancerClient>(loadBalancerClient);
bool flag = true;
ILoadBalanceService loadBalanceService = loadBalancerClient;
if (!loadBalancerClient.ServerVersion[1])
{
flag = false;
loadBalanceService = this.CreateCompatibilityLoadBalanceClient(server);
}
else if (!loadBalancerClient.ServerVersion[2])
{
loadBalanceService = new SoftDeletedRemovalCapabilityDecorator(loadBalanceService, server);
}
if (!loadBalancerClient.ServerVersion[3])
{
loadBalanceService = new ConsumerMetricsLoadBalanceCapabilityDecorator(loadBalanceService, server);
}
if (!loadBalancerClient.ServerVersion[5])
{
loadBalanceService = new CapacitySummaryCapabilityDecorator(loadBalanceService, server, this.serviceContext);
}
if (flag)
{
disposeGuard.Success();
}
result = loadBalanceService;
}
return(result);
}
public SettingsProvider(ILogger <SettingsProvider> logger, IDirectoryProvider directoryProvider)
{
this.logger = logger;
this.directoryProvider = directoryProvider;
LoadOrCreateSettings();
}
public InjectorService(IDirectoryProvider directory, ILogger logger, MoveInjector moveInjector) : base(logger)
{
this.directory = directory;
this.moveInjector = moveInjector;
}
protected virtual IInjectorService CreateInjectorClient(DirectoryServer server, IDirectoryProvider directory)
{
IInjectorService result;
using (DisposeGuard disposeGuard = default(DisposeGuard))
{
InjectorClient injectorClient = InjectorClient.Create(server.Fqdn, directory, this.logger);
disposeGuard.Add <InjectorClient>(injectorClient);
bool flag = true;
IInjectorService injectorService = injectorClient;
if (!injectorClient.ServerVersion[1])
{
flag = false;
injectorService = this.CreateCompatibilityInjectorClient(server);
}
if (!injectorClient.ServerVersion[2])
{
injectorService = new ConsumerMetricsInjectorCapabilityDecorator(injectorService);
}
if (flag)
{
disposeGuard.Success();
}
result = injectorService;
}
return(result);
}
public void PerformWork(DeleteLuceneWork work, IDirectoryProvider provider)
{
Remove(work.EntityClass, work.Id, provider);
}
public void PerformWork(AddLuceneWork work, IDirectoryProvider provider)
{
Add(work.EntityClass, work.Id, work.Document, provider);
}
public WorkWithPayload(LuceneWork work, IDirectoryProvider provider)
{
this.work = work;
this.provider = provider;
}
public void AddIndexingParameters(IDirectoryProvider provider, LuceneIndexingParameters indexingParameters)
{
dirProviderIndexingParams[provider] = indexingParameters;
}
public CopyDirectory(FSMasterDirectoryProvider parent, string source, string destination, IDirectoryProvider directoryProvider)
{
this.parent = parent;
this.source = source;
this.destination = destination;
this.directoryProvider = directoryProvider;
}
public void Run()
{
//TODO get rid of current and use the marker file instead?
DateTime start = DateTime.Now;
inProgress = true;
if (directoryProviderLock == null)
{
directoryProviderLock = parent.searchFactory.GetLockableDirectoryProviders()[directoryProvider];
directoryProvider = null;
}
try
{
lock (directoryProviderLock)
{
int oldIndex = parent.current;
int index = parent.current == 1 ? 2 : 1;
DirectoryInfo sourceFile = new DirectoryInfo(source);
DirectoryInfo destinationFile = new DirectoryInfo(Path.Combine(destination, index.ToString()));
//TODO make smart a parameter
try
{
log.Info("Copying " + sourceFile + " into " + destinationFile);
FileHelper.Synchronize(sourceFile, destinationFile, true);
parent.current = index;
}
catch (IOException e)
{
//don't change current
log.Error("Unable to synchronize source of " + parent.indexName, e);
return;
}
try
{
File.Delete(Path.Combine(destination, "current" + oldIndex));
}
catch (IOException e)
{
log.Warn("Unable to remove previous marker file from source of " + parent.indexName, e);
}
try
{
File.Create(Path.Combine(destination, "current" + index)).Dispose();
}
catch (IOException e)
{
log.Warn("Unable to create current marker in source of " + parent.indexName, e);
}
}
}
finally
{
inProgress = false;
}
log.InfoFormat("Copy for {0} took {1}.", parent.indexName, (DateTime.Now - start));
}
//[MethodImpl(MethodImplOptions.Synchronized)]
public void Run()
{
// TODO get rid of current and use the marker file instead?
DateTime start = DateTime.Now;
inProgress = true;
if (directoryProviderLock == null)
{
directoryProviderLock = parent.searchFactory.GetLockableDirectoryProviders()[directoryProvider];
directoryProvider = null;
}
try
{
lock (directoryProviderLock)
{
int oldIndex = parent.current;
int index = parent.current == 1 ? 2 : 1;
DirectoryInfo sourceFile = new DirectoryInfo(source);
DirectoryInfo destinationFile = new DirectoryInfo(Path.Combine(destination, index.ToString()));
// TODO make smart a parameter
try
{
log.Info("Copying " + sourceFile + " into " + destinationFile);
FileHelper.Synchronize(sourceFile, destinationFile, true);
parent.current = index;
}
catch (IOException e)
{
// Don't change current
log.Error("Unable to synchronize source of " + parent.indexName, e);
return;
}
try
{
File.Delete(Path.Combine(destination, "current" + oldIndex));
}
catch (IOException e)
{
log.Warn("Unable to remove previous marker file from source of " + parent.indexName, e);
}
try
{
File.Create(Path.Combine(destination, "current" + index)).Dispose();
}
catch (IOException e)
{
log.Warn("Unable to create current marker in source of " + parent.indexName, e);
}
}
}
finally
{
inProgress = false;
}
log.InfoFormat("Copy for {0} took {1}.", parent.indexName, (DateTime.Now - start));
}
public IOptimizerStrategy GetOptimizerStrategy(IDirectoryProvider provider)
{
return(dirProviderOptimizerStrategy[provider]);
}
public object GetLockObjForDirectoryProvider(IDirectoryProvider provider)
{
return lockableDirectoryProviders[provider];
}
public OutputFilesService(IFileProvider fileProvider, IDirectoryProvider directoryProvider, IPathProvider pathProvider)
{
_fileProvider = fileProvider;
_directoryProvider = directoryProvider;
_pathProvider = pathProvider;
}
public void RegisterDirectoryProviderForLocks(IDirectoryProvider provider)
{
if (lockableDirectoryProviders.ContainsKey(provider) == false)
{
lockableDirectoryProviders.Add(provider, new object());
}
}
public void AddOptimizerStrategy(IDirectoryProvider provider, IOptimizerStrategy optimizerStrategy)
{
dirProviderOptimizerStrategy[provider] = optimizerStrategy;
}
private static void ConfigureOptimizerStrategy(ISearchFactoryImplementor searchFactoryImplementor, IDictionary<string, string> indexProps, IDirectoryProvider provider)
{
bool incremental = indexProps.ContainsKey("optimizer.operation_limit.max") ||
indexProps.ContainsKey("optimizer.transaction_limit.max");
IOptimizerStrategy optimizerStrategy;
if (incremental)
{
optimizerStrategy = new IncrementalOptimizerStrategy();
optimizerStrategy.Initialize(provider, indexProps, searchFactoryImplementor);
}
else
{
optimizerStrategy = new NoOpOptimizerStrategy();
}
searchFactoryImplementor.AddOptimizerStrategy(provider, optimizerStrategy);
}
public IndexReader OpenReader(IDirectoryProvider[] directoryProviders)
{
bool trace = log.IsInfoEnabled;
int length = directoryProviders.Length;
IndexReader[] readers = new IndexReader[length];
if (trace)
{
log.Info("Opening IndexReader for directoryProviders: " + length);
}
for (int index = 0; index < length; index++)
{
IDirectoryProvider directoryProvider = directoryProviders[index];
IndexReader reader;
object directoryProviderLock = perDirectoryProviderManipulationLocks[directoryProvider];
if (trace)
{
log.Info("Opening IndexReader from " + directoryProvider.Directory);
}
lock (directoryProviderLock)
{
activeSearchIndexReaders.TryGetValue(directoryProvider, out reader);
}
if (reader == null)
{
if (trace)
{
log.Info("No shared IndexReader, opening a new one: " + directoryProvider.Directory);
}
reader = ReplaceActiveReader(null, directoryProviderLock, directoryProvider, readers);
}
else
{
bool isCurrent;
try
{
isCurrent = reader.IsCurrent();
}
catch (IOException e)
{
throw new SearchException("Unable to read current status of Lucene IndexReader", e);
}
if (!isCurrent)
{
if (trace)
{
log.Info("Out of date shared IndexReader found, opening a new one: " +
directoryProvider.Directory);
}
IndexReader outOfDateReader = reader;
reader = ReplaceActiveReader(outOfDateReader, directoryProviderLock, directoryProvider, readers);
}
else
{
if (trace)
{
log.Info("Valid shared IndexReader: " + directoryProvider.Directory);
}
lock (directoryProviderLock)
{
//read the latest active one, the current one could be out of date and closed already
//the latest active is guaranteed to be active because it's protected by the dp lock
reader = activeSearchIndexReaders[directoryProvider];
lock (semaphoreIndexReaderLock)
{
ReaderData readerData = searchIndexReaderSemaphores[reader];
//TODO if readerData is null????
readerData.Semaphore++;
searchIndexReaderSemaphores[reader] = readerData; //not necessary
if (trace)
{
log.Info("Semaphore increased: " + readerData.Semaphore + " for " + reader);
}
}
}
}
}
readers[index] = reader;
}
return(ReaderProviderHelper.BuildMultiReader(length, readers));
}
public TestClassGenerator(IDirectoryProvider directoryProvider, IAnalyzerManager analyzerManager)
: base(directoryProvider, analyzerManager)
{
}
public IOptimizerStrategy GetOptimizerStrategy(IDirectoryProvider provider)
{
return dirProviderOptimizerStrategy[provider];
}
public ReaderData(int semaphore, IDirectoryProvider provider)
{
Semaphore = semaphore;
Provider = provider;
}
public DirectoryTraverser(string directory, IDirectoryProvider directoryProvider)
{
this.CurrentDirectory = directory;
this.DirectoryProvider = directoryProvider;
}
private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock,
IDirectoryProvider directoryProvider, IndexReader[] readers)
{
bool trace = log.IsInfoEnabled;
IndexReader oldReader;
bool closeOldReader = false;
bool closeOutOfDateReader = false;
IndexReader reader;
/**
* Since out of lock protection, can have multiple readers created in //
* not worse than NotShared and limit the locking time, hence scalability
*/
try
{
// TODO:RB: check if reader should be writable
reader = IndexReader.Open(directoryProvider.Directory, false);
}
catch (IOException e)
{
throw new SearchException("Unable to open Lucene IndexReader", e);
}
lock (directoryProviderLock)
{
// Since not protected by a lock, other ones can have been added
oldReader = activeSearchIndexReaders[directoryProvider] = reader;
lock (semaphoreIndexReaderLock)
{
searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider);
if (trace)
{
log.Info("Semaphore: 1 for " + reader);
}
if (outOfDateReader != null)
{
ReaderData readerData;
searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData);
if (readerData == null)
{
closeOutOfDateReader = false; //already removed by another prevous thread
}
else if (readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(outOfDateReader);
closeOutOfDateReader = true;
}
else
{
closeOutOfDateReader = false;
}
}
if (oldReader != null && oldReader != outOfDateReader)
{
ReaderData readerData = searchIndexReaderSemaphores[oldReader];
if (readerData == null)
{
log.Warn("Semaphore should not be null");
closeOldReader = true; //TODO should be true or false?
}
else if (readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(oldReader);
closeOldReader = true;
}
else
{
closeOldReader = false;
}
}
}
}
if (closeOutOfDateReader)
{
if (trace)
{
log.Info("Closing out of date IndexReader " + outOfDateReader);
}
try
{
outOfDateReader.Close();
}
catch (IOException e)
{
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to close Lucene IndexReader", e);
}
}
if (closeOldReader)
{
if (trace)
{
log.Info("Closing old IndexReader " + oldReader);
}
try
{
oldReader.Close();
}
catch (IOException e)
{
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to close Lucene IndexReader", e);
}
}
return(reader);
}
private static void ConfigureIndexingParameters(ISearchFactoryImplementor searchFactoryImplementor, IDictionary<string, string> indexProps, IDirectoryProvider provider)
{
LuceneIndexingParameters indexingParams = new LuceneIndexingParameters();
ConfigureProp(
TRANSACTION + MERGE_FACTOR,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.MergeFactor = value;
indexingParams.TransactionIndexParameters.MergeFactor = value;
});
ConfigureProp(
TRANSACTION + MAX_MERGE_DOCS,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.MaxMergeDocs = value;
indexingParams.TransactionIndexParameters.MaxMergeDocs = value;
});
ConfigureProp(
TRANSACTION + MAX_BUFFERED_DOCS,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.MaxBufferedDocs = value;
indexingParams.TransactionIndexParameters.MaxBufferedDocs = value;
});
ConfigureProp(
TRANSACTION + RAM_BUFFER_SIZE,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.RamBufferSizeMb = value;
indexingParams.TransactionIndexParameters.RamBufferSizeMb = value;
});
ConfigureProp(
TRANSACTION + TERM_INDEX_INTERVAL,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.TermIndexInterval = value;
indexingParams.TransactionIndexParameters.TermIndexInterval = value;
});
ConfigureProp(
BATCH + MERGE_FACTOR,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.MergeFactor = value;
});
ConfigureProp(
BATCH + MAX_MERGE_DOCS,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.MaxMergeDocs = value;
});
ConfigureProp(
BATCH + MAX_BUFFERED_DOCS,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.MaxBufferedDocs = value;
});
ConfigureProp(
BATCH + RAM_BUFFER_SIZE,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.RamBufferSizeMb = value;
});
ConfigureProp(
BATCH + TERM_INDEX_INTERVAL,
indexProps,
delegate(int value)
{
indexingParams.BatchIndexParameters.TermIndexInterval = value;
});
searchFactoryImplementor.AddIndexingParameters(provider, indexingParams);
}
public PluginService(IPathProvider pathProvider, IReflectionProvider reflectionProvider, IDirectoryProvider directoryProvider, IAssemblyProvider assemblyProvider)
{
_pathProvider = pathProvider;
_reflectionProvider = reflectionProvider;
_directoryProvider = directoryProvider;
_assemblyProvider = assemblyProvider;
}
public DirectoryProviders CreateDirectoryProviders(DocumentMapping classMapping, Configuration cfg,
ISearchFactoryImplementor searchFactoryImplementor)
{
// Get properties
string directoryProviderName = GetDirectoryProviderName(classMapping, cfg);
IDictionary<string, string>[] indexProps = GetDirectoryProperties(cfg, directoryProviderName);
// Set up the directories
int nbrOfProviders = indexProps.Length;
IDirectoryProvider[] providers = new IDirectoryProvider[nbrOfProviders];
for (int index = 0; index < nbrOfProviders; index++)
{
string providerName = nbrOfProviders > 1
? directoryProviderName + "." + index
: directoryProviderName;
// NB Are the properties nested??
providers[index] = CreateDirectoryProvider(providerName, indexProps[index], searchFactoryImplementor);
}
// Define sharding strategy
IIndexShardingStrategy shardingStrategy;
IDictionary<string, string> shardingProperties = new Dictionary<string, string>();
// Any indexProperty will do, the indexProps[0] surely exists.
foreach (KeyValuePair<string, string> entry in indexProps[0])
{
if (entry.Key.StartsWith(SHARDING_STRATEGY))
{
shardingProperties.Add(entry);
}
}
string shardingStrategyName;
shardingProperties.TryGetValue(SHARDING_STRATEGY, out shardingStrategyName);
if (string.IsNullOrEmpty(shardingStrategyName))
{
if (indexProps.Length == 1)
{
shardingStrategy = new NotShardedStrategy();
}
else
{
shardingStrategy = new IdHashShardingStrategy();
}
}
else
{
try
{
System.Type shardingStrategyClass = ReflectHelper.ClassForName(shardingStrategyName);
shardingStrategy = (IIndexShardingStrategy) Activator.CreateInstance(shardingStrategyClass);
}
catch
{
// TODO: See if we can get a tigher exception trap here
throw new SearchException("Failed to instantiate lucene analyzer with type " + shardingStrategyName);
}
}
shardingStrategy.Initialize(shardingProperties, providers);
return new DirectoryProviders(shardingStrategy, providers);
}
/// <summary>
/// Attempts to log the user in to the FTP Server
/// </summary>
/// <returns></returns>
public async Task LoginAsync()
{
if (IsConnected)
{
await LogOutAsync();
}
string username = Configuration.Username.IsNullOrWhiteSpace()
? Constants.ANONYMOUS_USER
: Configuration.Username;
await ControlStream.ConnectAsync();
if (ControlStream.LocalEndPoint is System.Net.IPEndPoint)
{
LocalEndPoint = ((System.Net.IPEndPoint)ControlStream.LocalEndPoint).Address;
}
var usrResponse = await ControlStream.SendCommandAsync(new FtpCommandEnvelope
{
FtpCommand = FtpCommand.USER,
Data = username
});
await BailIfResponseNotAsync(usrResponse, FtpStatusCode.SendUserCommand, FtpStatusCode.SendPasswordCommand, FtpStatusCode.LoggedInProceed);
var pwd = Configuration.Password;
if (IsAnonymousPasswordHandle)
{
pwd = username != Constants.ANONYMOUS_USER ? Configuration.Password : string.Empty;
}
var passResponse = await ControlStream.SendCommandAsync(new FtpCommandEnvelope
{
FtpCommand = FtpCommand.PASS,
Data = pwd
});
await BailIfResponseNotAsync(passResponse, FtpStatusCode.LoggedInProceed);
IsAuthenticated = true;
UsePassive = Configuration.UsePassive;
if (ControlStream.IsEncrypted)
{
await ControlStream.SendCommandAsync(new FtpCommandEnvelope
{
FtpCommand = FtpCommand.PBSZ,
Data = "0"
});
await ControlStream.SendCommandAsync(new FtpCommandEnvelope
{
FtpCommand = FtpCommand.PROT,
Data = "P"
});
}
Features = await DetermineFeaturesAsync();
directoryProvider = DetermineDirectoryProvider();
await EnableUTF8IfPossible();
await SetTransferMode(Configuration.Mode, Configuration.ModeSecondType);
//if (Configuration.BaseDirectory != "/")
//{
// await CreateDirectoryAsync(Configuration.BaseDirectory);
//}
//await ChangeWorkingDirectoryAsync(Configuration.BaseDirectory);
}
public void Initialize(IDirectoryProvider directoryProvider, IDictionary<string, string> indexProperties, ISearchFactoryImplementor searchFactoryImplementor)
{
}
private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock,
IDirectoryProvider directoryProvider, IndexReader[] readers)
{
bool trace = log.IsInfoEnabled;
IndexReader oldReader;
bool closeOldReader = false;
bool closeOutOfDateReader = false;
IndexReader reader;
/**
* Since out of lock protection, can have multiple readers created in //
* not worse than NotShared and limit the locking time, hence scalability
*/
try
{
reader = IndexReader.Open(directoryProvider.Directory);
}
catch (IOException e)
{
throw new SearchException("Unable to open Lucene IndexReader", e);
}
lock (directoryProviderLock)
{
// Since not protected by a lock, other ones can have been added
oldReader = activeSearchIndexReaders[directoryProvider] = reader;
lock (semaphoreIndexReaderLock)
{
searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider);
if (trace) log.Info("Semaphore: 1 for " + reader);
if (outOfDateReader != null)
{
ReaderData readerData;
searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData);
if (readerData == null)
{
closeOutOfDateReader = false; //already removed by another prevous thread
}
else if (readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(outOfDateReader);
closeOutOfDateReader = true;
}
else
{
closeOutOfDateReader = false;
}
}
if (oldReader != null && oldReader != outOfDateReader)
{
ReaderData readerData = searchIndexReaderSemaphores[oldReader];
if (readerData == null)
{
log.Warn("Semaphore should not be null");
closeOldReader = true; //TODO should be true or false?
}
else if (readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(oldReader);
closeOldReader = true;
}
else
{
closeOldReader = false;
}
}
}
}
if (closeOutOfDateReader)
{
if (trace) log.Info("Closing out of date IndexReader " + outOfDateReader);
try
{
outOfDateReader.Close();
}
catch (IOException e)
{
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to close Lucene IndexReader", e);
}
}
if (closeOldReader)
{
if (trace) log.Info("Closing old IndexReader " + oldReader);
try
{
oldReader.Close();
}
catch (IOException e)
{
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to close Lucene IndexReader", e);
}
}
return reader;
}
public CopyDirectory(FSMasterDirectoryProvider parent, string source, string destination, IDirectoryProvider directoryProvider)
{
this.parent = parent;
this.source = source;
this.destination = destination;
this.directoryProvider = directoryProvider;
}
public IndexWriter GetIndexWriter(IDirectoryProvider provider)
{
return GetIndexWriter(provider, null, false);
}
private void LockProvider(IDirectoryProvider provider)
{
// Make sure to use a semaphore
object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
Monitor.Enter(syncLock);
try
{
if (!lockedProviders.Contains(provider))
{
lockedProviders.Add(provider);
dpStatistics.Add(provider, new DPStatistics());
}
}
catch (Exception)
{
// NB This is correct here, we release the lock *only* if we have an error
Monitor.Exit(syncLock);
throw;
}
}
public void Optimize(IDirectoryProvider provider)
{
IOptimizerStrategy optimizerStrategy = searchFactoryImplementor.GetOptimizerStrategy(provider);
dpStatistics[provider].OptimizationForced = true;
optimizerStrategy.OptimizationForced();
}
public NonConnectedMailbox(IDirectoryProvider directory, DirectoryIdentity identity, IEnumerable <IPhysicalMailbox> physicalMailboxes) : base(directory, identity, physicalMailboxes, DirectoryMailboxType.Organization)
{
}
public InterfaceGenerator(IDirectoryProvider directoryProvider, IAnalyzerManager analyzerManager)
: base(directoryProvider, analyzerManager)
{
}
public IndexReader OpenReader(IDirectoryProvider[] directoryProviders)
{
bool trace = log.IsInfoEnabled;
int length = directoryProviders.Length;
IndexReader[] readers = new IndexReader[length];
if (trace) log.Info("Opening IndexReader for directoryProviders: " + length);
for (int index = 0; index < length; index++)
{
IDirectoryProvider directoryProvider = directoryProviders[index];
IndexReader reader;
object directoryProviderLock = perDirectoryProviderManipulationLocks[directoryProvider];
if (trace) log.Info("Opening IndexReader from " + directoryProvider.Directory);
lock (directoryProviderLock)
{
activeSearchIndexReaders.TryGetValue(directoryProvider, out reader);
}
if (reader == null)
{
if (trace)
log.Info("No shared IndexReader, opening a new one: " + directoryProvider.Directory);
reader = ReplaceActiveReader(null, directoryProviderLock, directoryProvider, readers);
}
else
{
bool isCurrent;
try
{
isCurrent = reader.IsCurrent();
}
catch (IOException e)
{
throw new SearchException("Unable to read current status of Lucene IndexReader", e);
}
if (!isCurrent)
{
if (trace)
log.Info("Out of date shared IndexReader found, opening a new one: " +
directoryProvider.Directory);
IndexReader outOfDateReader = reader;
reader = ReplaceActiveReader(outOfDateReader, directoryProviderLock, directoryProvider, readers);
}
else
{
if (trace)
log.Info("Valid shared IndexReader: " + directoryProvider.Directory);
lock (directoryProviderLock)
{
//read the latest active one, the current one could be out of date and closed already
//the latest active is guaranteed to be active because it's protected by the dp lock
reader = activeSearchIndexReaders[directoryProvider];
lock (semaphoreIndexReaderLock)
{
ReaderData readerData = searchIndexReaderSemaphores[reader];
//TODO if readerData is null????
readerData.Semaphore++;
searchIndexReaderSemaphores[reader] = readerData; //not necessary
if (trace) log.Info("Semaphore increased: " + readerData.Semaphore + " for " + reader);
}
}
}
}
readers[index] = reader;
}
return ReaderProviderHelper.BuildMultiReader(length, readers);
}
public LuceneIndexingParameters GetIndexingParameters(IDirectoryProvider provider)
{
return(dirProviderIndexingParams[provider]);
}
public ReaderData(int semaphore, IDirectoryProvider provider)
{
Semaphore = semaphore;
Provider = provider;
}
public DirectoryTraverser(string directory, IDirectoryProvider directoryProvider)
{
this.CurrentDirectory = directory;
this.DirectoryProvider = directoryProvider;
}
/// <summary>
/// Retrieve a read/write <see cref="IndexWriter" />
/// </summary>
/// <param name="provider"></param>
/// <param name="entity"></param>
/// <param name="modificationOperation"></param>
/// <returns></returns>
public IndexWriter GetIndexWriter(IDirectoryProvider provider, System.Type entity, bool modificationOperation)
{
// Have to close the reader before the writer is accessed.
IndexReader reader;
readers.TryGetValue(provider, out reader);
if (reader != null)
{
try
{
reader.Close();
}
catch (IOException ex)
{
throw new SearchException("Exception while closing IndexReader", ex);
}
finally
{
readers.Remove(provider);
// PH - Moved the exit lock out of the try otherwise it won't take place when we have an error closing the reader.
// Exit Lock added by Kailuo Wang, because the lock needs to be obtained immediately afterwards
object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
Monitor.Exit(syncLock);
}
}
if (writers.ContainsKey(provider))
{
return writers[provider];
}
LockProvider(provider);
if (modificationOperation) dpStatistics[provider].Operations++;
try
{
Analyzer analyzer = entity != null
? searchFactoryImplementor.DocumentBuilders[entity].Analyzer
: new SimpleAnalyzer();
IndexWriter writer = new IndexWriter(provider.Directory, analyzer, false);
LuceneIndexingParameters indexingParams = searchFactoryImplementor.GetIndexingParameters(provider);
if (IsBatch)
{
writer.SetMergeFactor(indexingParams.BatchMergeFactor);
writer.SetMaxMergeDocs(indexingParams.BatchMaxMergeDocs);
writer.SetMaxBufferedDocs(indexingParams.BatchMaxBufferedDocs);
}
else
{
writer.SetMergeFactor(indexingParams.TransactionMergeFactor);
writer.SetMaxMergeDocs(indexingParams.TransactionMaxMergeDocs);
writer.SetMaxBufferedDocs(indexingParams.TransactionMaxBufferedDocs);
}
writers.Add(provider, writer);
return writer;
}
catch (IOException ex)
{
CleanUp(new SearchException("Unable to open IndexWriter" + (entity != null ? " for " + entity : ""), ex));
}
return null;
}
public YearMonthHistoricalFoldersTraits(IDirectoryProvider directoryProvider)
{
_directoryProvider = directoryProvider;
}
private void LockProvider(IDirectoryProvider provider)
{
//make sure to use a semaphore
object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
Monitor.Enter(syncLock);
if (!lockedProviders.Contains(provider))
{
lockedProviders.Add(provider);
dpStatistics.Add(provider, new DPStatistics());
}
}
public DirectoryMailbox(IDirectoryProvider directory, DirectoryIdentity identity, IEnumerable <IPhysicalMailbox> physicalMailboxes, DirectoryMailboxType mailboxType = DirectoryMailboxType.Organization) : base(directory, identity)
{
this.physicalMailboxes = physicalMailboxes.ToList <IPhysicalMailbox>();
this.MailboxType = mailboxType;
}