public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, InMemoryRavenConfiguration configuration)
{
this.indexDefinitionStorage = indexDefinitionStorage;
this.configuration = configuration;
path = Path.Combine(configuration.DataDirectory, "Indexes");
if (Directory.Exists(path) == false && configuration.RunInMemory == false)
{
Directory.CreateDirectory(path);
}
foreach (var indexDirectory in indexDefinitionStorage.IndexNames)
{
log.DebugFormat("Loading saved index {0}", indexDirectory);
var indexDefinition = indexDefinitionStorage.GetIndexDefinition(indexDirectory);
if (indexDefinition == null)
{
continue;
}
var luceneDirectory = OpenOrCreateLuceneDirectory(IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path));
var indexImplementation = CreateIndexImplementation(indexDirectory, indexDefinition, luceneDirectory);
indexes.TryAdd(indexDirectory, indexImplementation);
}
}
public void DeleteIndex(string name)
{
name = IndexDefinitionStorage.FixupIndexName(name);
IndexDefinitionStorage.RemoveIndex(name);
IndexStorage.DeleteIndex(name);
//we may run into a conflict when trying to delete if the index is currently
//busy indexing documents, worst case scenario, we will have an orphaned index
//row which will get cleaned up on next db restart.
for (var i = 0; i < 10; i++)
{
try
{
TransactionalStorage.Batch(action =>
{
action.Indexing.DeleteIndex(name);
workContext.ShouldNotifyAboutWork();
});
return;
}
catch (ConcurrencyException)
{
Thread.Sleep(100);
}
}
}
public void DeleteIndex(string name)
{
name = IndexDefinitionStorage.FixupIndexName(name);
IndexDefinitionStorage.RemoveIndex(name);
IndexStorage.DeleteIndex(name);
//we may run into a conflict when trying to delete if the index is currently
//busy indexing documents
for (var i = 0; i < 10; i++)
{
try
{
TransactionalStorage.Batch(action =>
{
action.Indexing.DeleteIndex(name);
workContext.ShouldNotifyAboutWork();
});
return;
}
catch (ConcurrencyException)
{
Thread.Sleep(100);
}
}
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, string indexName = null)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close(); // creating index structure
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory));
directory = FSDirectory.Open(new DirectoryInfo(indexFullPath));
if (!IndexReader.IndexExists(directory))
{
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close();
}
else
{
// forcefully unlock locked indexes if any
if (IndexWriter.IsLocked(directory))
{
IndexWriter.Unlock(directory);
}
}
}
return(directory);
}
private void OpenIndexOnStartup(DocumentDatabase documentDatabase, string indexName)
{
if (indexName == null)
{
throw new ArgumentNullException("indexName");
}
log.Debug("Loading saved index {0}", indexName);
var indexDefinition = indexDefinitionStorage.GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return;
}
Index indexImplementation;
bool resetTried = false;
while (true)
{
try
{
var luceneDirectory = OpenOrCreateLuceneDirectory(indexDefinition);
indexImplementation = CreateIndexImplementation(indexName, indexDefinition, luceneDirectory);
break;
}
catch (Exception e)
{
if (resetTried)
{
throw new InvalidOperationException("Could not open / create index" + indexName + ", reset already tried", e);
}
resetTried = true;
log.WarnException("Could not open index " + indexName + ", forcibly resetting index", e);
try
{
documentDatabase.TransactionalStorage.Batch(accessor =>
{
accessor.Indexing.DeleteIndex(indexName);
accessor.Indexing.AddIndex(indexName, indexDefinition.IsMapReduce);
});
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory));
IOExtensions.DeleteDirectory(indexFullPath);
}
catch (Exception exception)
{
throw new InvalidOperationException("Could not reset index " + indexName, exception);
}
}
}
indexes.TryAdd(indexName, indexImplementation);
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(
IndexDefinition indexDefinition,
string indexName = null,
bool createIfMissing = true)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose(); // creating index structure
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory));
directory = new LuceneCodecDirectory(indexFullPath, documentDatabase.IndexCodecs.OfType <AbstractIndexCodec>());
if (!IndexReader.IndexExists(directory))
{
if (createIfMissing == false)
{
throw new InvalidOperationException("Index does not exists: " + indexDirectory);
}
WriteIndexVersion(directory);
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
}
else
{
EnsureIndexVersionMatches(indexName, directory);
if (directory.FileExists("write.lock")) // force lock release, because it was still open when we shut down
{
IndexWriter.Unlock(directory);
// for some reason, just calling unlock doesn't remove this file
directory.DeleteFile("write.lock");
}
if (directory.FileExists("writing-to-index.lock")) // we had an unclean shutdown
{
if (configuration.ResetIndexOnUncleanShutdown)
{
throw new InvalidOperationException("Rude shutdown detected on: " + indexDirectory);
}
CheckIndexAndRecover(directory, indexDirectory);
directory.DeleteFile("writing-to-index.lock");
}
}
}
return(directory);
}
public void CreateIndexImplementation(IndexDefinition indexDefinition)
{
var encodedName = IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
log.InfoFormat("Creating index {0} with encoded name {1}", indexDefinition.Name, encodedName);
AssertAnalyzersValid(indexDefinition);
indexes.AddOrUpdate(indexDefinition.Name, n =>
{
var directory = OpenOrCreateLuceneDirectory(indexDefinition, encodedName);
return(CreateIndexImplementation(encodedName, indexDefinition, directory));
}, (s, index) => index);
}
public void ResetIndex(string index)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var indexDefinition = IndexDefinitionStorage.GetIndexDefinition(index);
if (indexDefinition == null)
{
throw new InvalidOperationException("There is no index named: " + index);
}
IndexStorage.DeleteIndex(index);
IndexStorage.CreateIndexImplementation(indexDefinition);
TransactionalStorage.Batch(actions =>
{
actions.Indexing.DeleteIndex(index);
AddIndexAndEnqueueIndexingTasks(actions, index);
});
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(
IndexDefinition indexDefinition,
string indexName = null,
bool createIfMissing = true)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close(); // creating index structure
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory));
directory = FSDirectory.Open(new DirectoryInfo(indexFullPath));
if (!IndexReader.IndexExists(directory))
{
if (createIfMissing == false)
{
throw new InvalidOperationException("Index does not exists: " + indexDirectory);
}
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close();
}
else
{
if (directory.FileExists("write.lock")) // we had an unclean shutdown
{
if (configuration.ResetIndexOnUncleanShutdown)
{
throw new InvalidOperationException("Rude shutdown detected on: " + indexDirectory);
}
CheckIndexAndRecover(directory, indexDirectory);
IndexWriter.Unlock(directory);
directory.DeleteFile("write.lock");
}
}
}
return(directory);
}
public void ResetIndex(string index)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var indexDefinition = IndexDefinitionStorage.GetIndexDefinition(index);
if (indexDefinition == null)
{
throw new InvalidOperationException("There is no index named: " + index);
}
IndexStorage.DeleteIndex(index);
IndexStorage.CreateIndexImplementation(indexDefinition);
TransactionalStorage.Batch(actions =>
{
actions.Indexing.DeleteIndex(index);
actions.Indexing.AddIndex(index, indexDefinition.IsMapReduce);
workContext.ShouldNotifyAboutWork();
});
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, string indexName = null)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
directory = FSDirectory.Open(new DirectoryInfo(Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory))));
}
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close();
return(directory);
}
public string PutIndex(string name, IndexDefinition definition)
{
definition.Name = name = IndexDefinitionStorage.FixupIndexName(name);
switch (IndexDefinitionStorage.FindIndexCreationOptions(definition))
{
case IndexCreationOptions.Noop:
return(name);
case IndexCreationOptions.Update:
// ensure that the code can compile
new DynamicViewCompiler(name, definition, Extensions, IndexDefinitionStorage.IndexDefinitionsPath).GenerateInstance();
DeleteIndex(name);
break;
}
IndexDefinitionStorage.AddIndex(definition);
IndexStorage.CreateIndexImplementation(definition);
TransactionalStorage.Batch(actions => AddIndexAndEnqueueIndexingTasks(actions, name));
return(name);
}
public IEnumerable <string> QueryDocumentIds(string index, IndexQuery query, out bool stale)
{
index = IndexDefinitionStorage.FixupIndexName(index);
bool isStale = false;
HashSet <string> loadedIds = null;
TransactionalStorage.Batch(
actions =>
{
isStale = actions.Staleness.IsIndexStale(index, query.Cutoff, null);
var indexFailureInformation = actions.Indexing.GetFailureRate(index)
;
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
loadedIds = new HashSet <string>(from queryResult in IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, AggregationOperation.None, Raven.Abstractions.Data.Constants.DocumentIdFieldName))
select queryResult.Key);
});
stale = isStale;
return(loadedIds);
}
public string PutIndex(string name, IndexDefinition definition)
{
definition.Name = name = IndexDefinitionStorage.FixupIndexName(name);
switch (IndexDefinitionStorage.FindIndexCreationOptions(definition))
{
case IndexCreationOptions.Noop:
return(name);
case IndexCreationOptions.Update:
// ensure that the code can compile
new DynamicViewCompiler(name, definition, Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Configuration).GenerateInstance();
DeleteIndex(name);
break;
}
IndexDefinitionStorage.AddIndex(definition);
IndexStorage.CreateIndexImplementation(definition);
TransactionalStorage.Batch(actions =>
{
actions.Indexing.AddIndex(name, definition.IsMapReduce);
workContext.ShouldNotifyAboutWork();
});
return(name);
}
internal Lucene.Net.Store.Directory MakeRAMDirectoryPhysical(RAMDirectory ramDir, string indexName)
{
var newDir = new LuceneCodecDirectory(Path.Combine(path, MonoHttpUtility.UrlEncode(IndexDefinitionStorage.FixupIndexName(indexName, path))), documentDatabase.IndexCodecs.OfType <AbstractIndexCodec>());
Lucene.Net.Store.Directory.Copy(ramDir, newDir, true);
return(newDir);
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List <JObject>();
var stale = false;
Tuple <DateTime, Guid> indexTimestamp = null;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
throw new InvalidOperationException("Could not find index named: " + index);
}
entityName = viewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Abstractions.Data.Constants.DocumentIdFieldName
: Abstractions.Data.Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List <string>();
IEnumerable <JObject> results;
if (viewGenerator != null &&
query.SkipTransformResults == false &&
viewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return(new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
});
}
public IndexDefinition GetIndexDefinition(string index)
{
index = IndexDefinitionStorage.FixupIndexName(index);
return(IndexDefinitionStorage.GetIndexDefinition(index));
}
internal Lucene.Net.Store.Directory MakeRAMDirectoryPhysical(RAMDirectory ramDir, string indexName)
{
var newDir = FSDirectory.Open(new DirectoryInfo(Path.Combine(path, MonoHttpUtility.UrlEncode(IndexDefinitionStorage.FixupIndexName(indexName, path)))));
Lucene.Net.Store.Directory.Copy(ramDir, newDir, true);
return(newDir);
}