private bool?MoveNext(IEnumerator en, StatefulEnumerableWrapper <object> innerEnumerator, WorkContext context,
DocumentStorageActions actions)
{
try
{
actions.IncrementIndexingAttempt();
var moveNext = en.MoveNext();
if (moveNext == false)
{
actions.DecrementIndexingAttempt();
}
return(moveNext);
}
catch (Exception e)
{
actions.IncrementIndexingFailure();
context.AddError(name,
TryGetDocKey(innerEnumerator.Current),
e.Message
);
log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name,
GetDocId(innerEnumerator));
}
return(null);
}
public void ReduceDocuments(AbstractViewGenerator viewGenerator,
IEnumerable <object> mappedResults,
WorkContext context,
DocumentStorageActions actions,
string reduceKey)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
indexWriter.DeleteDocuments(new Term("__reduce_key", reduceKey));
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, reduceKey));
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context))
{
count++;
var fields = GetFields(doc, ref properties);
var luceneDoc = new Document();
luceneDoc.Add(new Field("__reduce_key", reduceKey, Field.Store.NO, Field.Index.NOT_ANALYZED));
foreach (var field in fields)
{
luceneDoc.Add(field);
}
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, reduceKey, luceneDoc));
log.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKey, name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
actions.IncrementSuccessIndexing();
}
return(true);
});
log.DebugFormat("Reduce resulted in {0} entries for {1} for reduce key {2}", count, name, reduceKey);
}
public void Batch(Action <IStorageActionsAccessor> action)
{
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, new AbstractDocumentCodec[0], new DummyUuidGenerator(), this))
{
action(new StorageActionsAccessor(pht));
}
}
public void Batch(Action<DocumentStorageActions> action)
{
using (var pht = new DocumentStorageActions(instance, database, Id))
{
action(pht);
}
}
public void Batch(Action<IStorageActionsAccessor> action)
{
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, new AbstractDocumentCodec[0]))
{
action(new StorageActionsAccessor(pht));
}
}
public IStorageActionsAccessor CreateAccessor()
{
var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator,
documentCacher, null, this);
var accessor = new StorageActionsAccessor(pht, inFlightTransactionalState.GetSnapshot());
accessor.OnDispose += pht.Dispose;
return(accessor);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Func<object, object> documentIdFetcher = null;
var reduceKeys = new HashSet<string>();
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc);
var docIdValue = documentIdFetcher(doc);
if (docIdValue == null)
throw new InvalidOperationException("Could not find document id for this document");
var reduceValue = viewGenerator.GroupByExtraction(doc);
if (reduceValue == null)
{
log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue);
continue;
}
var reduceKey = ReduceKeyToString(reduceValue);
var docId = docIdValue.ToString();
reduceKeys.Add(reduceKey);
string data = GetMapedData(doc);
log.DebugFormat("Mapped result for '{0}': '{1}'", name, data);
var hash = ComputeHash(name, reduceKey);
actions.PutMappedResult(name, docId, reduceKey, data, hash);
actions.IncrementSuccessIndexing();
}
foreach (var reduceKey in reduceKeys)
{
actions.AddTask(new ReduceTask
{
Index = name,
ReduceKey = reduceKey
});
}
log.DebugFormat("Mapped {0} documents for {1}", count, name);
}
private void ReplicateDocument(DocumentStorageActions actions, string id, JObject metadata, JObject document, string src)
{
var existingDoc = actions.DocumentByKey(id, null);
if (existingDoc == null)
{
log.DebugFormat("New document {0} replicated successfully from {1}", id, src);
actions.AddDocument(id, Guid.Empty, document, metadata);
return;
}
var existingDocumentIsInConflict = existingDoc.Metadata[ReplicationConstants.RavenReplicationConflict] != null;
if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics
(IsDirectChildOfCurrentDocument(existingDoc, metadata))) // this update is direct child of the existing doc, so we are fine with overwriting this
{
log.DebugFormat("Existing document {0} replicated successfully from {1}", id, src);
actions.AddDocument(id, null, document, metadata);
return;
}
var newDocumentConflictId = id + "/conflicts/" + metadata.Value <string>("@etag");
metadata.Add(ReplicationConstants.RavenReplicationConflict, JToken.FromObject(true));
actions.AddDocument(newDocumentConflictId, null, document, metadata);
if (existingDocumentIsInConflict) // the existing document is in conflict
{
log.DebugFormat("Conflicted document {0} has a new version from {1}, adding to conflicted documents", id, src);
// just update the current doc with the new conflict document
existingDoc.DataAsJson.Value <JArray>("Conflicts").Add(JToken.FromObject(newDocumentConflictId));
actions.AddDocument(id, existingDoc.Etag, existingDoc.DataAsJson, existingDoc.Metadata);
return;
}
log.DebugFormat("Existing document {0} is in conflict with replicated version from {1}, marking document as conflicted", id, src);
// we have a new conflict
// move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + existingDoc.Etag;
existingDoc.Metadata.Add(ReplicationConstants.RavenReplicationConflict, JToken.FromObject(true));
actions.AddDocument(existingDocumentConflictId, null, existingDoc.DataAsJson, existingDoc.Metadata);
actions.AddDocument(id, null,
new JObject(
new JProperty("Conflicts", new JArray(existingDocumentConflictId, newDocumentConflictId))),
new JObject(
new JProperty(ReplicationConstants.RavenReplicationConflict, true),
new JProperty("@Http-Status-Code", 409),
new JProperty("@Http-Status-Description", "Conflict")
));
}
public void Index(string index, AbstractViewGenerator viewGenerator, IEnumerable <dynamic> docs, WorkContext context,
DocumentStorageActions actions)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index);
return;
}
value.IndexDocuments(viewGenerator, docs, context, actions);
}
//[DebuggerHidden, DebuggerNonUserCode, DebuggerStepThrough]
private Action ExecuteBatch(Action <IStorageActionsAccessor> action, EsentTransactionContext transactionContext)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
bool lockTaken = false;
if (transactionContext != null)
{
Monitor.Enter(transactionContext, ref lockTaken);
}
try
{
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, transactionContext, this))
{
var dtcSnapshot = inFlightTransactionalState != null? // might be not already initialized yet, during database creation
inFlightTransactionalState.GetSnapshot() : EmptyInFlightStateSnapshot.Instance;
var storageActionsAccessor = new StorageActionsAccessor(pht, dtcSnapshot);
if (disableBatchNesting.Value == null)
{
current.Value = storageActionsAccessor;
}
action(storageActionsAccessor);
storageActionsAccessor.SaveAllTasks();
pht.ExecuteBeforeStorageCommit();
if (pht.UsingLazyCommit)
{
txMode = CommitTransactionGrbit.None;
}
try
{
return(pht.Commit(txMode));
}
finally
{
pht.ExecuteAfterStorageCommit();
}
}
}
finally
{
if (lockTaken)
{
Monitor.Exit(transactionContext);
}
}
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable <object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
string currentId = null;
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
string newDocId;
IEnumerable <AbstractField> fields;
if (doc is DynamicJsonObject)
{
fields = ExtractIndexDataFromDocument((DynamicJsonObject)doc, out newDocId);
}
else
{
fields = ExtractIndexDataFromDocument(properties, doc, out newDocId);
}
if (currentId != newDocId) // new document id, so delete all old values matching it
{
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, newDocId));
indexWriter.DeleteDocuments(new Term("__document_id", newDocId));
}
if (newDocId != null)
{
var luceneDoc = new Document();
luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.NOT_ANALYZED));
currentId = newDocId;
CopyFieldsToDocumentButRemoveDuplicateValues(luceneDoc, fields);
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, newDocId, luceneDoc));
log.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
}
actions.IncrementSuccessIndexing();
}
return(currentId != null);
});
log.DebugFormat("Indexed {0} documents for {1}", count, name);
}
private void ExecuteBatch(Action <IStorageActionsAccessor> action)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher))
{
current.Value = new StorageActionsAccessor(pht);
action(current.Value);
pht.Commit(txMode);
}
}
public long GetDatabaseSizeInBytes()
{
long sizeInBytes;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, null, this))
{
int sizeInPages, pageSize;
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out sizeInPages, JET_DbInfo.Filesize);
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out pageSize, JET_DbInfo.PageSize);
sizeInBytes = ((long)sizeInPages) * pageSize;
}
return(sizeInBytes);
}
private void ExecuteBatch(Action <DocumentStorageActions> action)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs))
{
current.Value = pht;
action(pht);
pht.Commit(txMode);
onCommit();
}
}
private Action ExecuteBatch(Action <IStorageActionsAccessor> action)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, this))
{
var storageActionsAccessor = new StorageActionsAccessor(pht);
current.Value = storageActionsAccessor;
action(current.Value);
storageActionsAccessor.SaveAllTasks();
return(pht.Commit(txMode));
}
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
string currentId = null;
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
string newDocId;
IEnumerable<AbstractField> fields;
if (doc is DynamicJsonObject)
fields = ExtractIndexDataFromDocument((DynamicJsonObject) doc, out newDocId);
else
fields = ExtractIndexDataFromDocument(properties, doc, out newDocId);
if (currentId != newDocId) // new document id, so delete all old values matching it
{
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, newDocId));
indexWriter.DeleteDocuments(new Term("__document_id", newDocId));
}
if (newDocId != null)
{
var luceneDoc = new Document();
luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.NOT_ANALYZED));
currentId = newDocId;
CopyFieldsToDocumentButRemoveDuplicateValues(luceneDoc, fields);
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, newDocId, luceneDoc));
log.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
}
actions.IncrementSuccessIndexing();
}
return currentId != null;
});
log.DebugFormat("Indexed {0} documents for {1}", count, name);
}
private static JsonDocument RetrieveDocument(DocumentStorageActions actions, IndexQueryResult queryResult,
HashSet <string> loadedIds)
{
if (queryResult.Projection == null)
{
if (loadedIds.Add(queryResult.Key))
{
return(actions.DocumentByKey(queryResult.Key, null));
}
return(null);
}
return(new JsonDocument
{
Key = queryResult.Key,
Projection = queryResult.Projection,
});
}
public void Reduce(string index, AbstractViewGenerator viewGenerator, IEnumerable <object> mappedResults,
WorkContext context, DocumentStorageActions actions, string reduceKey)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index);
return;
}
var mapReduceIndex = value as MapReduceIndex;
if (mapReduceIndex == null)
{
log.WarnFormat("Tried to reduce on an index that is not a map/reduce index: {0}, ignoring", index);
return;
}
mapReduceIndex.ReduceDocuments(viewGenerator, mappedResults, context, actions, reduceKey);
}
private void AddIndexingTask(DocumentStorageActions actions, JToken metadata, Func <Task> taskGenerator)
{
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
continue;
}
var entityName = metadata.Value <string>("Raven-Entity-Name");
if (viewGenerator.ForEntityName != null &&
viewGenerator.ForEntityName != entityName)
{
continue;
}
var task = taskGenerator();
task.Index = indexName;
actions.AddTask(task);
}
}
public DatabaseSizeInformation GetDatabaseSize()
{
long allocatedSizeInBytes;
long usedSizeInBytes;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, null, this))
{
int sizeInPages, pageSize, spaceOwnedInPages;
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out sizeInPages, JET_DbInfo.Filesize);
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out spaceOwnedInPages, JET_DbInfo.SpaceOwned);
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out pageSize, JET_DbInfo.PageSize);
allocatedSizeInBytes = ((long)sizeInPages) * pageSize;
usedSizeInBytes = ((long)spaceOwnedInPages) * pageSize;
}
return(new DatabaseSizeInformation
{
AllocatedSizeInBytes = allocatedSizeInBytes,
UsedSizeInBytes = usedSizeInBytes
});
}
//[DebuggerHidden, DebuggerNonUserCode, DebuggerStepThrough]
private Action ExecuteBatch(Action <IStorageActionsAccessor> action, EsentTransactionContext transactionContext)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
bool lockTaken = false;
if (transactionContext != null)
{
Monitor.Enter(transactionContext, ref lockTaken);
}
try
{
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, transactionContext, this))
{
var storageActionsAccessor = new StorageActionsAccessor(pht);
if (disableBatchNesting.Value == null)
{
current.Value = storageActionsAccessor;
}
action(storageActionsAccessor);
storageActionsAccessor.SaveAllTasks();
if (pht.UsingLazyCommit)
{
txMode = CommitTransactionGrbit.None;
}
return(pht.Commit(txMode));
}
}
finally
{
if (lockTaken)
{
Monitor.Exit(transactionContext);
}
}
}
protected IEnumerable <object> RobustEnumeration(IEnumerable <object> input, IndexingFunc func,
DocumentStorageActions actions, WorkContext context)
{
var wrapped = new StatefulEnumerableWrapper <dynamic>(input.GetEnumerator());
IEnumerator <object> en = func(wrapped).GetEnumerator();
do
{
var moveSuccessful = MoveNext(en, wrapped, context, actions);
if (moveSuccessful == false)
{
yield break;
}
if (moveSuccessful == true)
{
yield return(en.Current);
}
else
{
en = func(wrapped).GetEnumerator();
}
} while (true);
}
public void ReduceDocuments(AbstractViewGenerator viewGenerator,
IEnumerable<object> mappedResults,
WorkContext context,
DocumentStorageActions actions,
string reduceKey)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
indexWriter.DeleteDocuments(new Term("__reduce_key", reduceKey));
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, reduceKey));
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context))
{
count++;
var fields = GetFields(doc, ref properties);
var luceneDoc = new Document();
luceneDoc.Add(new Field("__reduce_key", reduceKey, Field.Store.NO, Field.Index.NOT_ANALYZED));
foreach (var field in fields)
{
luceneDoc.Add(field);
}
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, reduceKey, luceneDoc));
log.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKey, name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
actions.IncrementSuccessIndexing();
}
return true;
});
log.DebugFormat("Reduce resulted in {0} entries for {1} for reduce key {2}", count, name, reduceKey);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable <dynamic> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Func <object, object> documentIdFetcher = null;
var reduceKeys = new HashSet <string>();
var documentsWrapped = documents.Select(doc =>
{
var documentId = doc.__document_id;
foreach (var reduceKey in actions.DeleteMappedResultsForDocumentId((string)documentId, name))
{
reduceKeys.Add(reduceKey);
}
return(doc);
});
foreach (var doc in RobustEnumeration(documentsWrapped, viewGenerator.MapDefinition, actions, context))
{
count++;
documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc);
var docIdValue = documentIdFetcher(doc);
if (docIdValue == null)
{
throw new InvalidOperationException("Could not find document id for this document");
}
var reduceValue = viewGenerator.GroupByExtraction(doc);
if (reduceValue == null)
{
log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue);
continue;
}
var reduceKey = ReduceKeyToString(reduceValue);
var docId = docIdValue.ToString();
reduceKeys.Add(reduceKey);
string data = GetMapedData(doc);
log.DebugFormat("Mapped result for '{0}': '{1}'", name, data);
var hash = ComputeHash(name, reduceKey);
actions.PutMappedResult(name, docId, reduceKey, data, hash);
actions.IncrementSuccessIndexing();
}
foreach (var reduceKey in reduceKeys)
{
actions.AddTask(new ReduceTask
{
Index = name,
ReduceKey = reduceKey
});
}
log.DebugFormat("Mapped {0} documents for {1}", count, name);
}
public void Reduce(string index, AbstractViewGenerator viewGenerator, IEnumerable<object> mappedResults,
WorkContext context, DocumentStorageActions actions, string reduceKey)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index);
return;
}
var mapReduceIndex = value as MapReduceIndex;
if (mapReduceIndex == null)
{
log.WarnFormat("Tried to reduce on an index that is not a map/reduce index: {0}, ignoring", index);
return;
}
mapReduceIndex.ReduceDocuments(viewGenerator, mappedResults, context, actions, reduceKey);
}
public void Index(string index, AbstractViewGenerator viewGenerator, IEnumerable<dynamic> docs, WorkContext context,
DocumentStorageActions actions)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index);
return;
}
value.IndexDocuments(viewGenerator, docs, context, actions);
}
private void AddIndexAndEnqueueIndexingTasks(DocumentStorageActions actions, string indexName)
{
actions.AddIndex(indexName);
var firstAndLast = actions.FirstAndLastDocumentIds();
if (firstAndLast.Item1 != 0 && firstAndLast.Item2 != 0)
{
for (var i = firstAndLast.Item1; i <= firstAndLast.Item2; i += Configuration.IndexingBatchSize)
{
actions.AddTask(new IndexDocumentRangeTask
{
FromId = i,
ToId = Math.Min(i + Configuration.IndexingBatchSize, firstAndLast.Item2),
Index = indexName
});
}
}
workContext.ShouldNotifyAboutWork();
}
public abstract void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> documents,
WorkContext context,
DocumentStorageActions actions);
public IStorageActionsAccessor CreateAccessor()
{
var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator,
documentCacher, null, this);
var accessor = new StorageActionsAccessor(pht);
accessor.OnDispose += pht.Dispose;
return accessor;
}
public long GetDatabaseSizeInBytes()
{
long sizeInBytes = -1;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, this))
{
int sizeInPages, pageSize;
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out sizeInPages, JET_DbInfo.Filesize);
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out pageSize, JET_DbInfo.PageSize);
sizeInBytes = ((long)sizeInPages) * pageSize;
}
return sizeInBytes;
}
public StorageActionsAccessor(DocumentStorageActions inner)
{
this.inner = inner;
}
public DatabaseSizeInformation GetDatabaseSize()
{
long allocatedSizeInBytes;
long usedSizeInBytes;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, null, this))
{
int sizeInPages, pageSize, spaceOwnedInPages;
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out sizeInPages, JET_DbInfo.Filesize);
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out spaceOwnedInPages, JET_DbInfo.SpaceOwned);
Api.JetGetDatabaseInfo(pht.Session, pht.Dbid, out pageSize, JET_DbInfo.PageSize);
allocatedSizeInBytes = ((long)sizeInPages) * pageSize;
usedSizeInBytes = ((long)spaceOwnedInPages) * pageSize;
}
return new DatabaseSizeInformation
{
AllocatedSizeInBytes = allocatedSizeInBytes,
UsedSizeInBytes = usedSizeInBytes
};
}
private void ExecuteBatch(Action<DocumentStorageActions> action)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs))
{
current.Value = pht;
action(pht);
pht.Commit(txMode);
onCommit();
}
}
public long GetDatabaseCacheSizeInBytes()
{
long cacheSizeInBytes = 0;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, this))
{
int cacheSizeInPages = 0, pageSize = 0;
string test;
Api.JetGetSystemParameter(instance, pht.Session, JET_param.CacheSize, ref cacheSizeInPages, out test, 1024);
Api.JetGetSystemParameter(instance, pht.Session, JET_param.DatabasePageSize, ref pageSize, out test, 1024);
cacheSizeInBytes = ((long) cacheSizeInPages) * pageSize;
}
return cacheSizeInBytes;
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
var i = 0;
JET_COLUMNID columnid;
using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
{
Api.JetAddColumn(session, scheduledReductions, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var scheduledReductionsColumns = Api.GetColumnDictionary(session, scheduledReductions);
Api.MoveBeforeFirst(session, scheduledReductions);
while (Api.TryMoveNext(session, scheduledReductions))
{
using (var update = new Update(session, scheduledReductions, JET_prep.Replace))
{
var reduceKey = Api.RetrieveColumnAsString(session, scheduledReductions, scheduledReductionsColumns["reduce_key"]);
Api.SetColumn(session, scheduledReductions, scheduledReductionsColumns["hashed_reduce_key"],
DocumentStorageActions.HashReduceKey(reduceKey));
update.Save();
}
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in scheduled_reductions");
}
}
Api.JetDeleteIndex(session, scheduledReductions, "by_view_level_reduce_key_and_bucket");
SchemaCreator.CreateIndexes(session, scheduledReductions, new JET_INDEXCREATE
{
szIndexName = "by_view_level_bucket_and_hashed_reduce_key",
szKey = "+view\0+level\0+bucket\0+hashed_reduce_key\0\0",
});
}
output("Finished processing scheduled_reductions");
i = 0;
using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
Api.JetAddColumn(session, mappedResults, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var mappedResultsColumns = Api.GetColumnDictionary(session, mappedResults);
Api.MoveBeforeFirst(session, mappedResults);
while (Api.TryMoveNext(session, mappedResults))
{
using (var update = new Update(session, mappedResults, JET_prep.Replace))
{
var reduceKey = Api.RetrieveColumnAsString(session, mappedResults, mappedResultsColumns["reduce_key"]);
Api.SetColumn(session, mappedResults, mappedResultsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey));
update.Save();
}
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in mapped_results");
}
}
Api.JetDeleteIndex(session, mappedResults, "by_view_reduce_key_and_bucket");
SchemaCreator.CreateIndexes(session, mappedResults, new JET_INDEXCREATE
{
szIndexName = "by_view_bucket_and_hashed_reduce_key",
szKey = "+view\0+bucket\0+hashed_reduce_key\0\0",
});
}
output("Finished processing mapped_results");
i = 0;
using (var reduceResults = new Table(session, dbid, "reduce_results", OpenTableGrbit.None))
{
Api.JetAddColumn(session, reduceResults, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var reduceResultsColumns = Api.GetColumnDictionary(session, reduceResults);
Api.MoveBeforeFirst(session, reduceResults);
while (Api.TryMoveNext(session, reduceResults))
{
using (var update = new Update(session, reduceResults, JET_prep.Replace))
{
var reduceKey = Api.RetrieveColumnAsString(session, reduceResults, reduceResultsColumns["reduce_key"]);
Api.SetColumn(session, reduceResults, reduceResultsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey));
update.Save();
}
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in reduce_results");
}
}
Api.JetDeleteIndex(session, reduceResults, "by_view_level_reduce_key_and_bucket");
Api.JetDeleteIndex(session, reduceResults, "by_view_level_reduce_key_and_source_bucket");
SchemaCreator.CreateIndexes(session, reduceResults,
new JET_INDEXCREATE
{
szIndexName = "by_view_level_bucket_and_hashed_reduce_key",
szKey = "+view\0+level\0+bucket\0+hashed_reduce_key\0\0",
},
new JET_INDEXCREATE
{
szIndexName = "by_view_level_source_bucket_and_hashed_reduce_key",
szKey = "+view\0+level\0+source_bucket\0+hashed_reduce_key\0\0",
});
}
output("Finished processing reduce_results");
SchemaCreator.UpdateVersion(session, dbid, "4.0");
}
private void AddIndexingTask(DocumentStorageActions actions, JToken metadata, Func<Task> taskGenerator)
{
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(indexName);
if(viewGenerator==null)
continue;
var entityName = metadata.Value<string>("Raven-Entity-Name");
if(viewGenerator.ForEntityName != null &&
viewGenerator.ForEntityName != entityName)
continue;
var task = taskGenerator();
task.Index = indexName;
actions.AddTask(task);
}
}
private static JsonDocument RetrieveDocument(DocumentStorageActions actions, IndexQueryResult queryResult,
HashSet<string> loadedIds)
{
if (queryResult.Projection == null)
{
if (loadedIds.Add(queryResult.Key))
return actions.DocumentByKey(queryResult.Key, null);
return null;
}
return new JsonDocument
{
Key = queryResult.Key,
Projection = queryResult.Projection,
};
}
//[DebuggerHidden, DebuggerNonUserCode, DebuggerStepThrough]
private Action ExecuteBatch(Action<IStorageActionsAccessor> action, EsentTransactionContext transactionContext)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
bool lockTaken = false;
if (transactionContext != null)
Monitor.Enter(transactionContext, ref lockTaken);
try
{
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, transactionContext, this))
{
var storageActionsAccessor = new StorageActionsAccessor(pht);
if (disableBatchNesting.Value == null)
current.Value = storageActionsAccessor;
action(storageActionsAccessor);
storageActionsAccessor.SaveAllTasks();
pht.ExecuteBeforeStorageCommit();
if (pht.UsingLazyCommit)
txMode = CommitTransactionGrbit.None;
try
{
return pht.Commit(txMode);
}
finally
{
pht.ExecuteAfterStorageCommit();
}
}
}
finally
{
if (lockTaken)
Monitor.Exit(transactionContext);
}
}
private bool? MoveNext(IEnumerator en, StatefulEnumerableWrapper<object> innerEnumerator, WorkContext context,
DocumentStorageActions actions)
{
try
{
actions.IncrementIndexingAttempt();
var moveNext = en.MoveNext();
if (moveNext == false)
actions.DecrementIndexingAttempt();
return moveNext;
}
catch (Exception e)
{
actions.IncrementIndexingFailure();
context.AddError(name,
TryGetDocKey(innerEnumerator.Current),
e.Message
);
log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name,
GetDocId(innerEnumerator));
}
return null;
}
public StorageActionsAccessor(DocumentStorageActions inner, IInFlightStateSnapshot snapshot)
{
this.inner = inner;
InFlightStateSnapshot = snapshot;
}
public abstract void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, DocumentStorageActions actions);
private void ExecuteBatch(Action<IStorageActionsAccessor> action)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, this))
{
var storageActionsAccessor = new StorageActionsAccessor(pht);
current.Value = storageActionsAccessor;
action(current.Value);
storageActionsAccessor.SaveAllTasks();
pht.Commit(txMode);
}
}
protected IEnumerable<object> RobustEnumeration(IEnumerable<object> input, IndexingFunc func,
DocumentStorageActions actions, WorkContext context)
{
var wrapped = new StatefulEnumerableWrapper<dynamic>(input.GetEnumerator());
IEnumerator<object> en = func(wrapped).GetEnumerator();
do
{
var moveSuccessful = MoveNext(en, wrapped, context, actions);
if (moveSuccessful == false)
yield break;
if (moveSuccessful == true)
yield return en.Current;
else
en = func(wrapped).GetEnumerator();
} while (true);
}
//[DebuggerHidden, DebuggerNonUserCode, DebuggerStepThrough]
private Action ExecuteBatch(Action<IStorageActionsAccessor> action, EsentTransactionContext transactionContext)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, transactionContext, this))
{
var storageActionsAccessor = new StorageActionsAccessor(pht);
if(disableBatchNesting.Value == null)
current.Value = storageActionsAccessor;
action(storageActionsAccessor);
storageActionsAccessor.SaveAllTasks();
if (pht.UsingLazyCommit)
{
txMode = CommitTransactionGrbit.None;
}
return pht.Commit(txMode);
}
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
var i = 0;
CreateReduceKeysCountsTable(session, dbid);
CreateReduceKeysStatusTable(session, dbid);
var countsPerKeyPerIndex = new Dictionary <string, Dictionary <string, int> >(StringComparer.OrdinalIgnoreCase);
using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
SchemaCreator.CreateIndexes(session, mappedResults, new JET_INDEXCREATE
{
szIndexName = "by_view_and_hashed_reduce_key",
szKey = "+view\0+hashed_reduce_key\0\0",
});
var columnDictionary = Api.GetColumnDictionary(session, mappedResults);
Api.MoveBeforeFirst(session, mappedResults);
while (Api.TryMoveNext(session, mappedResults))
{
var index = Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["view"], Encoding.Unicode);
var reduceKey = Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["reduce_key"], Encoding.Unicode);
var countPerKey = countsPerKeyPerIndex.GetOrAdd(index);
countPerKey[reduceKey] = countPerKey.GetOrDefault(reduceKey) + 1;
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in mapped_results");
}
}
}
output("Finished processing mapped_results");
using (var reduceKeys = new Table(session, dbid, "reduce_keys_status", OpenTableGrbit.None))
{
var columnDictionary = Api.GetColumnDictionary(session, reduceKeys);
foreach (var countPerKey in countsPerKeyPerIndex)
{
foreach (var keyCount in countPerKey.Value)
{
using (var update = new Update(session, reduceKeys, JET_prep.Insert))
{
Api.SetColumn(session, reduceKeys, columnDictionary["view"], countPerKey.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["reduce_key"], keyCount.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(keyCount.Key));
Api.SetColumn(session, reduceKeys, columnDictionary["reduce_type"], (int)ReduceType.MultiStep);
update.Save();
}
}
}
}
output("Finished processing reduce_keys_status");
using (var reduceKeys = new Table(session, dbid, "reduce_keys_counts", OpenTableGrbit.None))
{
var columnDictionary = Api.GetColumnDictionary(session, reduceKeys);
foreach (var countPerKey in countsPerKeyPerIndex)
{
foreach (var keyCount in countPerKey.Value)
{
using (var update = new Update(session, reduceKeys, JET_prep.Insert))
{
Api.SetColumn(session, reduceKeys, columnDictionary["view"], countPerKey.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["reduce_key"], keyCount.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(keyCount.Key));
Api.SetColumn(session, reduceKeys, columnDictionary["mapped_items_count"], keyCount.Value);
update.Save();
}
}
}
}
output("Finished processing reduce_keys_counts");
using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
{
SchemaCreator.CreateIndexes(session, scheduledReductions, new JET_INDEXCREATE
{
szIndexName = "by_view_level_and_hashed_reduce_key",
szKey = "+view\0+level\0+hashed_reduce_key\0\0",
});
}
output("Finished processing scheduled_reductions");
SchemaCreator.UpdateVersion(session, dbid, "4.1");
}
public StorageActionsAccessor(DocumentStorageActions inner)
{
this.inner = inner;
}
private Action ExecuteBatch(Action<IStorageActionsAccessor> action)
{
var txMode = configuration.TransactionMode == TransactionMode.Lazy
? CommitTransactionGrbit.LazyFlush
: CommitTransactionGrbit.None;
using (var pht = new DocumentStorageActions(instance, database, tableColumnsCache, DocumentCodecs, generator, documentCacher, this))
{
var storageActionsAccessor = new StorageActionsAccessor(pht);
current.Value = storageActionsAccessor;
action(current.Value);
storageActionsAccessor.SaveAllTasks();
pht.FlushMapReduceUpdates();
if (pht.UsingLazyCommit)
{
txMode = CommitTransactionGrbit.WaitLastLevel0Commit;
}
return pht.Commit(txMode);
}
}
