public static (long?LastProcessedCompareExchangeReferenceEtag, long?LastProcessedCompareExchangeReferenceTombstoneEtag) GetLastProcessedCompareExchangeReferenceEtags(Index index, AbstractStaticIndexBase compiled, TransactionOperationContext indexContext)
{
long?lastProcessedCompareExchangeReferenceEtag = null;
long?lastProcessedCompareExchangeReferenceTombstoneEtag = null;
foreach (var collection in compiled.CollectionsWithCompareExchangeReferences)
{
var lastProcessedCompareExchangeReferenceEtagForCollection = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, IndexStorage.CompareExchangeReferences.CompareExchange);
lastProcessedCompareExchangeReferenceEtag = lastProcessedCompareExchangeReferenceEtag.HasValue
? Math.Max(lastProcessedCompareExchangeReferenceEtag.Value, lastProcessedCompareExchangeReferenceEtagForCollection)
: lastProcessedCompareExchangeReferenceEtagForCollection;
var lastProcessedCompareExchangeReferenceTombstoneEtagForCollection = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, IndexStorage.CompareExchangeReferences.CompareExchange);
lastProcessedCompareExchangeReferenceTombstoneEtag = lastProcessedCompareExchangeReferenceTombstoneEtag.HasValue
? Math.Max(lastProcessedCompareExchangeReferenceTombstoneEtag.Value, lastProcessedCompareExchangeReferenceTombstoneEtagForCollection)
: lastProcessedCompareExchangeReferenceTombstoneEtagForCollection;
}
return(lastProcessedCompareExchangeReferenceEtag, lastProcessedCompareExchangeReferenceTombstoneEtag);
}
private static bool IsStaleDueToReferences(Index index, AbstractStaticIndexBase compiled, QueryOperationContext queryContext, TransactionOperationContext indexContext, long?referenceCutoff, long?compareExchangeReferenceCutoff, List <string> stalenessReasons)
{
foreach (var collection in index.Collections)
{
long lastIndexedEtag = -1;
if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections))
{
lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
// we haven't handled references for that collection yet
// in theory we could check what is the last etag for that collection in documents store
// but this was checked earlier by the base index class
if (lastIndexedEtag > 0)
{
foreach (var referencedCollection in referencedCollections)
{
var lastDocEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name);
var lastProcessedReferenceEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
if (referenceCutoff == null)
{
if (lastDocEtag > lastProcessedReferenceEtag)
{
if (stalenessReasons == null)
{
return(true);
}
var lastDoc = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetByEtag(queryContext.Documents, lastDocEtag);
stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " +
$"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " +
$"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " +
$"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}'), " +
$"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
}
var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name);
if (lastTombstoneEtag > lastProcessedTombstoneEtag)
{
if (stalenessReasons == null)
{
return(true);
}
var lastTombstone = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetTombstoneByEtag(queryContext.Documents, lastTombstoneEtag);
stalenessReasons.Add($"There are still some tombstone references to process from collection '{referencedCollection.Name}'. " +
$"The last tombstone etag in that collection is '{lastTombstoneEtag:#,#;;0}' " +
$"({Constants.Documents.Metadata.Id}: '{lastTombstone.LowerId}', " +
$"{Constants.Documents.Metadata.LastModified}: '{lastTombstone.LastModified}'), " +
$"but last processed tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'.");
}
}
else
{
var minDocEtag = Math.Min(referenceCutoff.Value, lastDocEtag);
if (minDocEtag > lastProcessedReferenceEtag)
{
if (stalenessReasons == null)
{
return(true);
}
var lastDoc = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetByEtag(queryContext.Documents, lastDocEtag);
stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " +
$"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " +
$"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " +
$"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}') " +
$"with cutoff set to '{referenceCutoff.Value}', " +
$"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
}
var hasTombstones = queryContext.Documents.DocumentDatabase.DocumentsStorage.HasTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(queryContext.Documents, referencedCollection.Name,
lastProcessedTombstoneEtag,
referenceCutoff.Value);
if (hasTombstones)
{
if (stalenessReasons == null)
{
return(true);
}
stalenessReasons.Add($"There are still some tombstones to process from collection '{referencedCollection.Name}' with etag range '{lastProcessedTombstoneEtag} - {referenceCutoff.Value}'.");
}
}
}
}
}
if (compiled.CollectionsWithCompareExchangeReferences.Contains(collection))
{
if (lastIndexedEtag == -1)
{
lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
// we haven't handled references for that collection yet
// in theory we could check what is the last etag for that collection in documents store
// but this was checked earlier by the base index class
if (lastIndexedEtag > 0)
{
var lastCompareExchangeEtag = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name);
var lastProcessedReferenceEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);
if (compareExchangeReferenceCutoff == null)
{
if (lastCompareExchangeEtag > lastProcessedReferenceEtag)
{
if (stalenessReasons == null)
{
return(true);
}
stalenessReasons.Add($"There are still some compare exchange references to process for collection '{collection}'. The last compare exchange etag is '{lastCompareExchangeEtag:#,#;;0}', but last processed compare exchange etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
}
var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeTombstoneIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name);
var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);
if (lastTombstoneEtag > lastProcessedTombstoneEtag)
{
if (stalenessReasons == null)
{
return(true);
}
stalenessReasons.Add($"There are still some compare exchange tombstone references to process for collection '{collection}'. The last compare exchange tombstone etag is '{lastTombstoneEtag:#,#;;0}', but last processed compare exchange tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'.");
}
}
else
{
var minCompareExchangeEtag = Math.Min(compareExchangeReferenceCutoff.Value, lastCompareExchangeEtag);
if (minCompareExchangeEtag > lastProcessedReferenceEtag)
{
if (stalenessReasons == null)
{
return(true);
}
stalenessReasons.Add($"There are still some compare exchange references to process for collection '{collection}'. The last compare exchange etag is '{lastCompareExchangeEtag:#,#;;0}' with cutoff set to '{compareExchangeReferenceCutoff.Value}', but last processed compare exchange etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
}
var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);
var hasTombstones = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.HasCompareExchangeTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(queryContext.Server, queryContext.Documents.DocumentDatabase.Name,
lastProcessedTombstoneEtag,
compareExchangeReferenceCutoff.Value);
if (hasTombstones)
{
if (stalenessReasons == null)
{
return(true);
}
stalenessReasons.Add($"There are still some compare exchange tombstones to process for collection '{collection}' with etag range '{lastProcessedTombstoneEtag} - {compareExchangeReferenceCutoff.Value}'.");
}
}
}
}
}
return(stalenessReasons?.Count > 0);
}
private void FillCollectionEtags(Transaction tx,
Dictionary <string, IndexTransactionCache.CollectionEtags> map)
{
AbstractStaticIndexBase compiled = null;
if (_index.Type.IsStatic())
{
switch (_index)
{
case MapIndex mapIndex:
compiled = mapIndex._compiled;
break;
case MapReduceIndex mapReduceIndex:
compiled = mapReduceIndex._compiled;
break;
case MapCountersIndex mapCountersIndex:
compiled = mapCountersIndex._compiled;
break;
case MapTimeSeriesIndex mapTimeSeriesIndex:
compiled = mapTimeSeriesIndex._compiled;
break;
}
}
foreach (string collection in _index.Collections)
{
using (Slice.From(tx.LowLevelTransaction.Allocator, collection, out Slice collectionSlice))
{
var etags = new IndexTransactionCache.CollectionEtags
{
LastIndexedEtag = IndexStorage.ReadLastEtag(tx,
IndexStorage.IndexSchema.EtagsTree,
collectionSlice
),
LastProcessedTombstoneEtag = IndexStorage.ReadLastEtag(tx,
IndexStorage.IndexSchema.EtagsTombstoneTree,
collectionSlice
)
};
if (compiled?.CollectionsWithCompareExchangeReferences.Contains(collection) == true)
{
etags.LastReferencedEtagsForCompareExchange = new IndexTransactionCache.ReferenceCollectionEtags
{
LastEtag = _index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(tx, collection, IndexStorage.CompareExchangeReferences.CompareExchange),
LastProcessedTombstoneEtag = _index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(tx, collection, IndexStorage.CompareExchangeReferences.CompareExchange)
};
}
map[collection] = etags;
}
}
var referencedCollections = _index.GetReferencedCollections();
if (referencedCollections == null || referencedCollections.Count == 0)
{
return;
}
foreach (var(src, collections) in referencedCollections)
{
var collectionEtags = map[src];
collectionEtags.LastReferencedEtags ??= new Dictionary <string, IndexTransactionCache.ReferenceCollectionEtags>(StringComparer.OrdinalIgnoreCase);
foreach (var collectionName in collections)
{
collectionEtags.LastReferencedEtags[collectionName.Name] = new IndexTransactionCache.ReferenceCollectionEtags
{
LastEtag = _index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceEtag(tx, src, collectionName),
LastProcessedTombstoneEtag = _index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceTombstoneEtag(tx, src, collectionName),
};
}
}
}
private static void ThrowMissingGroupByFieldsInMapOutput(object output, Dictionary <string, CompiledIndexField> groupByFields, AbstractStaticIndexBase compiledIndex)
{
throw new InvalidOperationException(
$"The output of the mapping function does not contain all fields that the index is supposed to group by.{Environment.NewLine}" +
$"Output: {output}{Environment.NewLine}" +
$"Group by fields: {string.Join(",", groupByFields.Select(x => x.Key))}{Environment.NewLine}" +
$"Compiled index def:{Environment.NewLine}{compiledIndex.Source}");
}
private static MapReduceIndexDefinition CreateIndexDefinition(IndexDefinition definition, RavenConfiguration configuration, long indexVersion, out AbstractStaticIndexBase staticIndex)
{
staticIndex = IndexCompilationCache.GetIndexInstance(definition, configuration);
return(new MapReduceIndexDefinition(definition, staticIndex.Maps.Keys, staticIndex.OutputFields, staticIndex.GroupByFields, staticIndex.HasDynamicFields, staticIndex.CollectionsWithCompareExchangeReferences.Count > 0, indexVersion));
}
public static void ValidateReduceResultsCollectionName(IndexDefinition definition, AbstractStaticIndexBase index, DocumentDatabase database, bool checkIfCollectionEmpty)
{
var outputReduceToCollection = definition.OutputReduceToCollection;
if (string.IsNullOrWhiteSpace(outputReduceToCollection))
{
return;
}
if (outputReduceToCollection.Equals(definition.PatternReferencesCollectionName, StringComparison.OrdinalIgnoreCase))
{
throw new IndexInvalidException($"Collection defined in {nameof(definition.PatternReferencesCollectionName)} must not be the same as in {nameof(definition.OutputReduceToCollection)}. Collection name: '{outputReduceToCollection}'");
}
var collections = index.Maps.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
if (collections.Contains(Constants.Documents.Collections.AllDocumentsCollection))
{
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as this index is mapping all documents " +
$"and this will result in an infinite loop.");
}
foreach (var referencedCollection in index.ReferencedCollections)
{
foreach (var collectionName in referencedCollection.Value)
{
collections.Add(collectionName.Name);
}
}
if (collections.Contains(outputReduceToCollection))
{
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as this index is mapping or referencing the '{outputReduceToCollection}' collection " +
$"and this will result in an infinite loop.");
}
var indexes = database.IndexStore.GetIndexes()
.Where(x => x.Type.IsStatic() && x.Type.IsMapReduce())
.Cast <MapReduceIndex>()
.Where(mapReduceIndex =>
{
// we have handling for side by side indexing with OutputReduceToCollection so we're checking only other indexes
string existingIndexName = mapReduceIndex.Name.Replace(Constants.Documents.Indexing.SideBySideIndexNamePrefix, string.Empty,
StringComparison.OrdinalIgnoreCase);
string newIndexName = definition.Name.Replace(Constants.Documents.Indexing.SideBySideIndexNamePrefix, string.Empty,
StringComparison.OrdinalIgnoreCase);
return(string.IsNullOrWhiteSpace(mapReduceIndex.Definition.OutputReduceToCollection) == false && string.Equals(existingIndexName, newIndexName, StringComparison.OrdinalIgnoreCase) == false);
})
.ToList();
foreach (var otherIndex in indexes)
{
if (otherIndex.Definition.OutputReduceToCollection.Equals(outputReduceToCollection, StringComparison.OrdinalIgnoreCase))
{
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as there is another index named '{otherIndex.Name}' " +
$"which also output reduce results to documents in the same '{outputReduceToCollection}' collection. " +
$"{nameof(IndexDefinition.OutputReduceToCollection)} must by set to unique value for each index or be null.");
}
var otherIndexCollections = new HashSet <string>(otherIndex.Collections);
foreach (var referencedCollection in otherIndex.GetReferencedCollections())
{
foreach (var collectionName in referencedCollection.Value)
{
otherIndexCollections.Add(collectionName.Name);
}
}
if (otherIndexCollections.Contains(outputReduceToCollection) &&
CheckIfThereIsAnIndexWhichWillOutputReduceDocumentsWhichWillBeUsedAsMapOnTheSpecifiedIndex(otherIndex, collections, indexes, out string description))
{
description += Environment.NewLine + $"--> {definition.Name}: {string.Join(",", collections)} => *{outputReduceToCollection}*";
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as '{outputReduceToCollection}' collection is consumed by other index in a way that would " +
$"lead to an infinite loop." +
Environment.NewLine + description);
}
}
var existingIndexOrSideBySide = database.IndexStore.GetIndexes()
.Where(x => x.Type.IsStatic() && x.Type.IsMapReduce())
.Cast <MapReduceIndex>()
.FirstOrDefault(x =>
{
var name = definition.Name.Replace(Constants.Documents.Indexing.SideBySideIndexNamePrefix, string.Empty,
StringComparison.OrdinalIgnoreCase);
return(x.Name.Equals(name, StringComparison.OrdinalIgnoreCase) &&
x.Definition.ReduceOutputIndex != null); // legacy index definitions don't have this field - side by side indexing isn't supported then
});
if (existingIndexOrSideBySide != null)
{
if (definition.OutputReduceToCollection.Equals(existingIndexOrSideBySide.Definition.OutputReduceToCollection, StringComparison.OrdinalIgnoreCase)) // we have handling for side by side indexing with OutputReduceToCollection
{
checkIfCollectionEmpty = false;
}
}
if (checkIfCollectionEmpty)
{
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var stats = database.DocumentsStorage.GetCollection(outputReduceToCollection, context);
if (stats.Count > 0)
{
throw new IndexInvalidException(
$"Index '{definition.Name}' is defined to output the Reduce results to documents in Collection '{outputReduceToCollection}'. " +
$"This collection currently has {stats.Count} document{(stats.Count == 1 ? ' ' : 's')}. " +
$"All documents in Collection '{stats.Name}' must be deleted first.");
}
}
}
}
protected override int GetFields <T>(T instance, LazyStringValue key, LazyStringValue sourceDocumentId, object document, JsonOperationContext indexContext, IWriteOperationBuffer writeBuffer)
{
if (!(document is ObjectInstance documentToProcess))
{
return(0);
}
int newFields = 0;
if (key != null)
{
instance.Add(GetOrCreateKeyField(key));
newFields++;
}
if (sourceDocumentId != null)
{
instance.Add(GetOrCreateSourceDocumentIdField(sourceDocumentId));
newFields++;
}
if (_storeValue)
{
var storedValue = JsBlittableBridge.Translate(indexContext,
documentToProcess.Engine,
documentToProcess);
instance.Add(GetStoredValueField(storedValue, writeBuffer));
newFields++;
}
if (TryGetBoostedValue(documentToProcess, out var boostedValue, out var documentBoost))
{
if (IsObject(boostedValue) == false)
{
throw new InvalidOperationException($"Invalid boosted value. Expected object but got '{boostedValue.Type}' with value '{boostedValue}'.");
}
documentToProcess = boostedValue.AsObject();
}
foreach (var(property, propertyDescriptor) in documentToProcess.GetOwnProperties())
{
var propertyAsString = property.AsString();
if (_fields.TryGetValue(propertyAsString, out var field) == false)
{
field = _fields[propertyAsString] = IndexField.Create(propertyAsString, new IndexFieldOptions(), _allFields);
}
object value;
float? propertyBoost = null;
int numberOfCreatedFields;
var actualValue = propertyDescriptor.Value;
var isObject = IsObject(actualValue);
if (isObject)
{
if (TryGetBoostedValue(actualValue.AsObject(), out boostedValue, out propertyBoost))
{
actualValue = boostedValue;
isObject = IsObject(actualValue);
}
if (isObject)
{
//In case TryDetectDynamicFieldCreation finds a dynamic field it will populate 'field.Name' with the actual property name
//so we must use field.Name and not property from this point on.
var val = TryDetectDynamicFieldCreation(propertyAsString, actualValue.AsObject(), field);
if (val != null)
{
if (val.IsObject() && val.AsObject().TryGetValue(SpatialPropertyName, out _))
{
actualValue = val; //Here we populate the dynamic spatial field that will be handled below.
}
else
{
value = TypeConverter.ToBlittableSupportedType(val, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
if (value is IDisposable toDispose1)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose1.Dispose();
}
continue;
}
}
var objectValue = actualValue.AsObject();
if (objectValue.HasOwnProperty(SpatialPropertyName) && objectValue.TryGetValue(SpatialPropertyName, out var inner))
{
SpatialField spatialField;
IEnumerable <AbstractField> spatial;
if (inner.IsString())
{
spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, inner.AsString());
}
else if (inner.IsObject())
{
var innerObject = inner.AsObject();
if (innerObject.HasOwnProperty("Lat") && innerObject.HasOwnProperty("Lng") && innerObject.TryGetValue("Lat", out var lat) &&
lat.IsNumber() && innerObject.TryGetValue("Lng", out var lng) && lng.IsNumber())
{
spatialField = AbstractStaticIndexBase.GetOrCreateSpatialField(field.Name);
spatial = AbstractStaticIndexBase.CreateSpatialField(spatialField, lat.AsNumber(), lng.AsNumber());
}
else
{
continue; //Ignoring bad spatial field
}
}
else
{
continue; //Ignoring bad spatial field
}
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(spatial, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
continue;
}
}
}
value = TypeConverter.ToBlittableSupportedType(actualValue, flattenArrays: false, forIndexing: true, engine: documentToProcess.Engine, context: indexContext);
numberOfCreatedFields = GetRegularFields(instance, field, CreateValueForIndexing(value, propertyBoost), indexContext, out _);
newFields += numberOfCreatedFields;
BoostDocument(instance, numberOfCreatedFields, documentBoost);
if (value is IDisposable toDispose)
{
// the value was converted to a lucene field and isn't needed anymore
toDispose.Dispose();
}
}
return(newFields);
