private static BlittableJsonReaderObject GetReduceResult(ulong reduceKeyHash, Index index, TransactionOperationContext context)
{
using (var reader = index.IndexPersistence.OpenIndexReader(context.Transaction.InnerTransaction))
using (index.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var queryParameters = context.ReadObject(new DynamicJsonValue
{
["p0"] = reduceKeyHash.ToString()
}, "query/parameters");
var query = new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE '{Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName}' = $p0", queryParameters);
var fieldsToFetch = new FieldsToFetch(query, index.Definition);
var retriever = new MapReduceQueryResultRetriever(null, null, null, null, context, fieldsToFetch, null);
var result = reader
.Query(query, null, fieldsToFetch, new Reference <int>(), new Reference <int>(), retriever, ctx, null, CancellationToken.None)
.ToList();
if (result.Count == 0)
{
return(context.ReadObject(new DynamicJsonValue(), "debug-reduce-result"));
}
if (result.Count > 1)
{
throw new InvalidOperationException("Cannot have multiple reduce results for a single reduce key");
}
return(result[0].Result.Data);
}
}
public override unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, bool isPassive, out object result)
{
result = null;
var subscriptionId = SubscriptionId ?? index;
SubscriptionName = string.IsNullOrEmpty(SubscriptionName) ? subscriptionId.ToString() : SubscriptionName;
var receivedSubscriptionState = context.ReadObject(new SubscriptionState
{
Query = Query,
ChangeVectorForNextBatchStartingPoint = InitialChangeVector,
SubscriptionId = subscriptionId,
SubscriptionName = SubscriptionName,
LastTimeServerMadeProgressWithDocuments = DateTime.UtcNow,
Disabled = Disabled,
LastClientConnectionTime = DateTime.Now
}.ToJson(), SubscriptionName);
BlittableJsonReaderObject modifiedSubscriptionState = null;
try
{
string subscriptionItemName = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName);
using (Slice.From(context.Allocator, subscriptionItemName, out Slice valueName))
using (Slice.From(context.Allocator, subscriptionItemName.ToLowerInvariant(), out Slice valueNameLowered))
{
if (items.ReadByKey(valueNameLowered, out TableValueReader tvr))
{
var ptr = tvr.Read(2, out int size);
var doc = new BlittableJsonReaderObject(ptr, size, context);
var existingSubscriptionState = JsonDeserializationClient.SubscriptionState(doc);
if (SubscriptionId != existingSubscriptionState.SubscriptionId)
{
throw new InvalidOperationException("A subscription could not be modified because the name '" + subscriptionItemName +
"' is already in use in a subscription with different Id.");
}
if (Enum.TryParse(InitialChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorState) && changeVectorState == Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange)
{
if (receivedSubscriptionState.Modifications == null)
{
receivedSubscriptionState.Modifications = new DynamicJsonValue();
}
receivedSubscriptionState.Modifications[nameof(SubscriptionState.ChangeVectorForNextBatchStartingPoint)] = existingSubscriptionState.ChangeVectorForNextBatchStartingPoint;
modifiedSubscriptionState = context.ReadObject(receivedSubscriptionState, SubscriptionName);
}
}
ClusterStateMachine.UpdateValue(subscriptionId, items, valueNameLowered, valueName, modifiedSubscriptionState ?? receivedSubscriptionState);
}
}
finally
{
receivedSubscriptionState.Dispose();
modifiedSubscriptionState?.Dispose();
}
}
public override unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, RachisState state, out object result)
{
result = null;
var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName);
using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered))
using (Slice.From(context.Allocator, itemKey, out Slice valueName))
{
if (items.ReadByKey(valueNameLowered, out var tvr) == false)
{
throw new RachisApplyException($"Cannot find subscription {index}");
}
var ptr = tvr.Read(2, out int size);
var doc = new BlittableJsonReaderObject(ptr, size, context);
var subscriptionState = JsonDeserializationClient.SubscriptionState(doc);
subscriptionState.Disabled = Disable;
using (var obj = context.ReadObject(subscriptionState.ToJson(), "subscription"))
{
ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, obj);
}
}
}
private unsafe void UpdateInternal(TransactionOperationContext context, string guid, string type, long index, long term, HistoryStatus status, object result, Exception exception)
{
var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice);
TableValueReader reader;
using (Slice.From(context.Allocator, guid, out var guidSlice))
{
if (table.ReadByKey(guidSlice, out reader) == false)
{
return;
}
}
if (TypeConverter.IsSupportedType(result) == false)
{
throw new RachisApplyException("We don't support type " + result.GetType().FullName + ".");
}
using (Slice.From(context.Allocator, guid, out var guidSlice))
using (Slice.From(context.Allocator, type, out var typeSlice))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(guidSlice);
tvb.Add(Bits.SwapBytes(index));
tvb.Add(Bits.SwapBytes(GetUniqueTicks(context.Transaction.InnerTransaction)));
tvb.Add(*(long *)reader.Read((int)(LogHistoryColumn.Term), out _));
tvb.Add(term);
tvb.Add(typeSlice);
tvb.Add((byte)status);
if (result == null)
{
tvb.Add(Slices.Empty);
}
else
{
var blittableResult = context.ReadObject(new DynamicJsonValue {
["Result"] = result
}, "set-history-result");
tvb.Add(blittableResult.BasePointer, blittableResult.Size);
}
if (exception == null)
{
tvb.Add(Slices.Empty);
tvb.Add(Slices.Empty);
}
else
{
var exceptionType = context.GetLazyString(exception.GetType().AssemblyQualifiedName);
var exceptionMsg = context.GetLazyString(exception.ToString());
tvb.Add(exceptionType.Buffer, exceptionType.Size);
tvb.Add(exceptionMsg.Buffer, exceptionMsg.Size);
}
table.Set(tvb);
}
}
public override unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, RachisState state, out object result)
{
result = null;
var subscriptionId = SubscriptionId ?? index;
SubscriptionName = string.IsNullOrEmpty(SubscriptionName) ? subscriptionId.ToString() : SubscriptionName;
var subscriptionItemName = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName);
using (Slice.From(context.Allocator, subscriptionItemName, out Slice valueName))
using (Slice.From(context.Allocator, subscriptionItemName.ToLowerInvariant(), out Slice valueNameLowered))
{
if (items.ReadByKey(valueNameLowered, out TableValueReader tvr))
{
var ptr = tvr.Read(2, out int size);
var doc = new BlittableJsonReaderObject(ptr, size, context);
var existingSubscriptionState = JsonDeserializationClient.SubscriptionState(doc);
if (SubscriptionId != existingSubscriptionState.SubscriptionId)
{
throw new RachisApplyException("A subscription could not be modified because the name '" + subscriptionItemName +
"' is already in use in a subscription with different Id.");
}
if (string.IsNullOrEmpty(InitialChangeVector) == false && InitialChangeVector == nameof(Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange))
{
InitialChangeVector = existingSubscriptionState.ChangeVectorForNextBatchStartingPoint;
}
else
{
AssertValidChangeVector();
}
}
else
{
AssertValidChangeVector();
}
using (var receivedSubscriptionState = context.ReadObject(new SubscriptionState
{
Query = Query,
ChangeVectorForNextBatchStartingPoint = InitialChangeVector,
SubscriptionId = subscriptionId,
SubscriptionName = SubscriptionName,
LastBatchAckTime = null,
Disabled = Disabled,
MentorNode = MentorNode,
LastClientConnectionTime = null
}.ToJson(), SubscriptionName))
{
ClusterStateMachine.UpdateValue(subscriptionId, items, valueNameLowered, valueName, receivedSubscriptionState);
}
}
}
public string GetHistoryLogsAsString(TransactionOperationContext context)
{
var sb = new StringBuilder();
foreach (var entry in GetHistoryLogs(context))
{
sb.AppendLine(context.ReadObject(entry, "raft-command-history").ToString());
}
return(sb.ToString());
}
private static unsafe BlittableJsonReaderObject GetAggregationResult(long pageNumber, Table table, TransactionOperationContext context)
{
var tmp = Bits.SwapBytes(pageNumber);
using (Slice.External(context.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
table.ReadByKey(pageNumberSlice, out TableValueReader tvr);
var numberOfResults = *(int *)tvr.Read(ReduceMapResultsOfStaticIndex.NumberOfResultsPosition, out int _);
if (numberOfResults == 0)
{
return(context.ReadObject(new DynamicJsonValue(), "debug-reduce-result"));
}
return(new BlittableJsonReaderObject(tvr.Read(3, out int size), size, context));
}
}
public unsafe void AddAlert(Alert alert, TransactionOperationContext context, RavenTransaction tx)
{
_store?.TrackChangeAfterTransactionCommit(context, "AlertRaised", alert.Key);
var table = tx.InnerTransaction.OpenTable(_alertsSchema, AlertsSchema.AlertsTree);
var alertId = alert.Id;
var alertAsJson = alert.ToJson();
// if previous alert has dismissed until value pass this value to newly saved alert
Slice slice;
using (Slice.From(tx.InnerTransaction.Allocator, alertId, out slice))
{
var existingTvr = table.ReadByKey(slice);
if (existingTvr != null)
{
var existingAlert = Read(context, existingTvr);
object dismissedUntilValue;
existingAlert.TryGetMember(nameof(alert.DismissedUntil), out dismissedUntilValue);
if (dismissedUntilValue != null)
{
var dismissedUntil = (LazyStringValue)dismissedUntilValue;
alertAsJson[nameof(alert.DismissedUntil)] = dismissedUntil;
}
}
}
using (var id = context.GetLazyString(alertId))
using (var json = context.ReadObject(alertAsJson, "Alert", BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
var tvb = new TableValueBuilder
{
{ id.Buffer, id.Size },
{ json.BasePointer, json.Size }
};
table.Set(tvb);
}
}
public BlittableJsonReaderObject GetDatabaseRecordForDebugPackage(TransactionOperationContext context, string databaseName)
{
var databaseRecord = Server.ServerStore.Cluster.ReadRawDatabaseRecord(context, databaseName);
if (databaseRecord == null)
{
throw new RavenException($"Couldn't fetch {nameof(DatabaseRecord)} from server for database '{databaseName}'");
}
var djv = new DynamicJsonValue();
foreach (string fld in FieldsThatShouldBeExposedForDebug)
{
if (databaseRecord.Raw.TryGetMember(fld, out var obj))
{
djv[fld] = obj;
}
}
return(context.ReadObject(djv, "databaserecord"));
}
private static string GetTreeName(BlittableJsonReaderObject reduceEntry, IndexDefinitionBase indexDefinition, TransactionOperationContext context)
{
HashSet <string> groupByFields;
if (indexDefinition is MapReduceIndexDefinition)
{
groupByFields = ((MapReduceIndexDefinition)indexDefinition).GroupByFields;
}
else if (indexDefinition is AutoMapReduceIndexDefinition)
{
groupByFields = ((AutoMapReduceIndexDefinition)indexDefinition).GroupByFields.Keys.ToHashSet();
}
else
{
throw new InvalidOperationException("Invalid map reduce index definition: " + indexDefinition.GetType());
}
foreach (var prop in reduceEntry.GetPropertyNames())
{
if (groupByFields.Contains(prop))
{
continue;
}
if (reduceEntry.Modifications == null)
{
reduceEntry.Modifications = new DynamicJsonValue(reduceEntry);
}
reduceEntry.Modifications.Remove(prop);
}
var reduceKey = context.ReadObject(reduceEntry, "debug: creating reduce tree name");
return(reduceKey.ToString());
}
public static async Task PutCertificateCollectionInCluster(CertificateDefinition certDef, byte[] certBytes, string password, ServerStore serverStore, TransactionOperationContext ctx)
{
var collection = new X509Certificate2Collection();
if (string.IsNullOrEmpty(password))
{
collection.Import(certBytes);
}
else
{
collection.Import(certBytes, password, X509KeyStorageFlags.PersistKeySet | X509KeyStorageFlags.Exportable);
}
var first = true;
var collectionPrimaryKey = string.Empty;
foreach (var x509Certificate in collection)
{
if (serverStore.Server.Certificate.Certificate?.Thumbprint != null && serverStore.Server.Certificate.Certificate.Thumbprint.Equals(x509Certificate.Thumbprint))
{
throw new InvalidOperationException($"You are trying to import the same server certificate ({x509Certificate.Thumbprint}) as the one which is already loaded. This is not supported.");
}
}
foreach (var x509Certificate in collection)
{
var currentCertDef = new CertificateDefinition
{
Name = certDef.Name,
Permissions = certDef.Permissions,
SecurityClearance = certDef.SecurityClearance,
Password = certDef.Password,
};
if (x509Certificate.HasPrivateKey)
{
// avoid storing the private key
currentCertDef.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
}
// In case of a collection, we group all the certificates together and treat them as one unit.
// They all have the same name and permissions but a different thumbprint.
// The first certificate in the collection will be the primary certificate and its thumbprint will be the one shown in a GET request
// The other certificates are secondary certificates and will contain a link to the primary certificate.
currentCertDef.Thumbprint = x509Certificate.Thumbprint;
currentCertDef.NotAfter = x509Certificate.NotAfter;
currentCertDef.Certificate = Convert.ToBase64String(x509Certificate.Export(X509ContentType.Cert));
if (first)
{
var firstKey = Constants.Certificates.Prefix + x509Certificate.Thumbprint;
collectionPrimaryKey = firstKey;
foreach (var cert in collection)
{
if (Constants.Certificates.Prefix + cert.Thumbprint != firstKey)
{
currentCertDef.CollectionSecondaryKeys.Add(Constants.Certificates.Prefix + cert.Thumbprint);
}
}
}
else
{
currentCertDef.CollectionPrimaryKey = collectionPrimaryKey;
}
var certKey = Constants.Certificates.Prefix + currentCertDef.Thumbprint;
if (serverStore.CurrentRachisState == RachisState.Passive)
{
using (var certificate = ctx.ReadObject(currentCertDef.ToJson(), "Client/Certificate/Definition"))
using (var tx = ctx.OpenWriteTransaction())
{
serverStore.Cluster.PutLocalState(ctx, certKey, certificate);
tx.Commit();
}
}
else
{
var putResult = await serverStore.PutValueInClusterAsync(new PutCertificateCommand(certKey, currentCertDef));
await serverStore.Cluster.WaitForIndexNotification(putResult.Index);
}
first = false;
}
}
protected override AggregationResult AggregateOn(List <BlittableJsonReaderObject> aggregationBatch, TransactionOperationContext indexContext, CancellationToken token)
{
var aggregatedResultsByReduceKey = new Dictionary <BlittableJsonReaderObject, Dictionary <string, PropertyResult> >(ReduceKeyComparer.Instance);
foreach (var obj in aggregationBatch)
{
token.ThrowIfCancellationRequested();
using (obj)
{
var aggregatedResult = new Dictionary <string, PropertyResult>();
foreach (var propertyName in obj.GetPropertyNames())
{
if (_indexDefinition.TryGetField(propertyName, out var indexField))
{
switch (indexField.Aggregation)
{
case AggregationOperation.None:
if (obj.TryGet(propertyName, out object groupByValue) == false)
{
throw new InvalidOperationException($"Could not read group by value of '{propertyName}' property");
}
aggregatedResult[propertyName] = new PropertyResult
{
ResultValue = groupByValue
};
break;
case AggregationOperation.Count:
case AggregationOperation.Sum:
object value;
if (obj.TryGetMember(propertyName, out value) == false)
{
throw new InvalidOperationException($"Could not read numeric value of '{propertyName}' property");
}
double doubleValue;
long longValue;
var numberType = BlittableNumber.Parse(value, out doubleValue, out longValue);
var aggregate = new PropertyResult(numberType);
switch (numberType)
{
case NumberParseResult.Double:
aggregate.ResultValue = aggregate.DoubleValue = doubleValue;
break;
case NumberParseResult.Long:
aggregate.ResultValue = aggregate.LongValue = longValue;
break;
default:
throw new ArgumentOutOfRangeException($"Unknown number type: {numberType}");
}
aggregatedResult[propertyName] = aggregate;
break;
//case FieldMapReduceOperation.None:
default:
throw new ArgumentOutOfRangeException($"Unhandled field type '{indexField.Aggregation}' to aggregate on");
}
}
if (_indexDefinition.GroupByFields.ContainsKey(propertyName) == false)
{
// we want to reuse existing entry to get a reduce key
if (obj.Modifications == null)
{
obj.Modifications = new DynamicJsonValue(obj);
}
obj.Modifications.Remove(propertyName);
}
}
var reduceKey = indexContext.ReadObject(obj, "reduce key");
if (aggregatedResultsByReduceKey.TryGetValue(reduceKey, out Dictionary <string, PropertyResult> existingAggregate) == false)
{
aggregatedResultsByReduceKey.Add(reduceKey, aggregatedResult);
}
else
{
reduceKey.Dispose();
foreach (var propertyResult in existingAggregate)
{
propertyResult.Value.Aggregate(aggregatedResult[propertyResult.Key]);
}
}
}
}
var resultObjects = new List <Document>(aggregatedResultsByReduceKey.Count);
foreach (var aggregationResult in aggregatedResultsByReduceKey)
{
aggregationResult.Key.Dispose();
var djv = new DynamicJsonValue();
foreach (var aggregate in aggregationResult.Value)
{
djv[aggregate.Key] = aggregate.Value.ResultValue;
}
resultObjects.Add(new Document {
Data = indexContext.ReadObject(djv, "map/reduce")
});
}
return(new AggregatedDocuments(resultObjects));
}
private void NegotiateWithLeader(TransactionOperationContext context, LogLengthNegotiation negotiation)
{
_debugRecorder.Start();
// only the leader can send append entries, so if we accepted it, it's the leader
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a negotiation request for term {negotiation.Term} where our term is {_term}");
}
if (negotiation.Term != _term)
{
// Our leader is no longer a valid one
var msg = $"The term was changed after leader validation from {_term} to {negotiation.Term}, so you are no longer a valid leader";
_connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Rejected,
Message = msg,
CurrentTerm = _term,
LastLogIndex = negotiation.PrevLogIndex
});
throw new InvalidOperationException($"We close this follower because: {msg}");
}
long prevTerm;
using (context.OpenReadTransaction())
{
prevTerm = _engine.GetTermFor(context, negotiation.PrevLogIndex) ?? 0;
}
if (prevTerm != negotiation.PrevLogTerm)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a negotiation request with PrevLogTerm={negotiation.PrevLogTerm} while our PrevLogTerm={prevTerm}" +
" will negotiate to find next matched index");
}
// we now have a mismatch with the log position, and need to negotiate it with
// the leader
NegotiateMatchEntryWithLeaderAndApplyEntries(context, _connection, negotiation);
}
else
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a negotiation request with identical PrevLogTerm will continue to steady state");
}
// this (or the negotiation above) completes the negotiation process
_connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = $"Found a log index / term match at {negotiation.PrevLogIndex} with term {prevTerm}",
CurrentTerm = _term,
LastLogIndex = negotiation.PrevLogIndex
});
}
_debugRecorder.Record("Matching Negotiation is over, waiting for snapshot");
_engine.Timeout.Defer(_connection.Source);
// at this point, the leader will send us a snapshot message
// in most cases, it is an empty snapshot, then start regular append entries
// the reason we send this is to simplify the # of states in the protocol
var snapshot = _connection.ReadInstallSnapshot(context);
_debugRecorder.Record("Snapshot received");
using (context.OpenWriteTransaction())
{
var lastTerm = _engine.GetTermFor(context, snapshot.LastIncludedIndex);
var lastCommitIndex = _engine.GetLastEntryIndex(context);
if (snapshot.LastIncludedTerm == lastTerm && snapshot.LastIncludedIndex < lastCommitIndex)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info(
$"{ToString()}: Got installed snapshot with last index={snapshot.LastIncludedIndex} while our lastCommitIndex={lastCommitIndex}, will just ignore it");
}
//This is okay to ignore because we will just get the committed entries again and skip them
ReadInstallSnapshotAndIgnoreContent(context);
}
else if (InstallSnapshot(context))
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info(
$"{ToString()}: Installed snapshot with last index={snapshot.LastIncludedIndex} with LastIncludedTerm={snapshot.LastIncludedTerm} ");
}
_engine.SetLastCommitIndex(context, snapshot.LastIncludedIndex, snapshot.LastIncludedTerm);
_engine.ClearLogEntriesAndSetLastTruncate(context, snapshot.LastIncludedIndex, snapshot.LastIncludedTerm);
}
else
{
var lastEntryIndex = _engine.GetLastEntryIndex(context);
if (lastEntryIndex < snapshot.LastIncludedIndex)
{
var message =
$"The snapshot installation had failed because the last included index {snapshot.LastIncludedIndex} in term {snapshot.LastIncludedTerm} doesn't match the last entry {lastEntryIndex}";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: {message}");
}
throw new InvalidOperationException(message);
}
}
// snapshot always has the latest topology
if (snapshot.Topology == null)
{
const string message = "Expected to get topology on snapshot";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: {message}");
}
throw new InvalidOperationException(message);
}
using (var topologyJson = context.ReadObject(snapshot.Topology, "topology"))
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: topology on install snapshot: {topologyJson}");
}
var topology = JsonDeserializationRachis <ClusterTopology> .Deserialize(topologyJson);
RachisConsensus.SetTopology(_engine, context, topology);
}
context.Transaction.Commit();
}
_engine.Timeout.Defer(_connection.Source);
_debugRecorder.Record("Invoking StateMachine.SnapshotInstalled");
// notify the state machine, we do this in an async manner, and start
// the operator in a separate thread to avoid timeouts while this is
// going on
var task = Task.Run(() => _engine.SnapshotInstalledAsync(snapshot.LastIncludedIndex));
var sp = Stopwatch.StartNew();
var timeToWait = (int)(_engine.ElectionTimeout.TotalMilliseconds / 4);
while (task.Wait(timeToWait) == false)
{
// this may take a while, so we let the other side know that
// we are still processing, and we reset our own timer while
// this is happening
MaybeNotifyLeaderThatWeAreStillAlive(context, sp);
}
_debugRecorder.Record("Done with StateMachine.SnapshotInstalled");
// we might have moved from passive node, so we need to start the timeout clock
_engine.Timeout.Start(_engine.SwitchToCandidateStateOnTimeout);
_debugRecorder.Record("Snapshot installed");
//Here we send the LastIncludedIndex as our matched index even for the case where our lastCommitIndex is greater
//So we could validate that the entries sent by the leader are indeed the same as the ones we have.
_connection.Send(context, new InstallSnapshotResponse
{
Done = true,
CurrentTerm = _term,
LastLogIndex = snapshot.LastIncludedIndex
});
_engine.Timeout.Defer(_connection.Source);
}
private void ReadAndCommitSnapshot(TransactionOperationContext context, InstallSnapshot snapshot, CancellationToken token)
{
using (context.OpenWriteTransaction())
{
var lastTerm = _engine.GetTermFor(context, snapshot.LastIncludedIndex);
var lastCommitIndex = _engine.GetLastEntryIndex(context);
if (_engine.GetSnapshotRequest(context) == false &&
snapshot.LastIncludedTerm == lastTerm && snapshot.LastIncludedIndex < lastCommitIndex)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info(
$"{ToString()}: Got installed snapshot with last index={snapshot.LastIncludedIndex} while our lastCommitIndex={lastCommitIndex}, will just ignore it");
}
//This is okay to ignore because we will just get the committed entries again and skip them
ReadInstallSnapshotAndIgnoreContent(token);
}
else if (InstallSnapshot(context, token))
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info(
$"{ToString()}: Installed snapshot with last index={snapshot.LastIncludedIndex} with LastIncludedTerm={snapshot.LastIncludedTerm} ");
}
_engine.SetLastCommitIndex(context, snapshot.LastIncludedIndex, snapshot.LastIncludedTerm);
_engine.ClearLogEntriesAndSetLastTruncate(context, snapshot.LastIncludedIndex, snapshot.LastIncludedTerm);
}
else
{
var lastEntryIndex = _engine.GetLastEntryIndex(context);
if (lastEntryIndex < snapshot.LastIncludedIndex)
{
var message =
$"The snapshot installation had failed because the last included index {snapshot.LastIncludedIndex} in term {snapshot.LastIncludedTerm} doesn't match the last entry {lastEntryIndex}";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: {message}");
}
throw new InvalidOperationException(message);
}
}
// snapshot always has the latest topology
if (snapshot.Topology == null)
{
const string message = "Expected to get topology on snapshot";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: {message}");
}
throw new InvalidOperationException(message);
}
using (var topologyJson = context.ReadObject(snapshot.Topology, "topology"))
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: topology on install snapshot: {topologyJson}");
}
var topology = JsonDeserializationRachis <ClusterTopology> .Deserialize(topologyJson);
RachisConsensus.SetTopology(_engine, context, topology);
}
_engine.SetSnapshotRequest(context, false);
context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += t =>
{
if (t is LowLevelTransaction llt && llt.Committed)
{
// we might have moved from passive node, so we need to start the timeout clock
_engine.Timeout.Start(_engine.SwitchToCandidateStateOnTimeout);
}
};
context.Transaction.Commit();
}
}
private void NegotiateWithLeader(TransactionOperationContext context, LogLengthNegotiation negotiation)
{
// only the leader can send append entries, so if we accepted it, it's the leader
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"Follower {_engine.Tag}: Got a negotiation request for term {negotiation.Term} where our term is {_engine.CurrentTerm}");
}
if (negotiation.Term > _engine.CurrentTerm)
{
_engine.FoundAboutHigherTerm(negotiation.Term);
}
long prevTerm;
using (context.OpenReadTransaction())
{
prevTerm = _engine.GetTermFor(context, negotiation.PrevLogIndex) ?? 0;
}
if (prevTerm != negotiation.PrevLogTerm)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"Follower {_engine.Tag}: Got a negotiation request with PrevLogTerm={negotiation.PrevLogTerm} while our PrevLogTerm={prevTerm}" +
" will negotiate to find next matched index");
}
// we now have a mismatch with the log position, and need to negotiate it with
// the leader
NegotiateMatchEntryWithLeaderAndApplyEntries(context, _connection, negotiation);
}
else
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"Follower {_engine.Tag}: Got a negotiation request with identical PrevLogTerm will continue to steady state");
}
// this (or the negotiation above) completes the negotiation process
_connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = $"Found a log index / term match at {negotiation.PrevLogIndex} with term {prevTerm}",
CurrentTerm = _engine.CurrentTerm,
LastLogIndex = negotiation.PrevLogIndex
});
}
_engine.Timeout.Defer(_connection.Source);
// at this point, the leader will send us a snapshot message
// in most cases, it is an empty snapshot, then start regular append entries
// the reason we send this is to simplify the # of states in the protocol
var snapshot = _connection.ReadInstallSnapshot(context);
using (context.OpenWriteTransaction())
{
var lastCommitIndex = _engine.GetLastCommitIndex(context);
if (snapshot.LastIncludedIndex < lastCommitIndex)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info(
$"Follower {_engine.Tag}: Got installed snapshot with last index={snapshot.LastIncludedIndex} while our lastCommitIndex={lastCommitIndex}, will just ignore it");
}
//This is okay to ignore because we will just get the commited entries again and skip them
ReadInstallSnapshotAndIgnoreContent(context);
}
else if (InstallSnapshot(context))
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info(
$"Follower {_engine.Tag}: Installed snapshot with last index={snapshot.LastIncludedIndex} with LastIncludedTerm={snapshot.LastIncludedTerm} ");
}
_engine.SetLastCommitIndex(context, snapshot.LastIncludedIndex, snapshot.LastIncludedTerm);
_engine.ClearLogEntriesAndSetLastTruncate(context, snapshot.LastIncludedIndex, snapshot.LastIncludedTerm);
}
else
{
var lastEntryIndex = _engine.GetLastEntryIndex(context);
if (lastEntryIndex < snapshot.LastIncludedIndex)
{
var message =
$"The snapshot installation had failed because the last included index {snapshot.LastIncludedIndex} in term {snapshot.LastIncludedTerm} doesn't match the last entry {lastEntryIndex}";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"Follower {_engine.Tag}: {message}");
}
throw new InvalidOperationException(message);
}
}
// snapshot always has the latest topology
if (snapshot.Topology == null)
{
const string message = "Expected to get topology on snapshot";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"Follower {_engine.Tag}: {message}");
}
throw new InvalidOperationException(message);
}
using (var topologyJson = context.ReadObject(snapshot.Topology, "topology"))
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"Follower {_engine.Tag}: topology on install snapshot: {topologyJson}");
}
var topology = JsonDeserializationRachis <ClusterTopology> .Deserialize(topologyJson);
RachisConsensus.SetTopology(_engine, context, topology);
}
context.Transaction.Commit();
}
//Here we send the LastIncludedIndex as our matched index even for the case where our lastCommitIndex is greater
//So we could validate that the entries sent by the leader are indeed the same as the ones we have.
_connection.Send(context, new InstallSnapshotResponse
{
Done = true,
CurrentTerm = _engine.CurrentTerm,
LastLogIndex = snapshot.LastIncludedIndex
});
_engine.Timeout.Defer(_connection.Source);
// notify the state machine
_engine.SnapshotInstalled(context, snapshot.LastIncludedIndex);
_engine.Timeout.Defer(_connection.Source);
}
public override int HandleMap(LazyStringValue key, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
EnsureValidStats(stats);
var mappedResult = new DynamicJsonValue();
using (_stats.BlittableJsonAggregation.Start())
{
var document = ((Document[])mapResults)[0];
Debug.Assert(key == document.LoweredKey);
foreach (var indexField in Definition.MapFields.Values)
{
switch (indexField.MapReduceOperation)
{
case FieldMapReduceOperation.Count:
mappedResult[indexField.Name] = 1;
break;
case FieldMapReduceOperation.Sum:
object fieldValue;
StringSegment leftPath;
BlittableJsonTraverser.Default.TryRead(document.Data, indexField.Name, out fieldValue, out leftPath);
var arrayResult = fieldValue as IEnumerable <object>;
if (arrayResult == null)
{
// explicitly adding this even if the value isn't there, as a null
mappedResult[indexField.Name] = fieldValue;
continue;
}
decimal total = 0;
foreach (var item in arrayResult)
{
if (item == null)
{
continue;
}
double doubleValue;
long longValue;
switch (BlittableNumber.Parse(item, out doubleValue, out longValue))
{
case NumberParseResult.Double:
total += (decimal)doubleValue;
break;
case NumberParseResult.Long:
total += longValue;
break;
}
}
mappedResult[indexField.Name] = total;
break;
case FieldMapReduceOperation.None:
object result;
BlittableJsonTraverser.Default.TryRead(document.Data, indexField.Name, out result, out leftPath);
// explicitly adding this even if the value isn't there, as a null
mappedResult[indexField.Name] = result;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
_reduceKeyProcessor.Reset();
foreach (var groupByFieldName in Definition.GroupByFields.Keys)
{
object result;
StringSegment leftPath;
BlittableJsonTraverser.Default.TryRead(document.Data, groupByFieldName, out result, out leftPath);
// explicitly adding this even if the value isn't there, as a null
mappedResult[groupByFieldName] = result;
_reduceKeyProcessor.Process(indexContext.Allocator, result);
}
}
BlittableJsonReaderObject mr;
using (_stats.CreateBlittableJson.Start())
mr = indexContext.ReadObject(mappedResult, key);
var mapResult = _singleOutputList[0];
mapResult.Data = mr;
mapResult.ReduceKeyHash = _reduceKeyProcessor.Hash;
var resultsCount = PutMapResults(key, _singleOutputList, indexContext, stats);
DocumentDatabase.Metrics.MapReduceMappedPerSecond.Mark(resultsCount);
return(resultsCount);
}
