public void CanDetectAndSuggestOptionsForConflict_SimpleProp()
{
using (var ctx = new JsonOperationContext(4096, 16 * 1024, SharedMultipleUseFlag.None))
{
DynamicJsonValue obj1 = new DynamicJsonValue();
DynamicJsonValue obj2 = new DynamicJsonValue();
obj1["Name"] = "Oren";
obj2["Name"] = "Ayende";
var conflictResovlerAdvisor = new ConflictResolverAdvisor(
new List <BlittableJsonReaderObject> {
ctx.ReadObject(obj1, "doc/1"), ctx.ReadObject(obj2, "doc/1")
},
ctx);
var resolvled = conflictResovlerAdvisor.Resolve().Document;
BlittableJsonReaderArray name;
resolvled.TryGet("Name", out name);
Assert.Equal(">>>> conflict start", name[0].ToString());
Assert.Equal("Oren", name[1].ToString());
Assert.Equal("Ayende", name[2].ToString());
Assert.Equal("<<<< conflict end", name[3].ToString());
}
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/timeseries?docId={Uri.EscapeDataString(_documentId)}";
return(new HttpRequestMessage
{
Method = HttpMethod.Post,
Content = new BlittableJsonContent(async stream =>
{
var op = ctx.ReadObject(_operation.ToJson(), "convert-time-series-operation");
await ctx.WriteAsync(stream, op).ConfigureAwait(false);
})
});
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/timeseries/names/config";
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
Content = new BlittableJsonContent(stream =>
{
var config = ctx.ReadObject(_parameters.ToJson(), "convert time-series configuration");
ctx.Write(stream, config);
})
};
return(request);
}
private static string GetTreeName(BlittableJsonReaderObject reduceEntry, IndexDefinitionBase indexDefinition, JsonOperationContext context)
{
HashSet <CompiledIndexField> groupByFields;
if (indexDefinition is MapReduceIndexDefinition)
{
groupByFields = ((MapReduceIndexDefinition)indexDefinition).GroupByFields;
}
else if (indexDefinition is AutoMapReduceIndexDefinition)
{
groupByFields = ((AutoMapReduceIndexDefinition)indexDefinition).GroupByFields.Keys
.Select(x => (CompiledIndexField) new SimpleField(x))
.ToHashSet();
}
else
{
throw new InvalidOperationException("Invalid map reduce index definition: " + indexDefinition.GetType());
}
foreach (var prop in reduceEntry.GetPropertyNames())
{
var skip = false;
foreach (var groupByField in groupByFields)
{
if (groupByField.IsMatch(prop))
{
skip = true;
break;
}
}
if (skip)
{
continue;
}
if (reduceEntry.Modifications == null)
{
reduceEntry.Modifications = new DynamicJsonValue(reduceEntry);
}
reduceEntry.Modifications.Remove(prop);
}
var reduceKey = context.ReadObject(reduceEntry, "debug: creating reduce tree name");
return(reduceKey.ToString());
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/admin/timeseries/policy?collection={Uri.EscapeDataString(_collection)}";
var request = new HttpRequestMessage
{
Method = HttpMethod.Put,
Content = new BlittableJsonContent(async stream =>
{
var config = ctx.ReadObject(_configuration.ToJson(), "convert time-series policy");
await ctx.WriteAsync(stream, config).ConfigureAwait(false);
})
};
return(request);
}
public override BlittableJsonReaderObject GetUpdatedValue(JsonOperationContext context, BlittableJsonReaderObject previousValue, long index)
{
if (Value == null)
{
throw new RachisApplyException($"{nameof(Value)} cannot be null");
}
if (Value.DetailsPerNode == null || Value.NodeTag == null || Value.AllNodes == null)
{
throw new RachisApplyException($"{nameof(Value.DetailsPerNode)}, {nameof(Value.NodeTag)}, {nameof(Value.AllNodes)} cannot be null");
}
if (previousValue == null)
{
throw new RachisApplyException("Cannot apply node details to a non existing license limits");
}
var licenseLimits = JsonDeserializationServer.LicenseLimits(previousValue);
if (licenseLimits.NodeLicenseDetails.TryGetValue(Value.NodeTag, out var currentDetailsPerNode) == false)
{
return(null);
}
if (Value.DetailsPerNode.NumberOfCores == currentDetailsPerNode.NumberOfCores)
{
// the number of cores on this node hasn't changed, keep the old value
Value.DetailsPerNode.UtilizedCores = currentDetailsPerNode.UtilizedCores;
}
else
{
var utilizedCores = licenseLimits.NodeLicenseDetails.Sum(x => x.Value.UtilizedCores);
var availableCoresToDistribute = Value.LicensedCores - utilizedCores + currentDetailsPerNode.UtilizedCores;
// nodes that aren't in yet in license limits, need to "reserve" cores for them
var nodesThatArentRegistered = licenseLimits.NodeLicenseDetails.Keys.Except(Value.AllNodes).Count();
availableCoresToDistribute -= nodesThatArentRegistered;
Value.DetailsPerNode.UtilizedCores = availableCoresToDistribute > 0
? Math.Min(Value.DetailsPerNode.NumberOfCores, availableCoresToDistribute)
: 1;
}
licenseLimits.NodeLicenseDetails[Value.NodeTag] = Value.DetailsPerNode;
return(context.ReadObject(licenseLimits.ToJson(), "update-license-limits"));
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/admin/tasks/pull-replication/hub/access?name={Uri.EscapeUriString(_hubName)}";
var blittable = ctx.ReadObject(_access.ToJson(), "register-access");
var request = new HttpRequestMessage
{
Method = HttpMethod.Put,
Content = new BlittableJsonContent(stream =>
{
blittable.WriteJsonTo(stream);
})
};
return(request);
}
public override IEnumerable <BlittableJsonReaderObject> GetOutputsToStore()
{
foreach (var output in _outputs)
{
var djv = new DynamicJsonValue();
foreach (var property in _propertyAccessor.PropertiesInOrder)
{
var value = property.Value.GetValue(output);
djv[property.Key] = TypeConverter.ToBlittableSupportedType(value);
}
var item = _indexContext.ReadObject(djv, "map/reduce result to store");
_jsons.Add(item);
yield return(item);
}
}
public Document create_doc(DynamicJsonValue document, string id)
{
var data = _ctx.ReadObject(document, id);
_docs.Add(data);
//_lazyStrings.
var lazyStringValueRegular = _ctx.GetLazyString(id);
var lazyStringValueLowerCase = _ctx.GetLazyString(id.ToLowerInvariant());
return(new Document
{
Data = data,
Id = lazyStringValueRegular,
LowerId = lazyStringValueLowerCase
});
}
public SingleNodeBatchCommand(DocumentConventions conventions, JsonOperationContext context, IList <ICommandData> commands, BatchOptions options = null, TransactionMode mode = TransactionMode.SingleNode)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (commands == null)
{
throw new ArgumentNullException(nameof(commands));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_commands = new BlittableJsonReaderObject[commands.Count];
for (var i = 0; i < commands.Count; i++)
{
var command = commands[i];
var json = command.ToJson(conventions, context);
_commands[i] = context.ReadObject(json, "command");
if (command is PutAttachmentCommandData putAttachmentCommandData)
{
if (_attachmentStreams == null)
{
_attachmentStreams = new List <Stream>();
_uniqueAttachmentStreams = new HashSet <Stream>();
}
var stream = putAttachmentCommandData.Stream;
PutAttachmentCommandHelper.ValidateStream(stream);
if (_uniqueAttachmentStreams.Add(stream) == false)
{
PutAttachmentCommandHelper.ThrowStreamWasAlreadyUsed();
}
_attachmentStreams.Add(stream);
}
}
_options = options;
_mode = mode;
Timeout = options?.RequestTimeout;
}
public void Can_serialize_completed_state_to_json()
{
var state = new OperationState
{
Status = OperationStatus.Completed,
Result = new SampleOperationResult
{
Message = "Done"
}
};
using (var context = new JsonOperationContext(1024, 1024, 32 * 1024, SharedMultipleUseFlag.None))
{
var json = context.ReadObject(state.ToJson(), "state");
var result = json["Result"] as BlittableJsonReaderObject;
Assert.NotNull(result);
Assert.Equal("Done", result["Message"].ToString());
}
}
public void Can_serialize_in_progress_state_to_json()
{
var state = new OperationState
{
Progress = new DeterminateProgress
{
Processed = 1,
Total = 100
}
};
using (var context = new JsonOperationContext(1024, 1024, 32 * 1024, SharedMultipleUseFlag.None))
{
var json = context.ReadObject(state.ToJson(), "state");
var progress = json["Progress"];
Assert.NotNull(progress);
Assert.Equal("InProgress", json["Status"].ToString());
}
}
private static Document ReturnProjection(DynamicJsonValue result, Document doc, float score, JsonOperationContext context)
{
var newData = context.ReadObject(result, "projection result");
try
{
doc.Data?.Dispose();
}
catch (Exception)
{
newData.Dispose();
throw;
}
doc.Data = newData;
doc.IndexScore = score;
return(doc);
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, bool isPassive)
{
var itemId = GetItemId();
if (existingValue == null)
{
throw new InvalidOperationException($"Subscription with id {itemId} does not exist");
}
if (record.Topology.WhoseTaskIsIt(this, isPassive) != NodeTag)
{
throw new InvalidOperationException($"Can't update subscription with name {itemId} by node {NodeTag}, because it's not it's task to update this subscription");
}
var subscription = JsonDeserializationCluster.SubscriptionState(existingValue);
subscription.LastClientConnectionTime = LastClientConnectionTime;
return(context.ReadObject(subscription.ToJson(), itemId));
}
public override DynamicJsonValue ToJson(JsonOperationContext context)
{
var djv = base.ToJson(context);
djv[nameof(ClusterCommands)] = new DynamicJsonArray(ClusterCommands.Select(x => x.ToJson(context)));
djv[nameof(SerializedDatabaseCommands)] = SerializedDatabaseCommands?.Clone(context);
if (SerializedDatabaseCommands == null && DatabaseCommands.Count > 0)
{
var databaseCommands = new DynamicJsonValue
{
[nameof(DatabaseCommands)] = new DynamicJsonArray(DatabaseCommands.Select(x => x.ToJson(context))),
[nameof(Options)] = Options.ToJson(),
};
djv[nameof(SerializedDatabaseCommands)] = context.ReadObject(databaseCommands, "read database commands");
}
djv[nameof(Database)] = Database;
djv[nameof(DatabaseCommandsCount)] = DatabaseCommandsCount;
return(djv);
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/admin/tasks/external-replication";
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
Content = new BlittableJsonContent(async stream =>
{
var json = new DynamicJsonValue
{
["Watcher"] = _newWatcher.ToJson()
};
await ctx.WriteAsync(stream, ctx.ReadObject(json, "update-replication")).ConfigureAwait(false);
})
};
return(request);
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/admin/tasks/sink-pull-replication";
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
Content = new BlittableJsonContent(stream =>
{
var json = new DynamicJsonValue
{
["PullReplicationAsSink"] = _pullReplication.ToJson()
};
ctx.Write(stream, ctx.ReadObject(json, "update-pull-replication"));
})
};
return(request);
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/cmpxchg?key={_key}&index={_index}";
var djv = new DynamicJsonValue
{
["Object"] = EntityToBlittable.ConvertToBlittableIfNeeded(_value, _conventions, ctx, _conventions.CreateSerializer(), documentInfo: null, removeIdentityProperty: false)
};
var blittable = ctx.ReadObject(djv, _key);
var request = new HttpRequestMessage
{
Method = HttpMethods.Put,
Content = new BlittableJsonContent(stream =>
{
ctx.Write(stream, blittable);
})
};
return(request);
}
public BatchCommand(DocumentConventions conventions, JsonOperationContext context, List <ICommandData> commands, BatchOptions options = null)
{
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (commands == null)
{
throw new ArgumentNullException(nameof(commands));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_commands = new BlittableJsonReaderObject[commands.Count];
for (var i = 0; i < commands.Count; i++)
{
var command = commands[i];
_commands[i] = context.ReadObject(command.ToJson(conventions, context), "command");
if (command is PutAttachmentCommandData putAttachmentCommandData)
{
if (_attachmentStreams == null)
{
_attachmentStreams = new HashSet <Stream>();
}
var stream = putAttachmentCommandData.Stream;
PutAttachmentCommandHelper.ValidateStream(stream);
if (_attachmentStreams.Add(stream) == false)
{
PutAttachmentCommandHelper.ThrowStreamAlready();
}
}
}
_options = options;
Timeout = options?.RequestTimeout;
}
public override BlittableJsonReaderObject GetUpdatedValue(JsonOperationContext context, BlittableJsonReaderObject previousValue, long index)
{
if (previousValue == null)
{
return(null);
}
var propertyIndex = previousValue.GetPropertyIndex(Value);
if (propertyIndex == -1)
{
return(null);
}
previousValue.Modifications ??= new DynamicJsonValue();
previousValue.Modifications.Removals = new HashSet <int> {
propertyIndex
};
return(context.ReadObject(previousValue, Name));
}
public static object ConvertResult(JsonOperationContext ctx, object result)
{
if (result is ValueTuple <long, object> tuple)
{
if (tuple.Item2 is BlittableJsonReaderObject value)
{
return(new CompareExchangeResult
{
Index = tuple.Item1,
Value = ctx.ReadObject(value, "cmpXchg result clone")
});
}
return(new CompareExchangeResult
{
Index = tuple.Item1,
Value = tuple.Item2
});
}
throw new RachisApplyException("Unable to convert result type: " + result?.GetType()?.FullName + ", " + result);
}
public static BlittableJsonReaderObject ConvertEntityToBlittable(object entity, DocumentConventions conventions, JsonOperationContext context, DocumentInfo documentInfo = null)
{
using (var writer = new BlittableJsonWriter(context, documentInfo))
{
var serializer = conventions.CreateSerializer();
serializer.Serialize(writer, entity);
writer.FinalizeDocument();
var reader = writer.CreateReader();
var type = entity.GetType();
var changes = TryRemoveIdentityProperty(reader, type, conventions);
changes |= TrySimplifyJson(reader);
if (changes)
{
reader = context.ReadObject(reader, "convert/entityToBlittable");
}
return(reader);
}
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, RachisState state)
{
EtlProcessState etlState;
if (existingValue != null)
{
etlState = JsonDeserializationClient.EtlProcessState(existingValue);
var databaseTask = GetMatchingConfiguration(record);
if (databaseTask == null)
{
throw new Exception($"Can't update progress of ETL {ConfigurationName} by node {NodeTag}, because it's configuration can't be found");
}
var lastResponsibleNode = GetLastResponsibleNode(HasHighlyAvailableTasks, record.Topology, NodeTag);
if (record.Topology.WhoseTaskIsIt(state, databaseTask, lastResponsibleNode) != NodeTag)
{
throw new Exception($"Can't update progress of ETL {ConfigurationName} by node {NodeTag}, because it's not its task to update this ETL");
}
}
else
{
etlState = new EtlProcessState
{
ConfigurationName = ConfigurationName,
TransformationName = TransformationName
};
}
etlState.LastProcessedEtagPerNode[NodeTag] = LastProcessedEtag;
etlState.ChangeVector = ChangeVector;
etlState.NodeTag = NodeTag;
return(context.ReadObject(etlState.ToJson(), GetItemId()));
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, RachisState state)
{
var itemId = GetItemId();
if (existingValue == null)
{
throw new RachisApplyException($"Subscription with id {itemId} does not exist");
}
var subscription = JsonDeserializationCluster.SubscriptionState(existingValue);
var lastResponsibleNode = AcknowledgeSubscriptionBatchCommand.GetLastResponsibleNode(HasHighlyAvailableTasks, record.Topology, NodeTag);
if (record.Topology.WhoseTaskIsIt(state, subscription, lastResponsibleNode) != NodeTag)
{
throw new RachisApplyException($"Can't update subscription with name {itemId} by node {NodeTag}, because it's not it's task to update this subscription");
}
subscription.LastClientConnectionTime = LastClientConnectionTime;
subscription.NodeTag = NodeTag;
return(context.ReadObject(subscription.ToJson(), itemId));
}
public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
{
url = $"{node.Url}/databases/{node.Database}/cmpxchg?key={Uri.EscapeDataString(_key)}&index={_index}";
var djv = new DynamicJsonValue
{
[Constants.CompareExchange.ObjectFieldName] = CompareExchangeValueBlittableJsonConverter.ConvertToBlittable(_value, _conventions, ctx)
};
if (_metadata != null)
{
var metadata = ClusterTransactionOperationsBase.CompareExchangeSessionValue.PrepareMetadataForPut(_key, _metadata, _conventions, ctx);
djv[Constants.Documents.Metadata.Key] = metadata;
}
var blittable = ctx.ReadObject(djv, _key);
var request = new HttpRequestMessage
{
Method = HttpMethods.Put,
Content = new BlittableJsonContent(async stream => await ctx.WriteAsync(stream, blittable).ConfigureAwait(false))
};
return(request);
}
private JsValue GetMetadata(JsValue self, JsValue[] args)
{
if (args.Length != 1 || !(args[0].AsObject() is BlittableObjectInstance boi))
{
throw new InvalidOperationException("getMetadata(doc) must be called with a single entity argument");
}
if (!(boi.Blittable[Constants.Documents.Metadata.Key] is BlittableJsonReaderObject metadata))
{
return(JsValue.Null);
}
metadata.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.ChangeVector] = boi.ChangeVector,
[Constants.Documents.Metadata.Id] = boi.DocumentId,
[Constants.Documents.Metadata.LastModified] = boi.LastModified,
};
metadata = _jsonCtx.ReadObject(metadata, boi.DocumentId);
return(TranslateToJs(ScriptEngine, _jsonCtx, metadata));
}
private static BlittableJsonReaderObject ConvertEntityToBlittableInternal(
object entity,
DocumentConventions conventions,
JsonOperationContext context,
JsonSerializer serializer,
BlittableJsonWriter writer)
{
serializer.Serialize(writer, entity);
writer.FinalizeDocument();
var reader = writer.CreateReader();
var type = entity.GetType();
var changes = TryRemoveIdentityProperty(reader, type, conventions);
changes |= TrySimplifyJson(reader);
if (changes)
{
reader = context.ReadObject(reader, "convert/entityToBlittable");
}
return(reader);
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, bool isPassive)
{
EtlProcessState state;
if (existingValue != null)
{
state = JsonDeserializationClient.EtlProcessState(existingValue);
}
else
{
state = new EtlProcessState
{
ConfigurationName = ConfigurationName,
TransformationName = TransformationName
};
}
state.LastProcessedEtagPerNode[NodeTag] = LastProcessedEtag;
state.ChangeVector = ChangeVector;
return(context.ReadObject(state.ToJson(), GetItemId()));
}
public override BlittableJsonReaderObject GetUpdatedValue(JsonOperationContext context, BlittableJsonReaderObject previousValue, long index)
{
if (previousValue == null)
{
throw new RachisApplyException($"Cannot find any server wide tasks of type '{Value.Type}'");
}
if (previousValue.TryGet(Value.TaskName, out BlittableJsonReaderObject task) == false)
{
throw new RachisApplyException($"Cannot find server wide task of type '{Value.Type}' with name '{Value.TaskName}'");
}
if (task.Modifications == null)
{
task.Modifications = new DynamicJsonValue();
}
task.Modifications = new DynamicJsonValue
{
[GetDisabledPropertyName()] = Value.Disable
};
return(context.ReadObject(previousValue, Name));
}
private static string GetTreeName(BlittableJsonReaderObject reduceEntry, IndexDefinitionBase indexDefinition, JsonOperationContext context)
{
HashSet <string> groupByFields;
if (indexDefinition is MapReduceIndexDefinition)
{
groupByFields = ((MapReduceIndexDefinition)indexDefinition).GroupByFields;
}
else if (indexDefinition is AutoMapReduceIndexDefinition)
{
groupByFields = ((AutoMapReduceIndexDefinition)indexDefinition).GroupByFields.Keys.ToHashSet();
}
else
{
throw new InvalidOperationException("Invalid map reduce index definition: " + indexDefinition.GetType());
}
foreach (var prop in reduceEntry.GetPropertyNames())
{
if (groupByFields.Contains(prop))
{
continue;
}
if (reduceEntry.Modifications == null)
{
reduceEntry.Modifications = new DynamicJsonValue(reduceEntry);
}
reduceEntry.Modifications.Remove(prop);
}
var reduceKey = context.ReadObject(reduceEntry, "debug: creating reduce tree name");
return(reduceKey.ToString());
}
