public static void WriteMetadata(this BlittableJsonTextWriter writer, Document document, BlittableJsonReaderObject metadata)
{
writer.WritePropertyName(Constants.Documents.Metadata.Key);
writer.WriteStartObject();
bool first = true;
if (metadata != null)
{
var size = metadata.Count;
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < size; i++)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
metadata.GetPropertyByIndex(i, ref prop);
writer.WritePropertyName(prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
}
if (first == false)
{
writer.WriteComma();
}
writer.WritePropertyName(Constants.Documents.Metadata.ChangeVector);
writer.WriteString(document.ChangeVector);
if (document.Flags != DocumentFlags.None)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Flags);
writer.WriteString(document.Flags.ToString());
}
if (document.Id != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Id);
writer.WriteString(document.Id);
}
if (document.IndexScore != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.IndexScore);
writer.WriteDouble(document.IndexScore.Value);
}
if (document.LastModified != DateTime.MinValue)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.LastModified);
writer.WriteDateTime(document.LastModified, isUtc: true);
}
writer.WriteEndObject();
}
public override int Execute(DocumentsOperationContext context)
{
var hiLoDocumentId = RavenIdGeneratorsHilo + Key;
var prefix = Key + Separator;
long oldMax = 0;
var newDoc = new DynamicJsonValue();
BlittableJsonReaderObject hiloDocReader = null;
try
{
try
{
hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data;
}
catch (DocumentConflictException e)
{
throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " +
"This shouldn't happen, since it this conflict should've been resolved during replication. " +
"This exception should not happen and is likely a bug.", e);
}
if (hiloDocReader != null)
{
var prop = new BlittableJsonReaderObject.PropertyDetails();
foreach (var propertyId in hiloDocReader.GetPropertiesByInsertionOrder())
{
hiloDocReader.GetPropertyByIndex(propertyId, ref prop);
if (prop.Name == "Max")
{
oldMax = (long)prop.Value;
continue;
}
newDoc[prop.Name] = prop.Value;
}
}
oldMax = Math.Max(oldMax, LastRangeMax);
newDoc["Max"] = oldMax + Capacity;
using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);
OldMax = oldMax;
Prefix = prefix;
}
finally
{
hiloDocReader?.Dispose();
}
return(1);
}
private void Init()
{
_metadata = new Dictionary <string, object>();
var indexes = _source.GetPropertiesByInsertionOrder();
foreach (var index in indexes)
{
var propDetails = new BlittableJsonReaderObject.PropertyDetails();
_source.GetPropertyByIndex(index, ref propDetails);
_metadata[propDetails.Name] = ConvertValue(propDetails.Value);
}
}
public void Init()
{
_metadata = new Dictionary <string, string>();
var indexes = _source.GetPropertiesByInsertionOrder();
foreach (var index in indexes)
{
var propDetails = new BlittableJsonReaderObject.PropertyDetails();
_source.GetPropertyByIndex(index, ref propDetails);
_metadata[propDetails.Name] = propDetails.Value.ToString();
}
;
}
private void WriteObject(BlittableJsonReaderObject obj)
{
var propDetails = new BlittableJsonReaderObject.PropertyDetails();
_manualBlittableJsonDocumentBuilder.StartWriteObject();
for (int i = 0; i < obj.Count; i++)
{
obj.GetPropertyByIndex(i, ref propDetails);
_manualBlittableJsonDocumentBuilder.WritePropertyName(propDetails.Name);
WritePropertyValue(propDetails);
}
_manualBlittableJsonDocumentBuilder.WriteObjectEnd();
}
private void Init()
{
_metadata = new Dictionary <string, object>();
for (int i = 0; i < _source.Count; i++)
{
var propDetails = new BlittableJsonReaderObject.PropertyDetails();
_source.GetPropertyByIndex(i, ref propDetails);
_metadata[propDetails.Name] = ConvertValue(propDetails.Name, propDetails.Value);
}
if (_parent != null) // mark parent as dirty
{
_parent[_parentKey] = this;
}
}
public static void FetchColumnNames(BlittableJsonReaderObject data, HashSet <LazyStringValue> columns, BlittableJsonReaderObject.PropertiesInsertionBuffer buffers)
{
var size = data.GetPropertiesByInsertionOrder(buffers);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < size; i++)
{
data.GetPropertyByIndex(buffers.Properties[i], ref prop);
var propName = prop.Name;
if (columns.Contains(propName) == false)
{
columns.Add(prop.Name);
}
}
}
private void WriteObject(BlittableJsonReaderObject obj)
{
var propDetails = new BlittableJsonReaderObject.PropertyDetails();
_manualBlittableJsonDocumentBuilder.StartWriteObject();
var propsIndexes = obj.GetPropertiesByInsertionOrder();
foreach (var index in propsIndexes)
{
obj.GetPropertyByIndex(index, ref propDetails);
_manualBlittableJsonDocumentBuilder.WritePropertyName(propDetails.Name);
WritePropertyValue(propDetails);
}
_manualBlittableJsonDocumentBuilder.WriteObjectEnd();
}
public static void FetchColumnNames(BlittableJsonReaderObject data, HashSet <LazyStringValue> columns)
{
using (var buffers = data.GetPropertiesByInsertionOrder())
{
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < buffers.Properties.Count; i++)
{
data.GetPropertyByIndex(buffers.Properties.Array[i + buffers.Properties.Offset], ref prop);
var propName = prop.Name;
if (columns.Contains(propName) == false)
{
columns.Add(prop.Name);
}
}
}
}
public ObjectInstance ToJsObject(Engine engine, BlittableJsonReaderObject json, string propertyName = null)
{
var jsObject = engine.Object.Construct(Arguments.Empty);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < json.Count; i++)
{
json.GetPropertyByIndex(i, ref prop);
var name = prop.Name.ToString();
var propertyKey = CreatePropertyKey(name, propertyName);
var value = prop.Value;
JsValue jsValue = ToJsValue(engine, value, prop.Token, propertyKey);
_propertiesByValue[propertyKey] = new KeyValuePair <object, JsValue>(value, jsValue);
jsObject.FastAddProperty(name, jsValue, true, true, true);
}
return(jsObject);
}
private unsafe static void ValidateProperties(List <string> properties, BlittableJsonReaderObject json)
{
var propDetails = new BlittableJsonReaderObject.PropertyDetails();
var propertiesByInsertionOrder = json.GetPropertiesByInsertionOrder();
for (var i = 0; i < properties.Count; i++)
{
json.GetPropertyByIndex(propertiesByInsertionOrder.Properties[i], ref propDetails);
if (propDetails.Name == Constants.Documents.Metadata.Key)
{
continue;
}
var expected = properties[i];
Assert.Equal(expected, propDetails.Name);
}
}
private static Dictionary <string, string> ConvertToAdditionalSources(BlittableJsonReaderObject json)
{
if (json == null || json.Count == 0)
{
return(null);
}
var result = new Dictionary <string, string>();
BlittableJsonReaderObject.PropertyDetails propertyDetails = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < json.Count; i++)
{
json.GetPropertyByIndex(i, ref propertyDetails);
result[propertyDetails.Name] = propertyDetails.Value?.ToString();
}
return(result);
}
internal ClassType GenerateClassTypesFromObject(string name, BlittableJsonReaderObject blittableObject)
{
var fields = new Dictionary <string, FieldType>();
for (var i = 0; i < blittableObject.Count; i++)
{
// this call ensures properties to be returned in the same order, regardless their storing order
var prop = new BlittableJsonReaderObject.PropertyDetails();
blittableObject.GetPropertyByIndex(i, ref prop);
if (prop.Name.ToString().Equals("@metadata", StringComparison.OrdinalIgnoreCase))
{
continue;
}
switch (prop.Token)
{
case BlittableJsonToken.EmbeddedBlittable:
case BlittableJsonToken.StartObject:
case BlittableJsonToken.StartObject | BlittableJsonToken.OffsetSizeByte | BlittableJsonToken.PropertyIdSizeByte:
var type = GenerateClassTypesFromObject(prop.Name, (BlittableJsonReaderObject)prop.Value);
fields[prop.Name] = new FieldType(type.Name, type.IsArray);
break;
case BlittableJsonToken.StartArray:
case BlittableJsonToken.StartArray | BlittableJsonToken.OffsetSizeByte:
case BlittableJsonToken.StartArray | BlittableJsonToken.OffsetSizeShort:
var array = (BlittableJsonReaderArray)prop.Value;
fields[prop.Name] = GetArrayField(array, prop.Name);
break;
default:
fields[prop.Name] = GetTokenTypeFromPrimitiveType(prop.Token, prop.Value);
break;
}
}
// check if we can get the name from the metadata.
var clazz = new ClassType(name, fields);
clazz = IncludeGeneratedClass(clazz);
return(clazz);
}
public void FetchFields(BlittableJsonReaderObject data, Dictionary <LazyStringValue, FieldType> fields, BlittableJsonReaderObject.PropertiesInsertionBuffer buffers)
{
var size = data.GetPropertiesByInsertionOrder(buffers);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < size; i++)
{
data.GetPropertyByIndex(buffers.Properties[i], ref prop);
var type = GetFieldType(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
if (fields.TryGetValue(prop.Name, out var token))
{
if (token != type)
{
fields[prop.Name] = token | type;
}
}
else
{
fields[prop.Name] = type;
}
}
}
private bool ReadNestedObjects(BlittableJsonReaderObject nested, StringSegment path, out object result, out StringSegment leftPath)
{
var propCount = nested.Count;
var prop = new BlittableJsonReaderObject.PropertyDetails();
List <object> results = null;
leftPath = path; // note that we assume identical sturcutre of all values in this document
void AddItemToResults(object i)
{
if (results == null)
{
results = new List <object>();
}
results.Add(i);
}
for (int i = 0; i < propCount; i++)
{
nested.GetPropertyByIndex(i, ref prop);
if (prop.Value is BlittableJsonReaderObject nestedVal)
{
if (TryRead(nestedVal, path, out var item, out leftPath))
{
AddItemToResults(item);
}
else
{
item = item ?? nested;
if (BlittableJsonTraverserHelper.TryReadComputedProperties(this, leftPath, ref item))
{
leftPath = string.Empty;// consumed entirely
AddItemToResults(item);
}
}
}
}
public override void Execute(DocumentsOperationContext context, RavenTransaction tx)
{
var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key;
var prefix = Key + Separator;
long oldMax = 0;
var newDoc = new DynamicJsonValue();
BlittableJsonReaderObject hiloDocReader = null, serverPrefixDocReader = null;
try
{
try
{
serverPrefixDocReader = Database.DocumentsStorage.Get(context, RavenKeyServerPrefix)?.Data;
hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentKey)?.Data;
}
catch (DocumentConflictException e)
{
throw new InvalidDataException(@"Failed to fetch HiLo document due to a conflict
on the document. This shouldn't happen, since
it this conflict should've been resolved during replication.
This exception should not happen and is likely a bug.", e);
}
string serverPrefix;
if (serverPrefixDocReader != null &&
serverPrefixDocReader.TryGet("ServerPrefix", out serverPrefix))
{
prefix += serverPrefix;
}
if (hiloDocReader != null)
{
hiloDocReader.TryGet("Max", out oldMax);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < hiloDocReader.Count; i++)
{
hiloDocReader.GetPropertyByIndex(0, ref prop);
if (prop.Name == "Max")
{
continue;
}
newDoc[prop.Name] = prop.Value;
}
}
}
finally
{
serverPrefixDocReader?.Dispose();
hiloDocReader?.Dispose();
}
oldMax = Math.Max(oldMax, LastRangeMax);
newDoc["Max"] = oldMax + Capacity;
using (
var freshHilo = context.ReadObject(newDoc, hiLoDocumentKey,
BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, freshHilo);
}
OldMax = oldMax;
Prefix = prefix;
}
private static unsafe bool CompareBlittable(string fieldPath, string id, BlittableJsonReaderObject originalBlittable,
BlittableJsonReaderObject newBlittable, IDictionary <string, DocumentsChanges[]> changes,
List <DocumentsChanges> docChanges)
{
BlittableJsonReaderObject.AssertNoModifications(originalBlittable, id, assertChildren: false);
BlittableJsonReaderObject.AssertNoModifications(newBlittable, id, assertChildren: false);
var newBlittableProps = newBlittable.GetPropertyNames();
var oldBlittableProps = originalBlittable.GetPropertyNames();
var newFields = new HashSet <string>(newBlittableProps);
newFields.ExceptWith(oldBlittableProps);
var removedFields = new HashSet <string>(oldBlittableProps);
removedFields.ExceptWith(newBlittableProps);
using var orderedProperties = newBlittable.GetPropertiesByInsertionOrder();
foreach (var field in removedFields)
{
if (changes == null)
{
return(true);
}
if (field.Equals(LastModified) ||
field.Equals(ChangeVector) ||
field.Equals(Id))
{
continue;
}
NewChange(fieldPath, field, null, null, docChanges, DocumentsChanges.ChangeType.RemovedField);
}
var newProp = new BlittableJsonReaderObject.PropertyDetails();
var oldProp = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < orderedProperties.Size; i++)
{
newBlittable.GetPropertyByIndex(orderedProperties.Properties[i], ref newProp);
if (newProp.Name.Equals(LastModified) ||
newProp.Name.Equals(Collection) ||
newProp.Name.Equals(ChangeVector) ||
newProp.Name.Equals(Id))
{
continue;
}
if (newFields.Contains(newProp.Name))
{
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, null, docChanges, DocumentsChanges.ChangeType.NewField);
continue;
}
var oldPropId = originalBlittable.GetPropertyIndex(newProp.Name);
originalBlittable.GetPropertyByIndex(oldPropId, ref oldProp);
switch ((newProp.Token & BlittableJsonReaderBase.TypesMask))
{
case BlittableJsonToken.Integer:
case BlittableJsonToken.Boolean:
case BlittableJsonToken.LazyNumber:
case BlittableJsonToken.CompressedString:
case BlittableJsonToken.String:
if (newProp.Value.Equals(oldProp.Value) || CompareValues(oldProp, newProp) ||
CompareStringsWithEscapePositions(newBlittable._context, oldProp, newProp))
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.Null:
if (oldProp.Value == null)
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, null, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.StartArray:
var newArray = newProp.Value as BlittableJsonReaderArray;
var oldArray = oldProp.Value as BlittableJsonReaderArray;
if (newArray == null)
{
throw new InvalidDataException($"Invalid blittable, expected array but got {newProp.Value}");
}
if (oldArray == null)
{
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
}
var changed = CompareBlittableArray(FieldPathCombine(fieldPath, newProp.Name), id, oldArray, newArray, changes, docChanges, newProp.Name);
if (changes == null && changed)
{
return(true);
}
break;
case BlittableJsonToken.StartObject:
if (oldProp.Value == null ||
!(oldProp.Value is BlittableJsonReaderObject oldObj))
{
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
}
if (!(newProp.Value is BlittableJsonReaderObject newObj))
{
throw new InvalidDataException($"Invalid blittable, expected object but got {newProp.Value}");
}
changed = CompareBlittable(FieldPathCombine(fieldPath, newProp.Name), id, oldObj, newObj, changes, docChanges);
if (changes == null && changed)
{
return(true);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
if ((changes == null) || (docChanges.Count <= 0))
{
return(false);
}
changes[id] = docChanges.ToArray();
return(true);
}
private static bool CompareBlittable(string id, BlittableJsonReaderObject originalBlittable,
BlittableJsonReaderObject newBlittable, IDictionary <string, DocumentsChanges[]> changes,
List <DocumentsChanges> docChanges)
{
BlittableJsonReaderObject.AssertNoModifications(originalBlittable, id, assertChildren: false);
BlittableJsonReaderObject.AssertNoModifications(newBlittable, id, assertChildren: false);
var newBlittableProps = newBlittable.GetPropertyNames();
var oldBlittableProps = originalBlittable.GetPropertyNames();
var newFields = newBlittableProps.Except(oldBlittableProps);
var removedFields = oldBlittableProps.Except(newBlittableProps);
var propertiesIds = newBlittable.GetPropertiesByInsertionOrder();
foreach (var field in removedFields)
{
if (changes == null)
{
return(true);
}
NewChange(field, null, null, docChanges, DocumentsChanges.ChangeType.RemovedField);
}
var newProp = new BlittableJsonReaderObject.PropertyDetails();
var oldProp = new BlittableJsonReaderObject.PropertyDetails();
foreach (var propId in propertiesIds)
{
newBlittable.GetPropertyByIndex(propId, ref newProp);
if (newProp.Name.Equals(LastModified) ||
newProp.Name.Equals(Collection) ||
newProp.Name.Equals(ChangeVector) ||
newProp.Name.Equals(Id))
{
continue;
}
if (newFields.Contains(newProp.Name))
{
if (changes == null)
{
return(true);
}
NewChange(newProp.Name, newProp.Value, null, docChanges, DocumentsChanges.ChangeType.NewField);
continue;
}
var oldPropId = originalBlittable.GetPropertyIndex(newProp.Name);
originalBlittable.GetPropertyByIndex(oldPropId, ref oldProp);
switch ((newProp.Token & BlittableJsonReaderBase.TypesMask))
{
case BlittableJsonToken.Integer:
case BlittableJsonToken.Boolean:
case BlittableJsonToken.LazyNumber:
case BlittableJsonToken.CompressedString:
case BlittableJsonToken.String:
if (newProp.Value.Equals(oldProp.Value) || ComapreValues(oldProp, newProp))
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.Null:
if (oldProp.Value == null)
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(newProp.Name, null, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.StartArray:
var newArray = newProp.Value as BlittableJsonReaderArray;
var oldArray = oldProp.Value as BlittableJsonReaderArray;
if ((newArray == null) || (oldArray == null))
{
throw new InvalidDataException("Invalid blittable");
}
var changed = CompareBlittableArray(id, oldArray, newArray, changes, docChanges, newProp.Name);
if (changed == false)
{
break;
}
if (changes == null)
{
return(true);
}
break;
case BlittableJsonToken.StartObject:
if (oldProp.Value == null)
{
if (changes == null)
{
return(true);
}
changed = true;
NewChange(newProp.Name, newProp.Value, null, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
}
else
{
changed = CompareBlittable(id, oldProp.Value as BlittableJsonReaderObject,
newProp.Value as BlittableJsonReaderObject, changes, docChanges);
}
if ((changes == null) && (changed))
{
return(true);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
if ((changes == null) || (docChanges.Count <= 0))
{
return(false);
}
changes[id] = docChanges.ToArray();
return(true);
}
private static void WriteMetadata(BlittableJsonTextWriter writer, Document document, BlittableJsonReaderObject metadata, Func <LazyStringValue, bool> filterMetadataProperty = null)
{
writer.WritePropertyName(Constants.Documents.Metadata.Key);
writer.WriteStartObject();
bool first = true;
if (metadata != null)
{
var size = metadata.Count;
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < size; i++)
{
metadata.GetPropertyByIndex(i, ref prop);
if (filterMetadataProperty != null && filterMetadataProperty(prop.Name))
{
continue;
}
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName((string)prop.Name);
writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
}
}
if (first == false)
{
writer.WriteComma();
}
writer.WritePropertyName(Constants.Documents.Metadata.ChangeVector);
writer.WriteString(document.ChangeVector);
if (document.Flags != DocumentFlags.None)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Flags);
writer.WriteString(document.Flags.ToString());
}
if (document.Id != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.Id);
writer.WriteString(document.Id);
}
if (document.IndexScore != null)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.IndexScore);
writer.WriteDouble(document.IndexScore.Value);
}
if (document.Distance != null)
{
writer.WriteComma();
var result = document.Distance.Value;
writer.WritePropertyName(Constants.Documents.Metadata.SpatialResult);
writer.WriteStartObject();
writer.WritePropertyName(nameof(result.Distance));
writer.WriteDouble(result.Distance);
writer.WriteComma();
writer.WritePropertyName(nameof(result.Latitude));
writer.WriteDouble(result.Latitude);
writer.WriteComma();
writer.WritePropertyName(nameof(result.Longitude));
writer.WriteDouble(result.Longitude);
writer.WriteEndObject();
}
if (document.LastModified != DateTime.MinValue)
{
writer.WriteComma();
writer.WritePropertyName(Constants.Documents.Metadata.LastModified);
writer.WriteDateTime(document.LastModified, isUtc: true);
}
writer.WriteEndObject();
}