public void Invalid_hash_calculation_on_null()
{
using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("RavenDB_9535"))
using (var bsc = new ByteStringContext(SharedMultipleUseFlag.None))
{
var sut = new ReduceKeyProcessor(1, bufferPool);
try
{
sut.Reset();
sut.Process(bsc, null);
Assert.Equal((ulong)0, sut.Hash);
sut.Reset();
sut.Process(bsc, 1);
Assert.NotEqual((ulong)0, sut.Hash);
sut.Reset();
sut.Process(bsc, null);
Assert.Equal((ulong)0, sut.Hash);
}
finally
{
sut.ReleaseBuffer();
}
}
}
public void SetBufferPoolForTestingPurposes(UnmanagedBuffersPoolWithLowMemoryHandling bufferPool)
{
if (ReduceOperation != null)
{
ReduceOperation.SetBufferPoolForTestingPurposes(bufferPool);
}
}
public CurrentIndexingScope(DocumentsStorage documentsStorage, DocumentsOperationContext documentsContext, IndexDefinitionBase indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool)
{
_documentsStorage = documentsStorage;
_documentsContext = documentsContext;
UnmanagedBuffersPool = _unmanagedBuffersPool;
IndexDefinition = indexDefinition;
IndexContext = indexContext;
_getSpatialField = getSpatialField;
}
public CurrentIndexingScope(Index index, DocumentsStorage documentsStorage, QueryOperationContext queryContext, IndexDefinitionBaseServerSide indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool)
{
_documentsStorage = documentsStorage;
QueryContext = queryContext;
Index = index;
UnmanagedBuffersPool = _unmanagedBuffersPool;
IndexDefinition = indexDefinition;
IndexContext = indexContext;
_getSpatialField = getSpatialField;
}
public GroupBykeyComparer(JavaScriptReduceOperation parent, UnmanagedBuffersPoolWithLowMemoryHandling buffersPool, ByteStringContext allocator)
{
_parent = parent;
_allocator = allocator;
_xKey = new ReduceKeyProcessor(_parent._groupByFields.Count(), buffersPool);
_yKey = new ReduceKeyProcessor(_parent._groupByFields.Count(), buffersPool);
_xKey.SetMode(ReduceKeyProcessor.Mode.MultipleValues);
_yKey.SetMode(ReduceKeyProcessor.Mode.MultipleValues);
_lastUsedBlittable = null;
_lastUsedBucket = null;
}
public void SerialAllocationAndRelease()
{
using (var pool = new UnmanagedBuffersPoolWithLowMemoryHandling(string.Empty))
{
var allocatedMemory = new List <AllocatedMemoryData>();
for (var i = 0; i < 1000; i++)
{
allocatedMemory.Add(pool.Allocate(i));
}
foreach (var data in allocatedMemory)
{
pool.Return(data);
}
}
}
public ReduceKeyProcessor(int numberOfReduceFields, UnmanagedBuffersPoolWithLowMemoryHandling buffersPool)
{
_numberOfReduceFields = numberOfReduceFields;
_buffersPool = buffersPool;
if (numberOfReduceFields == 1)
{
_mode = Mode.SingleValue;
}
else
{
_mode = Mode.MultipleValues;
_buffer = _buffersPool.Allocate(16);
_bufferPos = 0;
}
}
public ReduceKeyProcessor(int numberOfReduceFields, UnmanagedBuffersPoolWithLowMemoryHandling buffersPool)
{
_numberOfReduceFields = numberOfReduceFields;
_buffersPool = buffersPool;
if (numberOfReduceFields == 1)
{
_mode = Mode.SingleValue;
}
else
{
// numberOfReduceFields could be zero when we have 'group bankTotal by 1'
_mode = Mode.MultipleValues;
_bufferPos = 0;
}
}
public void ParallelAllocationAndReleaseSeperately()
{
using (var pool = new UnmanagedBuffersPoolWithLowMemoryHandling(string.Empty))
{
var allocatedMemory = new global::Sparrow.Collections.ConcurrentSet <AllocatedMemoryData>();
Parallel.For(0, 100, RavenTestHelper.DefaultParallelOptions, x =>
{
for (var i = 0; i < 10; i++)
{
allocatedMemory.Add(pool.Allocate(i));
}
});
Parallel.ForEach(allocatedMemory, RavenTestHelper.DefaultParallelOptions, item =>
{
pool.Return(item);
});
}
}
public SubscriptionStorage(DocumentDatabase db)
{
var options = db.Configuration.Core.RunInMemory
? StorageEnvironmentOptions.CreateMemoryOnly(Path.Combine(db.Configuration.Core.DataDirectory, "Subscriptions"))
: StorageEnvironmentOptions.ForPath(Path.Combine(db.Configuration.Core.DataDirectory, "Subscriptions"));
options.SchemaVersion = 1;
options.TransactionsMode = TransactionsMode.Lazy;
_environment = new StorageEnvironment(options);
var databaseName = db.Name;
_unmanagedBuffersPool = new UnmanagedBuffersPoolWithLowMemoryHandling("Subscriptions", databaseName);
_logger = LoggingSource.Instance.GetLogger <SubscriptionStorage>(databaseName);
_subscriptionsSchema.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1
});
}
public void Can_handle_values_of_different_types()
{
using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("ReduceKeyProcessorTests"))
using (var context = JsonOperationContext.ShortTermSingleUse())
using (var bsc = new ByteStringContext(SharedMultipleUseFlag.None))
{
var sut = new ReduceKeyProcessor(9, bufferPool);
sut.Reset();
sut.Process(bsc, 1);
sut.Process(bsc, (long)1);
sut.Process(bsc, 1.0);
sut.Process(bsc, (decimal)1.0);
sut.Process(bsc, "abc");
using (var lazyStringValue = context.GetLazyString("abc"))
{
sut.Process(bsc, lazyStringValue);
}
sut.Process(bsc, context.ReadObject(new DynamicJsonValue()
{
["Name"] = "Arek",
["Age"] = null
}, "foo"));
sut.Process(bsc, new DynamicJsonArray()
{
1,
2,
null,
3
});
sut.Process(bsc, SystemTime.UtcNow);
Assert.NotEqual((ulong)0, sut.Hash);
sut.ReleaseBuffer();
}
}
public void ParallelSerialAllocationAndRelease()
{
using (var pool = new UnmanagedBuffersPoolWithLowMemoryHandling(string.Empty))
{
var allocatedMemory = new BlockingCollection <AllocatedMemoryData>();
Task.Run(() =>
{
for (var i = 0; i < 100; i++)
{
allocatedMemory.Add(pool.Allocate(i));
}
allocatedMemory.CompleteAdding();
});
while (allocatedMemory.IsCompleted == false)
{
AllocatedMemoryData tuple;
if (allocatedMemory.TryTake(out tuple, 100))
{
pool.Return(tuple);
}
}
}
}
public async Task TestJavaScriptIndex()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex");
if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false)
{
ThrowRequiredPropertyNameInRequest("Definition");
}
input.TryGet("Ids", out BlittableJsonReaderArray ids);
var indexDefinition = JsonDeserializationServer.IndexDefinition(index);
if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0)
{
throw new ArgumentException("Index must have a 'Maps' fields");
}
indexDefinition.Type = indexDefinition.DetectStaticIndexType();
if (indexDefinition.Type.IsJavaScript() == false)
{
throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes.");
}
var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration);
var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex;
var collections = new HashSet <string>(compiledIndex.Maps.Keys);
var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >();
using (context.OpenReadTransaction())
{
if (ids == null)
{
foreach (var collection in collections)
{
docsPerCollection.Add(collection,
Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList());
}
}
else
{
var listOfIds = ids.Select(x => x.ToString());
var _ = new Reference <int>
{
Value = 0
};
var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _);
foreach (var doc in docs)
{
if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr))
{
if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false)
{
listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>();
}
listOfDocs.Add(new DynamicBlittableJson(doc));
}
}
}
var mapRes = new List <ObjectInstance>();
//all maps
foreach (var ListOfFunctions in compiledIndex.Maps)
{
//multi maps per collection
foreach (var mapFunc in ListOfFunctions.Value)
{
if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs))
{
foreach (var res in mapFunc(docs))
{
mapRes.Add((ObjectInstance)res);
}
}
}
}
var first = true;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("MapResults");
writer.WriteStartArray();
foreach (var mapResult in mapRes)
{
if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteString(jsStr.ToString());
first = false;
}
}
writer.WriteEndArray();
if (indexDefinition.Reduce != null)
{
using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name))
{
compiledIndex.SetBufferPoolForTestingPurposes(bufferPool);
compiledIndex.SetAllocatorForTestingPurposes(context.Allocator);
first = true;
writer.WritePropertyName("ReduceResults");
writer.WriteStartArray();
var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr))));
foreach (JsValue reduceResult in reduceResults)
{
if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr)
{
if (first == false)
{
writer.WriteComma();
}
writer.WriteString(jsStr.ToString());
first = false;
}
}
}
writer.WriteEndArray();
}
writer.WriteEndObject();
}
}
}
}
public void SetBufferPoolForTestingPurposes(UnmanagedBuffersPoolWithLowMemoryHandling bufferPool)
{
_bufferPool = bufferPool;
}
