public void Invalid_hash_calculation_on_null() { using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("RavenDB_9535")) using (var bsc = new ByteStringContext(SharedMultipleUseFlag.None)) { var sut = new ReduceKeyProcessor(1, bufferPool); try { sut.Reset(); sut.Process(bsc, null); Assert.Equal((ulong)0, sut.Hash); sut.Reset(); sut.Process(bsc, 1); Assert.NotEqual((ulong)0, sut.Hash); sut.Reset(); sut.Process(bsc, null); Assert.Equal((ulong)0, sut.Hash); } finally { sut.ReleaseBuffer(); } } }
public void SetBufferPoolForTestingPurposes(UnmanagedBuffersPoolWithLowMemoryHandling bufferPool) { if (ReduceOperation != null) { ReduceOperation.SetBufferPoolForTestingPurposes(bufferPool); } }
public CurrentIndexingScope(DocumentsStorage documentsStorage, DocumentsOperationContext documentsContext, IndexDefinitionBase indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool) { _documentsStorage = documentsStorage; _documentsContext = documentsContext; UnmanagedBuffersPool = _unmanagedBuffersPool; IndexDefinition = indexDefinition; IndexContext = indexContext; _getSpatialField = getSpatialField; }
public CurrentIndexingScope(Index index, DocumentsStorage documentsStorage, QueryOperationContext queryContext, IndexDefinitionBaseServerSide indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool) { _documentsStorage = documentsStorage; QueryContext = queryContext; Index = index; UnmanagedBuffersPool = _unmanagedBuffersPool; IndexDefinition = indexDefinition; IndexContext = indexContext; _getSpatialField = getSpatialField; }
public GroupBykeyComparer(JavaScriptReduceOperation parent, UnmanagedBuffersPoolWithLowMemoryHandling buffersPool, ByteStringContext allocator) { _parent = parent; _allocator = allocator; _xKey = new ReduceKeyProcessor(_parent._groupByFields.Count(), buffersPool); _yKey = new ReduceKeyProcessor(_parent._groupByFields.Count(), buffersPool); _xKey.SetMode(ReduceKeyProcessor.Mode.MultipleValues); _yKey.SetMode(ReduceKeyProcessor.Mode.MultipleValues); _lastUsedBlittable = null; _lastUsedBucket = null; }
public void SerialAllocationAndRelease() { using (var pool = new UnmanagedBuffersPoolWithLowMemoryHandling(string.Empty)) { var allocatedMemory = new List <AllocatedMemoryData>(); for (var i = 0; i < 1000; i++) { allocatedMemory.Add(pool.Allocate(i)); } foreach (var data in allocatedMemory) { pool.Return(data); } } }
public ReduceKeyProcessor(int numberOfReduceFields, UnmanagedBuffersPoolWithLowMemoryHandling buffersPool) { _numberOfReduceFields = numberOfReduceFields; _buffersPool = buffersPool; if (numberOfReduceFields == 1) { _mode = Mode.SingleValue; } else { _mode = Mode.MultipleValues; _buffer = _buffersPool.Allocate(16); _bufferPos = 0; } }
public ReduceKeyProcessor(int numberOfReduceFields, UnmanagedBuffersPoolWithLowMemoryHandling buffersPool) { _numberOfReduceFields = numberOfReduceFields; _buffersPool = buffersPool; if (numberOfReduceFields == 1) { _mode = Mode.SingleValue; } else { // numberOfReduceFields could be zero when we have 'group bankTotal by 1' _mode = Mode.MultipleValues; _bufferPos = 0; } }
public void ParallelAllocationAndReleaseSeperately() { using (var pool = new UnmanagedBuffersPoolWithLowMemoryHandling(string.Empty)) { var allocatedMemory = new global::Sparrow.Collections.ConcurrentSet <AllocatedMemoryData>(); Parallel.For(0, 100, RavenTestHelper.DefaultParallelOptions, x => { for (var i = 0; i < 10; i++) { allocatedMemory.Add(pool.Allocate(i)); } }); Parallel.ForEach(allocatedMemory, RavenTestHelper.DefaultParallelOptions, item => { pool.Return(item); }); } }
public SubscriptionStorage(DocumentDatabase db) { var options = db.Configuration.Core.RunInMemory ? StorageEnvironmentOptions.CreateMemoryOnly(Path.Combine(db.Configuration.Core.DataDirectory, "Subscriptions")) : StorageEnvironmentOptions.ForPath(Path.Combine(db.Configuration.Core.DataDirectory, "Subscriptions")); options.SchemaVersion = 1; options.TransactionsMode = TransactionsMode.Lazy; _environment = new StorageEnvironment(options); var databaseName = db.Name; _unmanagedBuffersPool = new UnmanagedBuffersPoolWithLowMemoryHandling("Subscriptions", databaseName); _logger = LoggingSource.Instance.GetLogger <SubscriptionStorage>(databaseName); _subscriptionsSchema.DefineKey(new TableSchema.SchemaIndexDef { StartIndex = 0, Count = 1 }); }
public void Can_handle_values_of_different_types() { using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("ReduceKeyProcessorTests")) using (var context = JsonOperationContext.ShortTermSingleUse()) using (var bsc = new ByteStringContext(SharedMultipleUseFlag.None)) { var sut = new ReduceKeyProcessor(9, bufferPool); sut.Reset(); sut.Process(bsc, 1); sut.Process(bsc, (long)1); sut.Process(bsc, 1.0); sut.Process(bsc, (decimal)1.0); sut.Process(bsc, "abc"); using (var lazyStringValue = context.GetLazyString("abc")) { sut.Process(bsc, lazyStringValue); } sut.Process(bsc, context.ReadObject(new DynamicJsonValue() { ["Name"] = "Arek", ["Age"] = null }, "foo")); sut.Process(bsc, new DynamicJsonArray() { 1, 2, null, 3 }); sut.Process(bsc, SystemTime.UtcNow); Assert.NotEqual((ulong)0, sut.Hash); sut.ReleaseBuffer(); } }
public void ParallelSerialAllocationAndRelease() { using (var pool = new UnmanagedBuffersPoolWithLowMemoryHandling(string.Empty)) { var allocatedMemory = new BlockingCollection <AllocatedMemoryData>(); Task.Run(() => { for (var i = 0; i < 100; i++) { allocatedMemory.Add(pool.Allocate(i)); } allocatedMemory.CompleteAdding(); }); while (allocatedMemory.IsCompleted == false) { AllocatedMemoryData tuple; if (allocatedMemory.TryTake(out tuple, 100)) { pool.Return(tuple); } } } }
public async Task TestJavaScriptIndex() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var input = await context.ReadForMemoryAsync(RequestBodyStream(), "TestJavaScriptIndex"); if (input.TryGet("Definition", out BlittableJsonReaderObject index) == false) { ThrowRequiredPropertyNameInRequest("Definition"); } input.TryGet("Ids", out BlittableJsonReaderArray ids); var indexDefinition = JsonDeserializationServer.IndexDefinition(index); if (indexDefinition.Maps == null || indexDefinition.Maps.Count == 0) { throw new ArgumentException("Index must have a 'Maps' fields"); } indexDefinition.Type = indexDefinition.DetectStaticIndexType(); if (indexDefinition.Type.IsJavaScript() == false) { throw new UnauthorizedAccessException("Testing indexes is only allowed for JavaScript indexes."); } var compiledIndex = new JavaScriptIndex(indexDefinition, Database.Configuration); var inputSize = GetIntValueQueryString("inputSize", false) ?? defaultInputSizeForTestingJavaScriptIndex; var collections = new HashSet <string>(compiledIndex.Maps.Keys); var docsPerCollection = new Dictionary <string, List <DynamicBlittableJson> >(); using (context.OpenReadTransaction()) { if (ids == null) { foreach (var collection in collections) { docsPerCollection.Add(collection, Database.DocumentsStorage.GetDocumentsFrom(context, collection, 0, 0, inputSize).Select(d => new DynamicBlittableJson(d)).ToList()); } } else { var listOfIds = ids.Select(x => x.ToString()); var _ = new Reference <int> { Value = 0 }; var docs = Database.DocumentsStorage.GetDocuments(context, listOfIds, 0, int.MaxValue, _); foreach (var doc in docs) { if (doc.TryGetMetadata(out var metadata) && metadata.TryGet(Constants.Documents.Metadata.Collection, out string collectionStr)) { if (docsPerCollection.TryGetValue(collectionStr, out var listOfDocs) == false) { listOfDocs = docsPerCollection[collectionStr] = new List <DynamicBlittableJson>(); } listOfDocs.Add(new DynamicBlittableJson(doc)); } } } var mapRes = new List <ObjectInstance>(); //all maps foreach (var ListOfFunctions in compiledIndex.Maps) { //multi maps per collection foreach (var mapFunc in ListOfFunctions.Value) { if (docsPerCollection.TryGetValue(ListOfFunctions.Key, out var docs)) { foreach (var res in mapFunc(docs)) { mapRes.Add((ObjectInstance)res); } } } } var first = true; using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("MapResults"); writer.WriteStartArray(); foreach (var mapResult in mapRes) { if (JavaScriptIndexUtils.StringifyObject(mapResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } writer.WriteEndArray(); if (indexDefinition.Reduce != null) { using (var bufferPool = new UnmanagedBuffersPoolWithLowMemoryHandling("JavaScriptIndexTest", Database.Name)) { compiledIndex.SetBufferPoolForTestingPurposes(bufferPool); compiledIndex.SetAllocatorForTestingPurposes(context.Allocator); first = true; writer.WritePropertyName("ReduceResults"); writer.WriteStartArray(); var reduceResults = compiledIndex.Reduce(mapRes.Select(mr => new DynamicBlittableJson(JsBlittableBridge.Translate(context, mr.Engine, mr)))); foreach (JsValue reduceResult in reduceResults) { if (JavaScriptIndexUtils.StringifyObject(reduceResult) is JsString jsStr) { if (first == false) { writer.WriteComma(); } writer.WriteString(jsStr.ToString()); first = false; } } } writer.WriteEndArray(); } writer.WriteEndObject(); } } } }
public void SetBufferPoolForTestingPurposes(UnmanagedBuffersPoolWithLowMemoryHandling bufferPool) { _bufferPool = bufferPool; }