public Slice EncodeKey(FdbTuple <T1, T2> value) { return(EncodeComposite(value, 2)); }
public Slice EncodeValue(T key) { return(FdbTuple.EncodeKey(key)); }
public virtual Slice EncodeKey(T1 item1, T2 item2) { return(EncodeComposite(FdbTuple.Create <T1, T2>(item1, item2), 2)); }
public Slice EncodeComposite(FdbTuple <T1, T2> key, int items) { return(this.Encoder.EncodeComposite(new FdbTuple <T1, T2, T3>(key.Item1, key.Item2, default(T3)), items)); }
private static async Task MainAsync(CancellationToken ct) { // change the path to the native lib if not default if (NATIVE_PATH != null) { Fdb.Options.SetNativeLibPath(NATIVE_PATH); } // uncomment this to enable network thread tracing // FdbCore.TracePath = Path.Combine(Path.GetTempPath(), "fdb"); int apiVersion = Fdb.GetMaxApiVersion(); Console.WriteLine("Max API Version: " + apiVersion); try { Console.WriteLine("Starting network thread..."); Fdb.Start(); // this will select API version 21 Console.WriteLine("> Up and running"); Console.WriteLine("Connecting to local cluster..."); using (var cluster = await Fdb.CreateClusterAsync(CLUSTER_FILE, ct)) { Console.WriteLine("> Connected!"); Console.WriteLine("Opening database 'DB'..."); using (var db = await cluster.OpenDatabaseAsync(DB_NAME, FdbSubspace.Create(FdbTuple.Create(SUBSPACE)), false, ct)) { Console.WriteLine("> Connected to db '{0}'", db.Name); // get coordinators var cf = await Fdb.System.GetCoordinatorsAsync(db, ct); Console.WriteLine("Coordinators: " + cf.ToString()); // clear everything using (var tr = db.BeginTransaction(ct)) { Console.WriteLine("Clearing subspace " + db.GlobalSpace + " ..."); tr.ClearRange(db.GlobalSpace); await tr.CommitAsync(); Console.WriteLine("> Database cleared"); } Console.WriteLine("----------"); await TestSimpleTransactionAsync(db, ct); Console.WriteLine("----------"); await BenchInsertSmallKeysAsync(db, N, 16, ct); // some guid await BenchInsertSmallKeysAsync(db, N, 60 * 4, ct); // one Int32 per minutes, over an hour await BenchInsertSmallKeysAsync(db, N, 512, ct); // small JSON payload ////await BenchInsertSmallKeysAsync(db, N, 4096, ct); // typical small cunk size ////await BenchInsertSmallKeysAsync(db, N / 10, 65536, ct); // typical medium chunk size //await BenchInsertSmallKeysAsync(db, 1, 100000, ct); // Maximum value size (as of beta 1) ////// insert keys in parrallel await BenchConcurrentInsert(db, 1, 100, 512, ct); await BenchConcurrentInsert(db, 1, 1000, 512, ct); await BenchConcurrentInsert(db, 1, 10000, 512, ct); await BenchConcurrentInsert(db, 1, N, 16, ct); await BenchConcurrentInsert(db, 2, N, 16, ct); await BenchConcurrentInsert(db, 4, N, 16, ct); await BenchConcurrentInsert(db, 8, N, 16, ct); await BenchConcurrentInsert(db, 16, N, 16, ct); //await BenchSerialWriteAsync(db, N, ct); //await BenchSerialReadAsync(db, N, ct); //await BenchConcurrentReadAsync(db, N, ct); //await BenchClearAsync(db, N, ct); await BenchUpdateSameKeyLotsOfTimesAsync(db, 1000, ct); await BenchUpdateLotsOfKeysAsync(db, 1000, ct); await BenchBulkInsertThenBulkReadAsync(db, 100 * 1000, 50, 128, ct); await BenchBulkInsertThenBulkReadAsync(db, 100 * 1000, 128, 50, ct); ////await BenchBulkInsertThenBulkReadAsync(db, 1 * 1000 * 1000, 50, 128, ct); await BenchMergeSortAsync(db, 100, 3, 20, ct); await BenchMergeSortAsync(db, 1000, 10, 100, ct); await BenchMergeSortAsync(db, 100, 100, 100, ct); await BenchMergeSortAsync(db, 100, 1000, 100, ct); Console.WriteLine("time to say goodbye..."); } } } finally { Console.WriteLine("### DONE ###"); Fdb.Stop(); } #if DEBUG Console.ReadLine(); #endif }
public Slice EncodeKey(FdbTuple <T1, T2> key) { return(EncodeComposite(key, 2)); }
public void Clear([NotNull] IFdbTransaction trans, FdbTuple <T1, T2, T3> key) { trans.Clear(EncodeKey(key)); }
public async Task Test_Can_Batch_Aggregate_With_Transformed_Result() { const int N = 50 * 1000; using (var db = await OpenTestPartitionAsync()) { Log("Bulk inserting {0:N0} items...", N); var location = await GetCleanDirectory(db, "Bulk", "Aggregate"); Log("Preparing..."); var rnd = new Random(2403); var source = Enumerable.Range(1, N).Select((x) => new KeyValuePair <int, int>(x, rnd.Next(1000))).ToList(); await Fdb.Bulk.WriteAsync( db, source.Select((x) => new KeyValuePair <Slice, Slice>(location.Keys.Encode(x.Key), Slice.FromInt32(x.Value))), this.Cancellation ); Log("Reading..."); int chunks = 0; var sw = Stopwatch.StartNew(); double average = await Fdb.Bulk.AggregateAsync( db, source.Select(x => location.Keys.Encode(x.Key)), () => FdbTuple.Create(0L, 0L), async (xs, ctx, state) => { Interlocked.Increment(ref chunks); Log("> Called with batch of " + xs.Length.ToString("N0") + " at offset " + ctx.Position.ToString("N0") + " of gen " + ctx.Generation + " with step " + ctx.Step + " and cooldown " + ctx.Cooldown + " (genElapsed=" + ctx.ElapsedGeneration + ", totalElapsed=" + ctx.ElapsedTotal + ")"); var throttle = Task.Delay(TimeSpan.FromMilliseconds(10 + (xs.Length / 25) * 5)); // magic numbers to try to last longer than 5 sec var results = await ctx.Transaction.GetValuesAsync(xs); await throttle; long sum = 0L; for (int i = 0; i < results.Length; i++) { sum += results[i].ToInt32(); } return(FdbTuple.Create(state.Item1 + sum, state.Item2 + results.Length)); }, (state) => (double)state.Item1 / state.Item2, this.Cancellation ); sw.Stop(); Log("Done in {0:N3} sec and {1} chunks", sw.Elapsed.TotalSeconds, chunks); double actual = (double)source.Sum(x => (long)x.Value) / source.Count; Log("> Computed average of the {0:N0} random values is {1:N3}", N, average); Log("> Actual average of the {0:N0} random values is {1:N3}", N, actual); Assert.That(average, Is.EqualTo(actual).Within(double.Epsilon)); // cleanup because this test can produce a lot of data await location.RemoveAsync(db, this.Cancellation); } }
public virtual FdbKeyRange ToRange(T1 key1, T2 key2, T3 key3) { return(FdbTuple.ToRange(this.EncodeKey(key1, key2, key3))); }
public void Set([NotNull] IFdbTransaction trans, FdbTuple <T1, T2, T3> key, Slice value) { trans.Set(EncodeKey(key), value); }
Slice ICompositeKeyEncoder <FdbTuple <T1, T2, T3> > .EncodeComposite(FdbTuple <T1, T2, T3> key, int items) { return(this.Key + m_encoder.EncodeComposite(key, items)); }
public virtual Slice EncodeKey(FdbTuple <T1, T2, T3> key) { return(this.Key + m_encoder.EncodeKey(key)); }
protected IFdbTuple ToRelativePath(IEnumerable <string> path) { return(ToRelativePath(path == null ? null : FdbTuple.FromEnumerable <string>(path))); }
public FdbKeyRange ToRange(T value) { //REVIEW: which semantic for ToRange() should we use? return(FdbTuple.ToRange(Encode(value))); }
public void SetId(Dictionary <string, IFdbTuple> document, TId id) { document[this.IdName] = FdbTuple.Create(id); }
public async Task Test_Can_Batch_ForEach_WithContextAndState() { const int N = 50 * 1000; using (var db = await OpenTestPartitionAsync()) { Log("Bulk inserting {0:N0} items...", N); var location = await GetCleanDirectory(db, "Bulk", "ForEach"); Log("Preparing..."); await Fdb.Bulk.WriteAsync( db, Enumerable.Range(1, N).Select((x) => new KeyValuePair <Slice, Slice>(location.Keys.Encode(x), Slice.FromInt32(x))), this.Cancellation ); Log("Reading..."); long total = 0; long count = 0; int chunks = 0; var sw = Stopwatch.StartNew(); await Fdb.Bulk.ForEachAsync( db, Enumerable.Range(1, N).Select(x => location.Keys.Encode(x)), () => FdbTuple.Create(0L, 0L), // (sum, count) (xs, ctx, state) => { Interlocked.Increment(ref chunks); Log("> Called with batch of {0:N0} at offset {1:N0} of gen {2} with step {3} and cooldown {4} (generation = {5:N3} sec, total = {6:N3} sec)", xs.Length, ctx.Position, ctx.Generation, ctx.Step, ctx.Cooldown, ctx.ElapsedGeneration.TotalSeconds, ctx.ElapsedTotal.TotalSeconds); var t = ctx.Transaction.GetValuesAsync(xs); Thread.Sleep(TimeSpan.FromMilliseconds(10 + (xs.Length / 25) * 5)); // magic numbers to try to last longer than 5 sec var results = t.Result; // <-- this is bad practice, never do that in real life, 'mkay? long sum = 0; for (int i = 0; i < results.Length; i++) { sum += results[i].ToInt32(); } return(FdbTuple.Create( state.Item1 + sum, // updated sum state.Item2 + results.Length // updated count )); }, (state) => { Interlocked.Add(ref total, state.Item1); Interlocked.Add(ref count, state.Item2); }, this.Cancellation ); sw.Stop(); Log("Done in {0:N3} sec and {1} chunks", sw.Elapsed.TotalSeconds, chunks); Log("Sum of integers 1 to {0:N0} is {1:N0}", count, total); // cleanup because this test can produce a lot of data await location.RemoveAsync(db, this.Cancellation); } }
public abstract Slice EncodeComposite(FdbTuple <T1, T2> key, int items);
/// <summary>Estimate the number of keys in the specified range.</summary> /// <param name="db">Database used for the operation</param> /// <param name="beginInclusive">Key defining the beginning of the range</param> /// <param name="endExclusive">Key defining the end of the range</param> /// <param name="onProgress">Optional callback called everytime the count is updated. The first argument is the current count, and the second argument is the last key that was found.</param> /// <param name="cancellationToken">Token used to cancel the operation</param> /// <returns>Number of keys k such that <paramref name="beginInclusive"/> <= k > <paramref name="endExclusive"/></returns> /// <remarks>If the range contains a large of number keys, the operation may need more than one transaction to complete, meaning that the number will not be transactionally accurate.</remarks> public static async Task <long> EstimateCountAsync([NotNull] IFdbDatabase db, Slice beginInclusive, Slice endExclusive, IProgress <FdbTuple <long, Slice> > onProgress, CancellationToken cancellationToken) { const int INIT_WINDOW_SIZE = 1 << 8; // start at 256 //1024 const int MAX_WINDOW_SIZE = 1 << 13; // never use more than 4096 const int MIN_WINDOW_SIZE = 64; // use range reads when the windows size is smaller than 64 if (db == null) { throw new ArgumentNullException("db"); } if (endExclusive < beginInclusive) { throw new ArgumentException("The end key cannot be less than the begin key", "endExclusive"); } cancellationToken.ThrowIfCancellationRequested(); // To count the number of items in the range, we will scan it using a key selector with an offset equal to our window size // > if the returned key is still inside the range, we add the window size to the counter, and start again from the current key // > if the returned key is outside the range, we reduce the size of the window, and start again from the previous key // > if the returned key is exactly equal to the end of range, OR if the window size was 1, then we stop // Since we don't know in advance if the range contains 1 key or 1 Billion keys, choosing a good value for the window size is critical: // > if it is too small and the range is very large, we will need too many sequential reads and the network latency will quickly add up // > if it is too large and the range is small, we will spend too many times halving the window size until we get the correct value // A few optimizations are possible: // > we could start with a small window size, and then double its size on every full segment (up to a maximum) // > for the last segment, we don't need to wait for a GetKey to complete before issuing the next, so we could split the segment into 4 (or more), do the GetKeyAsync() in parallel, detect the quarter that cross the boundary, and iterate again until the size is small // > once the window size is small enough, we can switch to using GetRange to read the last segment in one shot, instead of iterating with window size 16, 8, 4, 2 and 1 (the wost case being 2^N - 1 items remaning) // note: we make a copy of the keys because the operation could take a long time and the key's could prevent a potentially large underlying buffer from being GCed var cursor = beginInclusive.Memoize(); var end = endExclusive.Memoize(); using (var tr = db.BeginReadOnlyTransaction(cancellationToken)) { #if TRACE_COUNTING tr.Annotate("Estimating number of keys in range {0}", FdbKeyRange.Create(beginInclusive, endExclusive)); #endif tr.SetOption(FdbTransactionOption.ReadYourWritesDisable); // start looking for the first key in the range cursor = await tr.Snapshot.GetKeyAsync(FdbKeySelector.FirstGreaterOrEqual(cursor)).ConfigureAwait(false); if (cursor >= end) { // the range is empty ! return(0); } // we already have seen one key, so add it to the count #if TRACE_COUNTING int iter = 1; #endif long counter = 1; // start with a medium-sized window int windowSize = INIT_WINDOW_SIZE; bool last = false; while (cursor < end) { Contract.Assert(windowSize > 0); var selector = FdbKeySelector.FirstGreaterOrEqual(cursor) + windowSize; Slice next = Slice.Nil; FdbException error = null; try { next = await tr.Snapshot.GetKeyAsync(selector).ConfigureAwait(false); #if TRACE_COUNTING ++iter; #endif } catch (FdbException e) { error = e; } if (error != null) { // => from this point, the count returned will not be transactionally accurate if (error.Code == FdbError.PastVersion) { // the transaction used up its time window tr.Reset(); } else { // check to see if we can continue... await tr.OnErrorAsync(error.Code).ConfigureAwait(false); } // retry tr.SetOption(FdbTransactionOption.ReadYourWritesDisable); continue; } //BUGBUG: GetKey(...) always truncate the result to \xFF if the selected key would be past the end, // so we need to fall back immediately to the binary search and/or get_range if next == \xFF if (next > end) { // we have reached past the end, switch to binary search last = true; // if window size is already 1, then we have counted everything (the range.End key does not exist in the db) if (windowSize == 1) { break; } if (windowSize <= MIN_WINDOW_SIZE) { // The window is small enough to switch to reading for counting (will be faster than binary search) #if TRACE_COUNTING tr.Annotate("Switch to reading all items (window size = {0})", windowSize); #endif // Count the keys by reading them. Also, we know that there can not be more than windowSize - 1 remaining int n = await tr.Snapshot .GetRange( FdbKeySelector.FirstGreaterThan(cursor), // cursor has already been counted once FdbKeySelector.FirstGreaterOrEqual(end), new FdbRangeOptions() { Limit = windowSize - 1 } ) .CountAsync() .ConfigureAwait(false); counter += n; if (onProgress != null) { onProgress.Report(FdbTuple.Create(counter, end)); } #if TRACE_COUNTING ++iter; #endif break; } windowSize >>= 1; continue; } // the range is not finished, advance the cursor counter += windowSize; cursor = next; if (onProgress != null) { onProgress.Report(FdbTuple.Create(counter, cursor)); } if (!last) { // double the size of the window if we are not in the last segment windowSize = Math.Min(windowSize << 1, MAX_WINDOW_SIZE); } } #if TRACE_COUNTING tr.Annotate("Found {0} keys in {1} iterations", counter, iter); #endif return(counter); } }
public void Test_RangeDictionary_Black_And_White() { // we have a space from 0 <= x < 100 that is empty // we insert a random serie of ranges that are either Black or White // after each run, we check that all ranges are correctly ordered, merged, and so on. const int S = 100; // [0, 100) const int N = 1000; // number of repetitions const int K = 25; // max number of ranges inserted per run var rnd = new Random(); int seed = rnd.Next(); Console.WriteLine("Using random seed " + seed); rnd = new Random(seed); for (int i = 0; i < N; i++) { var cola = new ColaRangeDictionary <int, RangeColor>(); var witnessColors = new RangeColor?[S]; var witnessIndexes = new int?[S]; // choose a random number of ranges int k = rnd.Next(3, K); Trace.WriteLine(""); Trace.WriteLine(String.Format("# Starting run {0} with {1} insertions", i, k)); int p = 0; for (int j = 0; j < k; j++) { var begin = rnd.Next(S); // 50/50 of inserting a single element, or a range var end = (rnd.Next(2) == 1 ? begin : rnd.Next(2) == 1 ? rnd.Next(begin, S) : Math.Min(S - 1, begin + rnd.Next(5))) + 1; // reminder: +1 because 'end' is EXCLUDED Assert.That(begin, Is.LessThan(end)); // 50/50 for the coloring var color = rnd.Next(2) == 1 ? RangeColor.White : RangeColor.Black; // uncomment this line if you want to reproduce this exact run //Console.WriteLine("\t\tcola.Mark(" + begin + ", " + end + ", RangeColor." + color + ");"); cola.Mark(begin, end, color); for (int z = begin; z < end; z++) { witnessColors[z] = color; witnessIndexes[z] = p; } //Console.WriteLine(" > |{0}|", String.Join("", witnessIndexes.Select(x => x.HasValue ? (char)('A' + x.Value) : ' '))); Debug.WriteLine(" |{0}| + [{1,2}, {2,2}) = {3} > #{4,2} [ {5} ]", String.Join("", witnessColors.Select(w => !w.HasValue ? ' ' : w.Value == RangeColor.Black ? '#' : '°')), begin, end, color, cola.Count, String.Join(", ", cola)); ++p; } // pack the witness list into ranges var witnessRanges = new List <FdbTuple <int, int, RangeColor> >(); RangeColor?prev = null; p = 0; for (int z = 1; z < S; z++) { if (witnessColors[z] != prev) { // switch if (prev.HasValue) { witnessRanges.Add(FdbTuple.Create(p, z, prev.Value)); } p = z; prev = witnessColors[z]; } } Trace.WriteLine(String.Format("> RANGES: #{0,2} [ {1} ]", cola.Count, String.Join(", ", cola))); Trace.WriteLine(String.Format(" #{0,2} [ {1} ]", witnessRanges.Count, String.Join(", ", witnessRanges))); var counter = new int[S]; var observedIndexes = new int?[S]; var observedColors = new RangeColor?[S]; p = 0; foreach (var range in cola) { Assert.That(range.Begin < range.End, "Begin < End {0}", range); for (int z = range.Begin; z < range.End; z++) { observedIndexes[z] = p; counter[z]++; observedColors[z] = range.Value; } ++p; } Trace.WriteLine(String.Format("> INDEXS: |{0}|", String.Join("", observedIndexes.Select(x => x.HasValue ? (char)('A' + x.Value) : ' ')))); Trace.WriteLine(String.Format(" |{0}|", String.Join("", witnessIndexes.Select(x => x.HasValue ? (char)('A' + x.Value) : ' ')))); Trace.WriteLine(String.Format("> COLORS: |{0}|", String.Join("", observedColors.Select(w => !w.HasValue ? ' ' : w.Value == RangeColor.Black ? '#' : '°')))); Trace.WriteLine(String.Format(" |{0}|", String.Join("", witnessColors.Select(w => !w.HasValue ? ' ' : w.Value == RangeColor.Black ? '#' : '°')))); // verify the colors foreach (var range in cola) { for (int z = range.Begin; z < range.End; z++) { Assert.That(range.Value, Is.EqualTo(witnessColors[z]), "#{0} color mismatch for {1}", z, range); Assert.That(counter[z], Is.EqualTo(1), "Duplicate at offset #{0} for {1}", z, range); } } // verify that nothing was missed for (int z = 0; z < S; z++) { if (witnessColors[z] == null) { if (counter[z] != 0) { Trace.WriteLine("@ FAIL!!! |" + new string('-', z) + "^"); } Assert.That(counter[z], Is.EqualTo(0), "Should be void at offset {0}", z); } else { if (counter[z] != 1) { Trace.WriteLine("@ FAIL!!! |" + new string('-', z) + "^"); } Assert.That(counter[z], Is.EqualTo(1), "Should be filled with {1} at offset {0}", z, witnessColors[z]); } } } }