static FingerTree <Node <T, M>, M> AddDigits0 <T, M>(Measured <T, M> m, FingerTree <Node <T, M>, M> m1, Digit <T, M> s1, Digit <T, M> p2, FingerTree <Node <T, M>, M> m2) { var mk = new MakeTree <T, M>(m); return(s1.Match(x1 => p2.Match(y1 => Append1(m, m1, mk.Node2(x1.V, y1.V), m2), y2 => Append1(m, m1, mk.Node3(x1.V, y2.V1, y2.V2), m2), y3 => Append2(m, m1, mk.Node2(x1.V, y3.V1), mk.Node2(y3.V2, y3.V3), m2), y4 => Append2(m, m1, mk.Node3(x1.V, y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)), x2 => p2.Match(y1 => Append1(m, m1, mk.Node3(x2.V1, x2.V2, y1.V), m2), y2 => Append2(m, m1, mk.Node2(x2.V1, x2.V2), mk.Node2(y2.V1, y2.V2), m2), y3 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, y3.V1), mk.Node2(y3.V2, y3.V3), m2), y4 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, y4.V1), mk.Node3(y4.V2, y4.V3, y4.V4), m2)), x3 => p2.Match(y1 => Append2(m, m1, mk.Node2(x3.V1, x3.V2), mk.Node2(x3.V3, y1.V), m2), y2 => Append2(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node2(y2.V1, y2.V2), m2), y3 => Append2(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(y3.V1, y3.V2, y3.V3), m2), y4 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node2(y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)), x4 => p2.Match(y1 => Append2(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node2(x4.V4, y1.V), m2), y2 => Append2(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, y2.V1, y2.V2), m2), y3 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node2(x4.V4, y3.V1), mk.Node2(y3.V2, y3.V3), m2), y4 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)))); }
public void Set(double Value) { var old = this.Value; #region Measuring if (this.Value != Value) { this.Value = Value; if (Measuring != null) { Measuring.Invoke(this, new StableMeasureEventArgs(Value)); } } #endregion if (IsStart) { if (!IsComplete) { #region Complete Start if (Math.Abs(Value - old) < ErrorRange) { IsComplete = true; CompleteTIme = DateTime.Now; } #endregion } else { #region Complete Instance if (Math.Abs(Value - old) > ErrorRange) { IsComplete = false; } else { var dv = (DateTime.Now - CompleteTIme).TotalMilliseconds; if (dv > MeasureTime) { IsComplete = IsStart = false; if (Measured != null) { Measured.Invoke(this, new StableMeasureEventArgs(Value)); } } } #endregion } } else { #region Start if (Math.Abs(old - Value) > ErrorRange) { IsStart = true; StarTime = DateTime.Now; } #endregion } }
public Node3(Measured <T, M> m, Node3 <T, M> other) : base(m, m.Append(m.Measure(other.V1), m.Append(m.Measure(other.V2), m.Measure(other.V3)))) { _v1 = other.V1; _v2 = other.V2; _v3 = other.V3; }
public Three(Measured <T, M> m, T v1, T v2, T v3) : base(m, m.Append(m.Measure(v1), m.Append(m.Measure(v2), m.Measure(v3)))) { _v1 = v1; _v2 = v2; _v3 = v3; }
private async Task <DataFileCommitInfo> CommitIfRequired() { if (_nonPersistedQ.Count < _batchSize) { return(null); } await _commitLock.WaitAsync().ConfigureAwait(false); try { if (_nonPersistedQ.Count < _batchSize) { return(null); } using (Measured.Operation("datafile_commit")) { return(await Commit().ConfigureAwait(false)); } } finally { _commitLock.Release(); } }
public Digit <U, M> Map <U>(Func <T, U> f, Measured <U, M> m) { return(Match <Digit <U, M> >(one => new One <U, M>(m, f(one.V)), two => new Two <U, M>(m, f(two.V1), f(two.V2)), three => new Three <U, M>(m, f(three.V1), f(three.V2), f(three.V3)), four => new Four <U, M>(m, f(four.V1), f(four.V2), f(four.V3), f(four.V4)))); }
static async Task TestWithoutBatching() { var instrumentWriter = new AggregatingInstrumentWriter(); Measured.Initialize(instrumentWriter); var docs = Enumerable.Range(1, TestSetSize).Select(id => GenerateRandomDocument(id)); using (var db = new Database(DataDir, RootIndexDir)) { var counter = 0; var sw = Stopwatch.StartNew(); foreach (var doc in docs) { await db.AddDocument(doc); if (++counter % 1000 == 0) { Console.WriteLine("Added {0} documents to the DB in {1}", counter, sw.Elapsed); } } await db.Commit(); Console.WriteLine("Finished adding documents in {0}", sw.Elapsed); instrumentWriter.WriteStats(Console.Out); RunSearches(db); instrumentWriter.WriteStats(Console.Out); } Console.ReadLine(); }
static FingerTree <Node <Node <T, M>, M>, M> AddDigits2 <T, M>(Measured <Node <T, M>, M> m, FingerTree <Node <Node <T, M>, M>, M> m1, Digit <Node <T, M>, M> x, Node <T, M> n1, Node <T, M> n2, Digit <Node <T, M>, M> y, FingerTree <Node <Node <T, M>, M>, M> m2) { var mk = new MakeTree <Node <T, M>, M>(m); return(x.Match(x1 => y.Match(y1 => Append2(m, m1, mk.Node2(x1.V, n1), mk.Node2(n2, y1.V), m2), y2 => Append2(m, m1, mk.Node3(x1.V, n1, n2), mk.Node2(y2.V1, y2.V2), m2), y3 => Append2(m, m1, mk.Node3(x1.V, n1, n2), mk.Node3(y3.V1, y3.V2, y3.V3), m2), y4 => Append3(m, m1, mk.Node3(x1.V, n1, n2), mk.Node2(y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)), x2 => y.Match(y1 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node2(n2, y1.V), m2), y2 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node3(n2, y2.V1, y2.V2), m2), y3 => Append3(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node2(n2, y3.V1), mk.Node2(y3.V2, y3.V3), m2), y4 => Append3(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node3(n2, y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)), x3 => y.Match(y1 => Append2(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(n1, n2, y1.V), m2), y2 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node2(n1, n2), mk.Node2(y2.V1, y2.V2), m2), y3 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(n1, n2, y3.V1), mk.Node2(y3.V2, y3.V3), m2), y4 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(n1, n2, y4.V1), mk.Node3(y4.V2, y4.V3, y4.V4), m2)), x4 => y.Match(y1 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node2(x4.V4, n1), mk.Node2(n2, y1.V), m2), y2 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, n1, n2), mk.Node2(y2.V1, y2.V2), m2), y3 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, n1, n2), mk.Node3(y3.V1, y3.V2, y3.V3), m2), y4 => Append4(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, n1, n2), mk.Node2(y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)))); }
/// <summary> /// Update measurement from the system reading. /// </summary> public void UpdateMeasurement() { var s = Measured.PointwiseDivide(Current).ToColumnWiseArray(); H = M.Diagonal(2, 2, s); _logger.Info("Current measurement " + F); }
public Four(Measured <T, M> m, T v1, T v2, T v3, T v4) : base(m, m.Append(m.Measure(v1), m.Append(m.Measure(v2), m.Append(m.Measure(v3), m.Measure(v4))))) { _v1 = v1; _v4 = v4; _v2 = v2; _v3 = v3; }
public override FingerTree <T, M> Concat(FingerTree <T, M> t) { Measured <T, M> m = Measured; return(t.Match(e => this, s => AddRight(s.Item), d => new Deep <T, M>(m, m.Append(Size, d.Size), _prefix, AddDigits0(m, _middle, _suffix, d._prefix, d._middle), d._suffix))); }
public async Task AddTerm(string term, long dataFileOffset) { using (Measured.Operation("add_term_to_in_memory_ix")) { AddTermToInMemoryIndex(term, dataFileOffset); } await CommitIfHighMemoryPressure(); }
/// <summary> /// Updates <see cref="AbsoluteSize" /> and <see cref="AbsolutePosition" />. /// </summary> public virtual void Measure() { var parentGui = Parent as GuiBase2D; var parentSize = parentGui?.AbsoluteSize ?? (parentGui?.AbsoluteSize ?? new Vector2()); var parentPosition = parentGui?.AbsolutePosition ?? new Vector2(); AbsoluteSize = Size.toAbsolute(parentSize).round(); var position = Position; var xScaleOffset = 0.0f; var yScaleOffset = 0.0f; var xAbsoluteOffset = 0; var yAbsoluteOffset = 0; var xMult = 1; var yMult = 1; if (_alignmentX == AlignmentX.Right) { xScaleOffset = 1; xAbsoluteOffset = (int)-AbsoluteSize.X; xMult = -1; } else if (_alignmentX == AlignmentX.Center) { xScaleOffset = 0.5f; xAbsoluteOffset = (int)-AbsoluteSize.X / 2; } if (_alignmentY == AlignmentY.Bottom) { yScaleOffset = 1; yAbsoluteOffset = (int)-AbsoluteSize.Y; yMult = -1; } else if (_alignmentY == AlignmentY.Middle) { yScaleOffset = 0.5f; yAbsoluteOffset = (int)-AbsoluteSize.Y / 2; } position = new UDim2(xScaleOffset + position.Scale.X * xMult, xAbsoluteOffset + position.Absolute.X * xMult, yScaleOffset + position.Scale.Y * xMult, yAbsoluteOffset + position.Absolute.Y * yMult); AbsolutePosition = (parentPosition + position.toAbsolute(parentSize)).round(); Measured?.Fire(); Arrange(); foreach (var kv in Children) { var el = kv as GuiElement; el?.Measure(); } }
public Deep(Measured <T, M> measured, M size, Digit <T, M> prefix, FingerTree <Node <T, M>, M> middle, Digit <T, M> suffix) : base(measured, size) { _mk = new MakeTree <T, M>(measured); _prefix = prefix; _middle = middle; _suffix = suffix; }
public IEnumerable <Document> Filter(SearchQuery query, IEnumerable <Document> docs) { using (Measured.Operation("grep_filter")) { foreach (var matchingDoc in docs.Where(doc => IsMatch(query, doc))) { yield return(matchingDoc); } } }
public override FingerTree <T, M> AddLeft(T a) { Measured <T, M> m = Measured; M size = m.Append(m.Measure(a), Size); return(_prefix.Match(x1 => new Deep <T, M>(m, size, _mk.Two(a, x1.V), _middle, _suffix), x2 => new Deep <T, M>(m, size, _mk.Three(a, x2.V1, x2.V2), _middle, _suffix), x3 => new Deep <T, M>(m, size, _mk.Four(a, x3.V1, x3.V2, x3.V3), _middle, _suffix), x4 => new Deep <T, M>(m, size, _mk.Two(a, x4.V1), _middle.AddLeft(_mk.Node3(x4.V2, x4.V3, x4.V4)), _suffix))); }
public override FingerTree <T, M> AddRight(T a) { Measured <T, M> m = Measured; M measure = m.Append(m.Measure(a), Size); return(_suffix.Match(x1 => new Deep <T, M>(m, measure, _prefix, _middle, _mk.Two(x1.V, a)), x2 => new Deep <T, M>(m, measure, _prefix, _middle, _mk.Three(x2.V1, x2.V2, a)), x3 => new Deep <T, M>(m, measure, _prefix, _middle, _mk.Four(x3.V1, x3.V2, x3.V3, a)), x4 => new Deep <T, M>(m, measure, _prefix, _middle.AddRight(_mk.Node3(x4.V1, x4.V2, x4.V3)), _mk.Two(x4.V4, a)))); }
static async Task SearchExistingDb() { var instrumentWriter = new AggregatingInstrumentWriter(); Measured.Initialize(instrumentWriter); using (var db = new Database(DataDir, RootIndexDir)) { RunSearches(db); instrumentWriter.WriteStats(Console.Out); } }
private IEnumerable <long> SearchIndexFile(string targetTerm, string indexFilePath) { var comparer = StringComparer.OrdinalIgnoreCase; using (Measured.Operation("search_index_file")) using (var ixFile = new FileStream(indexFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { // roughly navigate to the position in the index file where the term could be var preliminaryPosition = SearchIndexFileFooter(ixFile, targetTerm, comparer, out var footerPos); if (preliminaryPosition == null) { yield break; } ixFile.Position = preliminaryPosition.Value; while (ixFile.Position != footerPos) { var nextTermOffset = ixFile.ReadLong(); var termLengthInBytes = ixFile.ReadInt(); var term = ixFile.ReadUtf8String(termLengthInBytes); var cmpResult = comparer.Compare(term, targetTerm); if (cmpResult > 0) { // We've got past the place where the target term could have been in the index // (we conclude this because terms are sorted alphabetically) break; } else if (cmpResult < 0) { ixFile.Position = nextTermOffset; continue; } else { var postingListLength = ixFile.ReadInt(); for (int i = 0; i < postingListLength; i++) { var entry = ixFile.ReadLong(); yield return(entry); } break; } } } }
static FingerTree <Node <T, M>, M> Append1 <T, M>(Measured <T, M> m, FingerTree <Node <T, M>, M> t1, Node <T, M> n1, FingerTree <Node <T, M>, M> t2) { return(t1.Match(e => t2.AddLeft(n1), s => t2.AddLeft(n1).AddLeft(s.Item), d => t2.Match(e2 => t1.AddRight(n1), s2 => t1.AddRight(n1).AddRight(s2.Item), d2 => new Deep <Node <T, M>, M>(m.Node, m.Append(m.Append(d.Size, m.Node.Measure(n1)), d2.Size), d._prefix, AddDigits1(m.Node, d._middle, d._suffix, n1, d2._prefix, d2._middle), d2._suffix)))); }
static async Task RecreateTestDb() { ClearData(); var instrumentWriter = new AggregatingInstrumentWriter(); Measured.Initialize(instrumentWriter); using (var db = new Database(DataDir, RootIndexDir)) { await WriteToDbByBatches(db); instrumentWriter.WriteStats(Console.Out); } }
static async Task TestWithBatching() { ClearData(); var instrumentWriter = new AggregatingInstrumentWriter(); Measured.Initialize(instrumentWriter); using (var db = new Database(DataDir, RootIndexDir)) { await WriteToDbByBatches(db); instrumentWriter.WriteStats(Console.Out); RunSearches(db); instrumentWriter.WriteStats(Console.Out); } Console.ReadLine(); }
private async Task TruncateByDocId(long docId) { using (Measured.Operation("wal_truncate")) { _file.Position = _file.Length; while (_file.Position != 0) { var batch = await DocumentSerializer.DeserializeBatchFromRightToLeft(_file); var firstDocIdInBatch = batch[0].Id; if (docId >= firstDocIdInBatch) { await TruncateByFileOffset(_file.Position); break; } } } }
public async Task CommitTran(Tran tran) { await _commitLock.WaitAsync().ConfigureAwait(false); try { using (Measured.Operation("wal_commit")) { foreach (var doc in tran.Documents) { doc.Id = _docIdGenerator.GetNextId(); } await DocumentSerializer.SerializeBatch(tran.Documents, _file).ConfigureAwait(false); await _file.FlushAsync().ConfigureAwait(false); } } finally { _commitLock.Release(); } }
static FingerTree <Node <T, M>, M> Append4 <T, M>(Measured <T, M> m, FingerTree <Node <T, M>, M> t1, Node <T, M> n1, Node <T, M> n2, Node <T, M> n3, Node <T, M> n4, FingerTree <Node <T, M>, M> t2) { return(t1.Match(e => t2.AddLeft(n4).AddLeft(n3).AddLeft(n2).AddLeft(n1), s => t2.AddLeft(n4).AddLeft(n3).AddLeft(n2).AddLeft(n1).AddLeft(s.Item), d => t2.Match(e2 => t2.AddRight(n1).AddRight(n2).AddRight(n3).AddRight(n4), s2 => t2.AddRight(n1).AddRight(n2).AddRight(n3).AddRight(n4).AddRight(s2.Item), d2 => new Deep <Node <T, M>, M>(m.Node, m.Append( m.Append( m.Append( m.Append( m.Append(d.Size, n1.Measure), n2.Measure), n3.Measure), n4.Measure), d2.Size), d._prefix, AddDigits4(m.Node, d._middle, d._suffix, n1, n2, n3, n4, d2._prefix, d2._middle), d2._suffix)))); }
protected Digit(Measured <T, M> m, M size) { Size = size; _m = m; _mk = new MakeTree <T, M>(m); }
private async Task CommitInternal() { using (Measured.Operation("commit_index_file")) { // detaching old in memory index so that all new updates will go to the new // in memory index while the old one is being committed to disk var newInMemoryIx = new InMemoryIndex(); var oldInMemoryIx = Interlocked.Exchange(ref _inMemoryIx, newInMemoryIx); Interlocked.Exchange(ref _memoryPressureScore, 0); if (oldInMemoryIx.IsEmpty) { // nothing to commit return; } /* * Index file structure: * * <file offset (in bytes) of footer (long)> * <main part> * <footer> * * Footer includes sqrt(N) sorted terms with their position in the main part and allows for faster seeks in O(2*sqrt(N)) */ /* * <main part> structure: * <file offset (in bytes) of term 2> <term 1 length in bytes> <term 1 UTF-8 bytes> <number of items in the posting list 1> <item 1 of the posting list 1> ... <item N of the posting list 1> * <file offset (in bytes) of term 3> <term 2 length in bytes> <term 2 UTF-8 bytes> <number of items in the posting list 2> <item 2 of the posting list 2> ... <item N of the posting list 2> * ... * (this goes on for all terms in the index) */ /* * <footer> structure: * (N - total number of terms in the index file, K = sqrt(N)) * * <term 1 length in bytes> <term 1 UTF-8 bytes> <term 1 position in main index part> * <term K length in bytes> <term K UTF-8 bytes> <term K position in main index part> * <term 2K length in bytes> <term 2K UTF-8 bytes> <term 2K position in main index part> * ... * <last term length in bytes> <last term UTF-8 bytes> <last term position in main index part> * * (note that 1st and last terms are always included) */ var footeringFactor = (int)Math.Sqrt(oldInMemoryIx.Count); var footerItems = new List <KeyValuePair <string, long> >(footeringFactor + 1); var i = 0; using (var newIxFile = OpenNextIndexFileForWrite()) { newIxFile.Position += sizeof(long); foreach (var termWithPostingList in oldInMemoryIx.OrderBy(x => x.Key)) { var isLastItem = i == oldInMemoryIx.Count - 1; if (i % footeringFactor == 0 || isLastItem) { footerItems.Add(new KeyValuePair <string, long>(termWithPostingList.Key, newIxFile.Position)); } await WriteTermAndPostingList(newIxFile, termWithPostingList.Key, termWithPostingList.Value); ++i; } var footerStartPosition = newIxFile.Position; await WriteIndexFooter(newIxFile, footerItems); newIxFile.Position = 0; await newIxFile.WriteLongAsync(footerStartPosition); await newIxFile.FlushAsync(); } } }
public Node2(Measured <T, M> m, T v1, T v2) : base(m, m.Append(m.Measure(v1), m.Measure(v2))) { _v1 = v1; _v2 = v2; }
public Node2(Measured <T, M> m, Node2 <T, M> other) : base(m, m.Append(m.Measure(other.V1), m.Measure(other.V2))) { _v1 = other.V1; _v2 = other.V2; }
public static MakeTree <T, M> MakeTree(Measured <T, M> m) { return(new MakeTree <T, M>(m)); }