static FingerTree <Node <T, M>, M> AddDigits0 <T, M>(Measured <T, M> m, FingerTree <Node <T, M>, M> m1,
Digit <T, M> s1, Digit <T, M> p2, FingerTree <Node <T, M>, M> m2)
{
var mk = new MakeTree <T, M>(m);
return(s1.Match(x1 => p2.Match(y1 => Append1(m, m1, mk.Node2(x1.V, y1.V), m2),
y2 => Append1(m, m1, mk.Node3(x1.V, y2.V1, y2.V2), m2),
y3 => Append2(m, m1, mk.Node2(x1.V, y3.V1), mk.Node2(y3.V2, y3.V3), m2),
y4 => Append2(m, m1, mk.Node3(x1.V, y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2)),
x2 => p2.Match(y1 => Append1(m, m1, mk.Node3(x2.V1, x2.V2, y1.V), m2),
y2 => Append2(m, m1, mk.Node2(x2.V1, x2.V2), mk.Node2(y2.V1, y2.V2), m2),
y3 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, y3.V1), mk.Node2(y3.V2, y3.V3),
m2),
y4 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, y4.V1),
mk.Node3(y4.V2, y4.V3, y4.V4), m2)),
x3 => p2.Match(y1 => Append2(m, m1, mk.Node2(x3.V1, x3.V2), mk.Node2(x3.V3, y1.V), m2),
y2 => Append2(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node2(y2.V1, y2.V2), m2),
y3 => Append2(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3),
mk.Node3(y3.V1, y3.V2, y3.V3), m2),
y4 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node2(y4.V1, y4.V2),
mk.Node2(y4.V3, y4.V4), m2)),
x4 => p2.Match(y1 => Append2(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node2(x4.V4, y1.V), m2),
y2 => Append2(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, y2.V1, y2.V2), m2),
y3 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node2(x4.V4, y3.V1),
mk.Node2(y3.V2, y3.V3), m2),
y4 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, y4.V1, y4.V2),
mk.Node2(y4.V3, y4.V4), m2))));
}
public void Set(double Value)
{
var old = this.Value;
#region Measuring
if (this.Value != Value)
{
this.Value = Value;
if (Measuring != null)
{
Measuring.Invoke(this, new StableMeasureEventArgs(Value));
}
}
#endregion
if (IsStart)
{
if (!IsComplete)
{
#region Complete Start
if (Math.Abs(Value - old) < ErrorRange)
{
IsComplete = true;
CompleteTIme = DateTime.Now;
}
#endregion
}
else
{
#region Complete Instance
if (Math.Abs(Value - old) > ErrorRange)
{
IsComplete = false;
}
else
{
var dv = (DateTime.Now - CompleteTIme).TotalMilliseconds;
if (dv > MeasureTime)
{
IsComplete = IsStart = false;
if (Measured != null)
{
Measured.Invoke(this, new StableMeasureEventArgs(Value));
}
}
}
#endregion
}
}
else
{
#region Start
if (Math.Abs(old - Value) > ErrorRange)
{
IsStart = true;
StarTime = DateTime.Now;
}
#endregion
}
}
public Node3(Measured <T, M> m, Node3 <T, M> other)
: base(m, m.Append(m.Measure(other.V1), m.Append(m.Measure(other.V2), m.Measure(other.V3))))
{
_v1 = other.V1;
_v2 = other.V2;
_v3 = other.V3;
}
public Three(Measured <T, M> m, T v1, T v2, T v3)
: base(m, m.Append(m.Measure(v1), m.Append(m.Measure(v2), m.Measure(v3))))
{
_v1 = v1;
_v2 = v2;
_v3 = v3;
}
private async Task <DataFileCommitInfo> CommitIfRequired()
{
if (_nonPersistedQ.Count < _batchSize)
{
return(null);
}
await _commitLock.WaitAsync().ConfigureAwait(false);
try
{
if (_nonPersistedQ.Count < _batchSize)
{
return(null);
}
using (Measured.Operation("datafile_commit"))
{
return(await Commit().ConfigureAwait(false));
}
}
finally
{
_commitLock.Release();
}
}
public Digit <U, M> Map <U>(Func <T, U> f, Measured <U, M> m)
{
return(Match <Digit <U, M> >(one => new One <U, M>(m, f(one.V)),
two => new Two <U, M>(m, f(two.V1), f(two.V2)),
three => new Three <U, M>(m, f(three.V1), f(three.V2), f(three.V3)),
four => new Four <U, M>(m, f(four.V1), f(four.V2), f(four.V3), f(four.V4))));
}
static async Task TestWithoutBatching()
{
var instrumentWriter = new AggregatingInstrumentWriter();
Measured.Initialize(instrumentWriter);
var docs = Enumerable.Range(1, TestSetSize).Select(id => GenerateRandomDocument(id));
using (var db = new Database(DataDir, RootIndexDir))
{
var counter = 0;
var sw = Stopwatch.StartNew();
foreach (var doc in docs)
{
await db.AddDocument(doc);
if (++counter % 1000 == 0)
{
Console.WriteLine("Added {0} documents to the DB in {1}", counter, sw.Elapsed);
}
}
await db.Commit();
Console.WriteLine("Finished adding documents in {0}", sw.Elapsed);
instrumentWriter.WriteStats(Console.Out);
RunSearches(db);
instrumentWriter.WriteStats(Console.Out);
}
Console.ReadLine();
}
static FingerTree <Node <Node <T, M>, M>, M> AddDigits2 <T, M>(Measured <Node <T, M>, M> m,
FingerTree <Node <Node <T, M>, M>, M> m1,
Digit <Node <T, M>, M> x, Node <T, M> n1, Node <T, M> n2,
Digit <Node <T, M>, M> y,
FingerTree <Node <Node <T, M>, M>, M> m2)
{
var mk = new MakeTree <Node <T, M>, M>(m);
return(x.Match(x1 => y.Match(y1 => Append2(m, m1, mk.Node2(x1.V, n1), mk.Node2(n2, y1.V), m2),
y2 => Append2(m, m1, mk.Node3(x1.V, n1, n2), mk.Node2(y2.V1, y2.V2), m2),
y3 => Append2(m, m1, mk.Node3(x1.V, n1, n2), mk.Node3(y3.V1, y3.V2, y3.V3), m2),
y4 => Append3(m, m1, mk.Node3(x1.V, n1, n2), mk.Node2(y4.V1, y4.V2),
mk.Node2(y4.V3, y4.V4), m2)),
x2 => y.Match(y1 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node2(n2, y1.V), m2),
y2 => Append2(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node3(n2, y2.V1, y2.V2), m2),
y3 => Append3(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node2(n2, y3.V1),
mk.Node2(y3.V2, y3.V3), m2),
y4 => Append3(m, m1, mk.Node3(x2.V1, x2.V2, n1), mk.Node3(n2, y4.V1, y4.V2),
mk.Node2(y4.V3, y4.V4), m2)),
x3 => y.Match(y1 => Append2(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(n1, n2, y1.V), m2),
y2 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node2(n1, n2),
mk.Node2(y2.V1, y2.V2), m2),
y3 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(n1, n2, y3.V1),
mk.Node2(y3.V2, y3.V3), m2),
y4 => Append3(m, m1, mk.Node3(x3.V1, x3.V2, x3.V3), mk.Node3(n1, n2, y4.V1),
mk.Node3(y4.V2, y4.V3, y4.V4), m2)),
x4 => y.Match(y1 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node2(x4.V4, n1),
mk.Node2(n2, y1.V), m2),
y2 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, n1, n2),
mk.Node2(y2.V1, y2.V2), m2),
y3 => Append3(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, n1, n2),
mk.Node3(y3.V1, y3.V2, y3.V3), m2),
y4 => Append4(m, m1, mk.Node3(x4.V1, x4.V2, x4.V3), mk.Node3(x4.V4, n1, n2),
mk.Node2(y4.V1, y4.V2), mk.Node2(y4.V3, y4.V4), m2))));
}
/// <summary>
/// Update measurement from the system reading.
/// </summary>
public void UpdateMeasurement()
{
var s = Measured.PointwiseDivide(Current).ToColumnWiseArray();
H = M.Diagonal(2, 2, s);
_logger.Info("Current measurement " + F);
}
public Four(Measured <T, M> m, T v1, T v2, T v3, T v4)
: base(m, m.Append(m.Measure(v1), m.Append(m.Measure(v2), m.Append(m.Measure(v3), m.Measure(v4)))))
{
_v1 = v1;
_v4 = v4;
_v2 = v2;
_v3 = v3;
}
public override FingerTree <T, M> Concat(FingerTree <T, M> t)
{
Measured <T, M> m = Measured;
return(t.Match(e => this,
s => AddRight(s.Item),
d => new Deep <T, M>(m, m.Append(Size, d.Size), _prefix,
AddDigits0(m, _middle, _suffix, d._prefix, d._middle), d._suffix)));
}
public async Task AddTerm(string term, long dataFileOffset)
{
using (Measured.Operation("add_term_to_in_memory_ix"))
{
AddTermToInMemoryIndex(term, dataFileOffset);
}
await CommitIfHighMemoryPressure();
}
/// <summary>
/// Updates <see cref="AbsoluteSize" /> and <see cref="AbsolutePosition" />.
/// </summary>
public virtual void Measure()
{
var parentGui = Parent as GuiBase2D;
var parentSize = parentGui?.AbsoluteSize ?? (parentGui?.AbsoluteSize ?? new Vector2());
var parentPosition = parentGui?.AbsolutePosition ?? new Vector2();
AbsoluteSize = Size.toAbsolute(parentSize).round();
var position = Position;
var xScaleOffset = 0.0f;
var yScaleOffset = 0.0f;
var xAbsoluteOffset = 0;
var yAbsoluteOffset = 0;
var xMult = 1;
var yMult = 1;
if (_alignmentX == AlignmentX.Right)
{
xScaleOffset = 1;
xAbsoluteOffset = (int)-AbsoluteSize.X;
xMult = -1;
}
else if (_alignmentX == AlignmentX.Center)
{
xScaleOffset = 0.5f;
xAbsoluteOffset = (int)-AbsoluteSize.X / 2;
}
if (_alignmentY == AlignmentY.Bottom)
{
yScaleOffset = 1;
yAbsoluteOffset = (int)-AbsoluteSize.Y;
yMult = -1;
}
else if (_alignmentY == AlignmentY.Middle)
{
yScaleOffset = 0.5f;
yAbsoluteOffset = (int)-AbsoluteSize.Y / 2;
}
position = new UDim2(xScaleOffset + position.Scale.X * xMult, xAbsoluteOffset + position.Absolute.X * xMult,
yScaleOffset + position.Scale.Y * xMult, yAbsoluteOffset + position.Absolute.Y * yMult);
AbsolutePosition = (parentPosition + position.toAbsolute(parentSize)).round();
Measured?.Fire();
Arrange();
foreach (var kv in Children)
{
var el = kv as GuiElement;
el?.Measure();
}
}
public Deep(Measured <T, M> measured, M size, Digit <T, M> prefix, FingerTree <Node <T, M>, M> middle,
Digit <T, M> suffix)
: base(measured, size)
{
_mk = new MakeTree <T, M>(measured);
_prefix = prefix;
_middle = middle;
_suffix = suffix;
}
public IEnumerable <Document> Filter(SearchQuery query, IEnumerable <Document> docs)
{
using (Measured.Operation("grep_filter"))
{
foreach (var matchingDoc in docs.Where(doc => IsMatch(query, doc)))
{
yield return(matchingDoc);
}
}
}
public override FingerTree <T, M> AddLeft(T a)
{
Measured <T, M> m = Measured;
M size = m.Append(m.Measure(a), Size);
return(_prefix.Match(x1 => new Deep <T, M>(m, size, _mk.Two(a, x1.V), _middle, _suffix),
x2 => new Deep <T, M>(m, size, _mk.Three(a, x2.V1, x2.V2), _middle, _suffix),
x3 => new Deep <T, M>(m, size, _mk.Four(a, x3.V1, x3.V2, x3.V3), _middle, _suffix),
x4 => new Deep <T, M>(m, size, _mk.Two(a, x4.V1),
_middle.AddLeft(_mk.Node3(x4.V2, x4.V3, x4.V4)), _suffix)));
}
public override FingerTree <T, M> AddRight(T a)
{
Measured <T, M> m = Measured;
M measure = m.Append(m.Measure(a), Size);
return(_suffix.Match(x1 => new Deep <T, M>(m, measure, _prefix, _middle, _mk.Two(x1.V, a)),
x2 => new Deep <T, M>(m, measure, _prefix, _middle, _mk.Three(x2.V1, x2.V2, a)),
x3 => new Deep <T, M>(m, measure, _prefix, _middle, _mk.Four(x3.V1, x3.V2, x3.V3, a)),
x4 => new Deep <T, M>(m, measure, _prefix, _middle.AddRight(_mk.Node3(x4.V1, x4.V2, x4.V3)),
_mk.Two(x4.V4, a))));
}
static async Task SearchExistingDb()
{
var instrumentWriter = new AggregatingInstrumentWriter();
Measured.Initialize(instrumentWriter);
using (var db = new Database(DataDir, RootIndexDir))
{
RunSearches(db);
instrumentWriter.WriteStats(Console.Out);
}
}
private IEnumerable <long> SearchIndexFile(string targetTerm, string indexFilePath)
{
var comparer = StringComparer.OrdinalIgnoreCase;
using (Measured.Operation("search_index_file"))
using (var ixFile = new FileStream(indexFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
// roughly navigate to the position in the index file where the term could be
var preliminaryPosition = SearchIndexFileFooter(ixFile, targetTerm, comparer, out var footerPos);
if (preliminaryPosition == null)
{
yield break;
}
ixFile.Position = preliminaryPosition.Value;
while (ixFile.Position != footerPos)
{
var nextTermOffset = ixFile.ReadLong();
var termLengthInBytes = ixFile.ReadInt();
var term = ixFile.ReadUtf8String(termLengthInBytes);
var cmpResult = comparer.Compare(term, targetTerm);
if (cmpResult > 0)
{
// We've got past the place where the target term could have been in the index
// (we conclude this because terms are sorted alphabetically)
break;
}
else if (cmpResult < 0)
{
ixFile.Position = nextTermOffset;
continue;
}
else
{
var postingListLength = ixFile.ReadInt();
for (int i = 0; i < postingListLength; i++)
{
var entry = ixFile.ReadLong();
yield return(entry);
}
break;
}
}
}
}
static FingerTree <Node <T, M>, M> Append1 <T, M>(Measured <T, M> m, FingerTree <Node <T, M>, M> t1, Node <T, M> n1,
FingerTree <Node <T, M>, M> t2)
{
return(t1.Match(e => t2.AddLeft(n1),
s => t2.AddLeft(n1).AddLeft(s.Item),
d => t2.Match(e2 => t1.AddRight(n1),
s2 => t1.AddRight(n1).AddRight(s2.Item),
d2 =>
new Deep <Node <T, M>, M>(m.Node, m.Append(m.Append(d.Size, m.Node.Measure(n1)), d2.Size),
d._prefix,
AddDigits1(m.Node, d._middle, d._suffix, n1, d2._prefix,
d2._middle),
d2._suffix))));
}
static async Task RecreateTestDb()
{
ClearData();
var instrumentWriter = new AggregatingInstrumentWriter();
Measured.Initialize(instrumentWriter);
using (var db = new Database(DataDir, RootIndexDir))
{
await WriteToDbByBatches(db);
instrumentWriter.WriteStats(Console.Out);
}
}
static async Task TestWithBatching()
{
ClearData();
var instrumentWriter = new AggregatingInstrumentWriter();
Measured.Initialize(instrumentWriter);
using (var db = new Database(DataDir, RootIndexDir))
{
await WriteToDbByBatches(db);
instrumentWriter.WriteStats(Console.Out);
RunSearches(db);
instrumentWriter.WriteStats(Console.Out);
}
Console.ReadLine();
}
private async Task TruncateByDocId(long docId)
{
using (Measured.Operation("wal_truncate"))
{
_file.Position = _file.Length;
while (_file.Position != 0)
{
var batch = await DocumentSerializer.DeserializeBatchFromRightToLeft(_file);
var firstDocIdInBatch = batch[0].Id;
if (docId >= firstDocIdInBatch)
{
await TruncateByFileOffset(_file.Position);
break;
}
}
}
}
public async Task CommitTran(Tran tran)
{
await _commitLock.WaitAsync().ConfigureAwait(false);
try
{
using (Measured.Operation("wal_commit"))
{
foreach (var doc in tran.Documents)
{
doc.Id = _docIdGenerator.GetNextId();
}
await DocumentSerializer.SerializeBatch(tran.Documents, _file).ConfigureAwait(false);
await _file.FlushAsync().ConfigureAwait(false);
}
}
finally
{
_commitLock.Release();
}
}
static FingerTree <Node <T, M>, M> Append4 <T, M>(Measured <T, M> m, FingerTree <Node <T, M>, M> t1, Node <T, M> n1,
Node <T, M> n2,
Node <T, M> n3, Node <T, M> n4, FingerTree <Node <T, M>, M> t2)
{
return(t1.Match(e => t2.AddLeft(n4).AddLeft(n3).AddLeft(n2).AddLeft(n1),
s => t2.AddLeft(n4).AddLeft(n3).AddLeft(n2).AddLeft(n1).AddLeft(s.Item),
d => t2.Match(e2 => t2.AddRight(n1).AddRight(n2).AddRight(n3).AddRight(n4),
s2 => t2.AddRight(n1).AddRight(n2).AddRight(n3).AddRight(n4).AddRight(s2.Item),
d2 =>
new Deep <Node <T, M>, M>(m.Node,
m.Append(
m.Append(
m.Append(
m.Append(
m.Append(d.Size, n1.Measure),
n2.Measure), n3.Measure),
n4.Measure),
d2.Size), d._prefix,
AddDigits4(m.Node, d._middle, d._suffix, n1, n2, n3, n4,
d2._prefix,
d2._middle),
d2._suffix))));
}
protected Digit(Measured <T, M> m, M size)
{
Size = size;
_m = m;
_mk = new MakeTree <T, M>(m);
}
private async Task CommitInternal()
{
using (Measured.Operation("commit_index_file"))
{
// detaching old in memory index so that all new updates will go to the new
// in memory index while the old one is being committed to disk
var newInMemoryIx = new InMemoryIndex();
var oldInMemoryIx = Interlocked.Exchange(ref _inMemoryIx, newInMemoryIx);
Interlocked.Exchange(ref _memoryPressureScore, 0);
if (oldInMemoryIx.IsEmpty)
{
// nothing to commit
return;
}
/*
* Index file structure:
*
* <file offset (in bytes) of footer (long)>
* <main part>
* <footer>
*
* Footer includes sqrt(N) sorted terms with their position in the main part and allows for faster seeks in O(2*sqrt(N))
*/
/*
* <main part> structure:
* <file offset (in bytes) of term 2> <term 1 length in bytes> <term 1 UTF-8 bytes> <number of items in the posting list 1> <item 1 of the posting list 1> ... <item N of the posting list 1>
* <file offset (in bytes) of term 3> <term 2 length in bytes> <term 2 UTF-8 bytes> <number of items in the posting list 2> <item 2 of the posting list 2> ... <item N of the posting list 2>
* ...
* (this goes on for all terms in the index)
*/
/*
* <footer> structure:
* (N - total number of terms in the index file, K = sqrt(N))
*
* <term 1 length in bytes> <term 1 UTF-8 bytes> <term 1 position in main index part>
* <term K length in bytes> <term K UTF-8 bytes> <term K position in main index part>
* <term 2K length in bytes> <term 2K UTF-8 bytes> <term 2K position in main index part>
* ...
* <last term length in bytes> <last term UTF-8 bytes> <last term position in main index part>
*
* (note that 1st and last terms are always included)
*/
var footeringFactor = (int)Math.Sqrt(oldInMemoryIx.Count);
var footerItems = new List <KeyValuePair <string, long> >(footeringFactor + 1);
var i = 0;
using (var newIxFile = OpenNextIndexFileForWrite())
{
newIxFile.Position += sizeof(long);
foreach (var termWithPostingList in oldInMemoryIx.OrderBy(x => x.Key))
{
var isLastItem = i == oldInMemoryIx.Count - 1;
if (i % footeringFactor == 0 || isLastItem)
{
footerItems.Add(new KeyValuePair <string, long>(termWithPostingList.Key, newIxFile.Position));
}
await WriteTermAndPostingList(newIxFile, termWithPostingList.Key, termWithPostingList.Value);
++i;
}
var footerStartPosition = newIxFile.Position;
await WriteIndexFooter(newIxFile, footerItems);
newIxFile.Position = 0;
await newIxFile.WriteLongAsync(footerStartPosition);
await newIxFile.FlushAsync();
}
}
}
public Node2(Measured <T, M> m, T v1, T v2)
: base(m, m.Append(m.Measure(v1), m.Measure(v2)))
{
_v1 = v1;
_v2 = v2;
}
public Node2(Measured <T, M> m, Node2 <T, M> other)
: base(m, m.Append(m.Measure(other.V1), m.Measure(other.V2)))
{
_v1 = other.V1;
_v2 = other.V2;
}
public static MakeTree <T, M> MakeTree(Measured <T, M> m)
{
return(new MakeTree <T, M>(m));
}
