public TrimDatabase(TrimDatabaseOptions options)
{
_blockCache = options.BlockCache();
_options = options;
if (!System.IO.Directory.Exists(options.DatabaseFolder))
{
System.IO.Directory.CreateDirectory(options.DatabaseFolder);
}
_databaseFolder = options.DatabaseFolder;
var unsorted = new UnsortedStorageLayer(1, _databaseFolder, _blockCache);
_storageLayers.Add(unsorted);
for (var i = 2; i <= _options.Levels; i++)
{
var filesAtLevel = (int)Math.Ceiling(_options.FirstLevelMaxFileCount * Math.Pow(_options.FirstLevelMaxFileCount, i - 2));
_storageLayers.Add(new SortedStorageLayer(i, _databaseFolder, _blockCache, _options.FileSize, filesAtLevel));
}
if (!_options.OpenReadOnly)
{
_skipList = _options.MemoryTable();
}
}
public void ReturnsRowsThatDifferInMultipleColumns()
{
var table1 = new MemoryTable(new[]
{
new[] { "A", "B" },
new[] { "1", "0" },
new[] { "1", "2" },
new[] { "1", "4" },
new[] { "1", "6" },
new[] { "1", "8" },
});
var table2 = new MemoryTable(new[]
{
new[] { "A", "B" },
new[] { "1", "2" },
new[] { "2", "0" },
new[] { "1", "0" },
});
var sut = new TableComparer(new[] { "A", "B" });
var result = sut.Compare(table1, table2);
Assert.AreEqual(3, result.RowCount);
CollectionAssert.AreEqual(new[] { "1", "4" }, result.Data[0]);
CollectionAssert.AreEqual(new[] { "1", "6" }, result.Data[1]);
CollectionAssert.AreEqual(new[] { "1", "8" }, result.Data[2]);
}
public void TestJounalBroken()
{
using var js = new KeepOpenMemoryStream();
using (var journal = new Journal(js, new PlaneDBOptions())) {
for (var i = 0; i < COUNT; ++i)
{
var v = i.ToString();
journal.Put(v, v + v + v);
if (i % 10 == 0)
{
journal.Put("o" + v, v + v + v);
}
if (i % 30 == 0)
{
Assert.IsTrue(journal.Update("o" + v, v + v + v + v));
}
else if (i % 20 == 0)
{
journal.Remove("o" + v);
}
}
}
js.SetLength(js.Length - 1);
var table = new MemoryTable(new PlaneDBOptions());
Journal.ReplayOnto(js, new PlaneDBOptions(), table);
}
public MemoryDbObjectQuery(MemoryTable table, string indexName)
{
this.table = table;
query = new QueryOperation {
IndexName = indexName
};
}
/// <param name="location">Directory that will store the PlaneDB</param>
/// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param>
/// <param name="options">Options to use, such as the transformer, cache settings, etc.</param>
/// <summary>Opens or creates a new PlaneDB.</summary>
public PlaneDB(DirectoryInfo location, FileMode mode, PlaneDBOptions options)
{
options.Validate();
Location = location;
this.options = options.Clone();
blockCache = new BlockCache(options.BlockCacheCapacity);
memoryTable = new MemoryTable(options);
if (mode == FileMode.CreateNew || mode == FileMode.OpenOrCreate)
{
location.Create();
}
state = new PlaneDBState(location, mode, options);
ReopenSSTables();
if (tables.Count(i => i.Value.DiskSize < 524288) > 2)
{
MaybeMerge(true);
}
if (!options.ThreadSafe)
{
return;
}
mergeThread = new Thread(MergeLoop)
{
Priority = ThreadPriority.BelowNormal, Name = "Plane-Background-Merge"
};
mergeThread.Start();
}
public void ThrowsWhenNoColumns()
{
var sut = new MemoryTable(new[]
{
new string[0],
new string[0],
});
}
public void TestJounalVarLength()
{
using var ms = new KeepOpenMemoryStream();
using (var journal = new Journal(ms, new PlaneDBOptions().EnableCompression())) {
for (var i = 0; i < COUNT; ++i)
{
var v = i.ToString();
journal.Put(v, v + v + v);
if (i % 10 == 0)
{
journal.Put("o" + v, v + v + v);
}
if (i % 30 == 0)
{
Assert.IsTrue(journal.Update("o" + v, v + v + v + v));
}
else if (i % 20 == 0)
{
journal.Remove("o" + v);
}
}
}
var table = new MemoryTable(new PlaneDBOptions().EnableCompression());
Journal.ReplayOnto(ms, new PlaneDBOptions().EnableCompression(), table);
for (var i = 0; i < COUNT; ++i)
{
var v = i.ToString();
Assert.IsTrue(table.ContainsKey(v, out _));
Assert.IsFalse(table.ContainsKey($"nope{v}", out _));
Assert.IsTrue(table.TryGet(v, out var s));
Assert.AreEqual(v + v + v, s);
if (i % 30 == 0)
{
Assert.IsTrue(table.ContainsKey("o" + v, out _));
Assert.IsTrue(table.TryGet("o" + v, out var val));
Assert.AreEqual(v + v + v + v, val);
}
else if (i % 20 == 0)
{
Assert.IsTrue(table.ContainsKey("o" + v, out _));
Assert.IsTrue(table.TryGet("o" + v, out var val));
Assert.IsNull(val);
}
else if (i % 10 == 0)
{
Assert.IsTrue(table.ContainsKey("o" + v, out _));
Assert.IsTrue(table.TryGet("o" + v, out var val));
Assert.AreEqual(v + v + v, val);
}
}
}
public void SingleRow()
{
var sut = new MemoryTable(new[]
{
new[] { "A", "B" },
new[] { "1", "2" },
});
Assert.AreEqual(1, sut.RowCount);
}
public void ReturnsAllRowsWhenTheOtherTableIsEmpty()
{
var table1 = new MemoryTable(new[] { new[] { "A", "B" }, new[] { "1", "2" }, new[] { "3", "4" } });
var table2 = new MemoryTable(new[] { new[] { "A", "B" } });
var sut = new TableComparer(new[] { "A" });
var result = sut.Compare(table1, table2);
Assert.AreEqual(2, result.RowCount);
}
public void SingleColumn()
{
var sut = new MemoryTable(new[]
{
new[] { "A" },
new[] { "1" },
new[] { "2" },
});
Assert.AreEqual(1, sut.ColCount);
}
public void MultipleColumns()
{
var sut = new MemoryTable(new[]
{
new[] { "A", "B", "C" },
new[] { "1", "2", "3" },
new[] { "4", "5", "6" },
});
Assert.AreEqual(3, sut.ColCount);
}
public void SingleColumn()
{
var sut = new MemoryTable(new[]
{
new[] { "A" },
new[] { "1" },
new[] { "2" },
});
CollectionAssert.AreEqual(new[] { "A" }, sut.Columns);
}
/// <summary>
/// Invoke the function
/// </summary>
/// <param name="function"></param>
/// <param name="arguments"></param>
public object[] InvokeFunction(string function, MemoryTable table)
{
try
{
LuaFunction func = this.lua.GetFunction(function);
return(func.Call(new Object[] { ToLuaTable(table) }));
}
catch (Exception e)
{
return(new Object[] { });
}
}
public void MultipleRows()
{
var sut = new MemoryTable(new[]
{
new[] { "A", "B" },
new[] { "1", "2" },
new[] { "3", "4" },
new[] { "5", "6" },
});
Assert.AreEqual(3, sut.RowCount);
}
/// <summary>
/// Invoke the function
/// </summary>
/// <param name="function"></param>
/// <param name="arguments"></param>
public object[] InvokeFunction(string function, MemoryTable table)
{
try
{
LuaFunction func = this.lua.GetFunction(function);
return func.Call(new Object[]{ToLuaTable(table)});
}
catch (Exception e)
{
return new Object[] { };
}
}
public void PrintsOutTheHeader()
{
var sut = new MemoryTable(new[]
{
new[] { "A", "B", "C" },
new[] { "1", "2", "3" },
new[] { "4", "5", "6" },
});
var output = new List <string>();
sut.Dump(new[] { "A", "C" }, output.Add);
Assert.AreEqual(" A C", output[0]);
}
private Enumerator GetInternalEnumerator(bool readValues)
{
var t = new List <IReadOnlyTable>();
if (!memoryTable.IsEmpty)
{
var mt = new MemoryTable(options);
memoryTable.CopyTo(mt);
t.Add(mt);
}
t.AddRange(tables.Select(i => i.Value));
return(new Enumerator(t.ToArray(), readValues, options.Comparer));
}
public void MultipleRowsAndColumns()
{
var sut = new MemoryTable(new[]
{
new[] { "A", "B", "C" },
new[] { "1", "2", "3" },
new[] { "4", "5", "6" },
});
Assert.AreEqual("1", sut.Data[0][0]);
Assert.AreEqual("2", sut.Data[0][1]);
Assert.AreEqual("3", sut.Data[0][2]);
Assert.AreEqual("4", sut.Data[1][0]);
Assert.AreEqual("5", sut.Data[1][1]);
Assert.AreEqual("6", sut.Data[1][2]);
}
public void PrintsOutTheData()
{
var sut = new MemoryTable(new[]
{
new[] { "A", "B", "C" },
new[] { "1", "2", "3" },
new[] { "4", "5", "6" },
});
var output = new List <string>();
sut.Dump(new[] { "A", "C" }, output.Add);
Assert.AreEqual(3, output.Count);
Assert.AreEqual("00001 1 3", output[1]);
Assert.AreEqual("00002 4 6", output[2]);
}
/// <summary>Gets the table with the specified name.</summary>
/// <typeparam name="T"></typeparam>
/// <param name="name">The name.</param>
/// <returns></returns>
public ITable <T> GetTable <T>(string name = null)
where T : struct
{
var result = new MemoryTable <T>();
if (name == null)
{
name = result.Name;
}
if (HasTable(name))
{
result.LoadTable(GetTable(name));
}
return(result);
}
/// <summary>
/// Convert to friendly object
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public MemoryTable JSONToTable(JObject jobject)
{
MemoryTable mt = new MemoryTable();
foreach (KeyValuePair <String, Newtonsoft.Json.Linq.JToken> obj in jobject)
{
if (obj.Value.GetType() == typeof(JObject))
{
MemoryTable _mt = this.JSONToTable((JObject)obj.Value);
mt[obj.Key] = _mt;
}
else
{
mt[obj.Key] = obj.Value;
}
}
return(mt);
}
public void SetupTables()
{
var mockUserData = new[]
{
new User
{
Id = Guid.NewGuid(),
Username = "******",
Email = "*****@*****.**",
Password = Storage.EncryptPassword("somepassword"),
},
};
var mockGroupData = new[]
{
new Group {
Id = 1, Name = "PMI51", Deleted = false
}
};
var mockGroupUserData = new[]
{
new GroupUser {
GroupRef = 1, UserRef = mockUserData[0].Id
}
};
var mockUserRoleData = new[]
{
new UserRole {
UserRef = mockUserData[0].Id, RoleRef = (int)Role.Teacher
}
};
var mockUsers = new MemoryTable <User>(mockUserData);
var mockGroups = new MemoryTable <Group>(mockGroupData);
var mockGroupUsers = new MemoryTable <GroupUser>(mockGroupUserData);
var mockUserRoles = new MemoryTable <UserRole>(mockUserRoleData);
MockDataContext.SetupGet(c => c.Users).Returns(mockUsers);
MockDataContext.SetupGet(c => c.Groups).Returns(mockGroups);
MockDataContext.SetupGet(c => c.GroupUsers).Returns(mockGroupUsers);
MockDataContext.SetupGet(c => c.UserRoles).Returns(mockUserRoles);
}
public void SetupTables()
{
var mockUserData = new[]
{
new User
{
Id = Guid.NewGuid(),
Username = "******",
Email = "*****@*****.**",
Password = this.Storage.EncryptPassword("somepassword"),
},
};
var mockGroupData = new[] { new Group {
Id = 1, Name = "PMI51", Deleted = false
} };
var mockGroupUserData = new[] { new GroupUser {
GroupRef = 1, UserRef = mockUserData[0].Id
} };
var mockUserRoleData = new[] { new UserRole {
UserRef = mockUserData[0].Id, RoleRef = (int)Role.Teacher
} };
var mockUserRatingsData = new[] { new UserTopicRating {
UserId = mockUserData[0].Id, TopicId = 1, Rating = 5
} };
var mockUsers = new MemoryTable <User>(mockUserData);
var mockGroups = new MemoryTable <Group>(mockGroupData);
var mockGroupUsers = new MemoryTable <GroupUser>(mockGroupUserData);
var mockUserRoles = new MemoryTable <UserRole>(mockUserRoleData);
var mockUserRatings = new MemoryTable <UserTopicRating>(mockUserRatingsData);
this.MockDataContext.SetupGet(c => c.Users).Returns(mockUsers);
this.MockDataContext.SetupGet(c => c.Groups).Returns(mockGroups);
this.MockDataContext.SetupGet(c => c.GroupUsers).Returns(mockGroupUsers);
this.MockDataContext.SetupGet(c => c.UserRoles).Returns(mockUserRoles);
this.MockDataContext.SetupGet(c => c.UserTopicRatings).Returns(mockUserRatings);
this.MockLmsService.Setup(l => l.FindService <IUserService>().GetUsers()).Returns(this.DataContext.Users);
}
public MemoryTable getTable(JObject obj)
{
MemoryTable table = new MemoryTable();
foreach (KeyValuePair <String, Newtonsoft.Json.Linq.JToken> member in obj)
{
if (obj[member.Key].GetType() == typeof(String))
{
table[member.Key] = obj;
}
else if (obj[member.Key].GetType() == typeof(JObject))
{
MemoryTable _table = getTable((JObject)obj[member.Key]);
table[member.Key] = member.Value;
continue;
}
}
return(table);
}
private async Task CanPutInOrder(MemoryTable memoryTable)
{
var loadedWords = CommonData.Words;
foreach (var word in loadedWords)
{
var utf8 = Encoding.UTF8.GetBytes(word);
var value = Encoding.UTF8.GetBytes($"VALUE={word}");
var result = memoryTable.Put(utf8, value);
Assert.True(result);
}
foreach (var word in loadedWords)
{
var utf8 = Encoding.UTF8.GetBytes(word);
var value = Encoding.UTF8.GetBytes($"VALUE={word}");
var result = memoryTable.TryGet(utf8, out _);
Assert.Equal(SearchResult.Found, result);
}
}
private LuaTable ToLuaTable(MemoryTable table)
{
String name = "memory_table_" + new Random().Next().ToString();
lua.NewTable(name);
LuaTable newTable = lua.GetTable(name);
foreach (KeyValuePair <String, Object> obj in newTable)
{
if (obj.GetType() == typeof(MemoryTable))
{
newTable[obj.Key] = ToLuaTable((MemoryTable)obj.Value);
}
else
{
newTable[obj.Key] = obj.Value;
}
}
return(newTable);
}
public TrimDatabase(Func <MemoryTable> inMemoryFunc, BlockCache blockCache, int levels, string databaseFolder, int targetFileSizeL2)
{
_blockCache = blockCache;
if (!System.IO.Directory.Exists(databaseFolder))
{
System.IO.Directory.CreateDirectory(databaseFolder);
}
_databaseFolder = databaseFolder;
_inMemoryFunc = inMemoryFunc;
var unsorted = new UnsortedStorageLayer(1, _databaseFolder, _blockCache);
_storageLayers.Add(unsorted);
for (var i = 2; i <= levels; i++)
{
_storageLayers.Add(new SortedStorageLayer(i, _databaseFolder, _blockCache, targetFileSizeL2));
}
_skipList = _inMemoryFunc();
}
public async Task SaveMemoryTable(MemoryTable skipList)
{
using var fs = System.IO.File.Open(_fileName, System.IO.FileMode.CreateNew);
var pipeWriter = PipeWriter.Create(fs);
var iterator = skipList.GetEnumerator();
if (!iterator.MoveNext())
{
throw new ArgumentOutOfRangeException("Empty iterator nothing to save");
}
var itemCount = 0;
WriteBlocks(pipeWriter, iterator, ref itemCount);
await pipeWriter.FlushAsync();
WriteTOC(pipeWriter, _metaData.FirstKey.Span, _metaData.LastKey.Span, itemCount);
await pipeWriter.FlushAsync();
await pipeWriter.CompleteAsync();
}
private void SkipListPutWorking(MemoryTable memoryTable)
{
var string1 = Encoding.UTF8.GetBytes("This is the first test string");
var string2 = Encoding.UTF8.GetBytes("This is the second test string");
var string3 = Encoding.UTF8.GetBytes("This is the missing test string");
var value1 = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
var value2 = new byte[] { 10, 11, 12, 13, 14, 15 };
_ = new byte[] { 16, 17, 18, 19, 20 };
memoryTable.Put(string1, value1);
memoryTable.Put(string2, value2);
var result = memoryTable.TryGet(string1, out var valueResult);
Assert.Equal(SearchResult.Found, result);
Assert.Equal(value1, valueResult.ToArray());
var result2 = memoryTable.TryGet(string2, out _);
Assert.Equal(SearchResult.Found, result2);
var result3 = memoryTable.TryGet(string3, out _);
Assert.Equal(SearchResult.NotFound, result3);
}
public void SetupTables()
{
var mockTagsData = new[]
{
new Tag { Id = 1, Name = "C++" },
new Tag { Id = 2, Name = "C++ STL" },
new Tag { Id = 3, Name = "C#" },
new Tag { Id = 4, Name = "C#.NET" }
};
var mockTopicTagsData = new[]
{
new TopicTag { TopicId = 1, TagId = 1 },
new TopicTag { TopicId = 1, TagId = 2 },
new TopicTag { TopicId = 1, TagId = 4 },
new TopicTag { TopicId = 2, TagId = 1 },
new TopicTag { TopicId = 2, TagId = 2 },
new TopicTag { TopicId = 2, TagId = 4 }
};
this.UserId = Guid.NewGuid();
var mockUserScoresData = new[]
{
new UserScore { Score = 80, TagId = 1, UserId = this.UserId },
new UserScore { Score = 90, TagId = 2, UserId = this.UserId },
new UserScore { Score = 60, TagId = 3, UserId = this.UserId }
};
var mockTopicScoresData = new[]
{
new TopicScore { Score = 40, TagId = 1, TopicId = 1 },
new TopicScore { Score = 40, TagId = 2, TopicId = 1 },
new TopicScore { Score = 40, TagId = 4, TopicId = 1 }
};
var mockTags = new MemoryTable<Tag>(mockTagsData);
var mockTopicTags = new MemoryTable<TopicTag>(mockTopicTagsData);
var mockUserScores = new MemoryTable<UserScore>(mockUserScoresData);
var mockTopicScores = new MemoryTable<TopicScore>(mockTopicScoresData);
this.MockDataContext.SetupGet(c => c.Tags).Returns(mockTags);
this.MockDataContext.SetupGet(c => c.TopicTags).Returns(mockTopicTags);
this.MockDataContext.SetupGet(c => c.UserScores).Returns(mockUserScores);
this.MockDataContext.SetupGet(c => c.TopicScores).Returns(mockTopicScores);
}
public void SetupTables()
{
var mockUserData = new[]
{
new User {Id = Guid.NewGuid(), Username = "******", Email = "*****@*****.**", Password = Storage.EncryptPassword("somepassword"), },
};
var mockGroupData = new[]
{
new Group {Id = 1, Name = "PMI51", Deleted = false}
};
var mockGroupUserData = new[]
{
new GroupUser {GroupRef = 1, UserRef = mockUserData[0].Id}
};
var mockUserRoleData = new[]
{
new UserRole {UserRef = mockUserData[0].Id, RoleRef = (int)Role.Teacher}
};
var mockUsers = new MemoryTable<User>(mockUserData);
var mockGroups = new MemoryTable<Group>(mockGroupData);
var mockGroupUsers = new MemoryTable<GroupUser>(mockGroupUserData);
var mockUserRoles = new MemoryTable<UserRole>(mockUserRoleData);
MockDataContext.SetupGet(c => c.Users).Returns(mockUsers);
MockDataContext.SetupGet(c => c.Groups).Returns(mockGroups);
MockDataContext.SetupGet(c => c.GroupUsers).Returns(mockGroupUsers);
MockDataContext.SetupGet(c => c.UserRoles).Returns(mockUserRoles);
}
private LuaTable ToLuaTable(MemoryTable table)
{
String name = "memory_table_" + new Random().Next().ToString();
lua.NewTable(name);
LuaTable newTable = lua.GetTable(name);
foreach (KeyValuePair<String, Object> obj in newTable)
{
if (obj.GetType() == typeof(MemoryTable))
{
newTable[obj.Key] = ToLuaTable((MemoryTable)obj.Value);
}
else
{
newTable[obj.Key] = obj.Value;
}
}
return newTable;
}
public object TableToNative(MemoryTable table)
{
return ToLuaTable(table);
}
public MemoryTable getTable(JObject obj)
{
MemoryTable table = new MemoryTable();
foreach (KeyValuePair<String, Newtonsoft.Json.Linq.JToken> member in obj)
{
if (obj[member.Key].GetType() == typeof(String))
{
table[member.Key] = obj;
} else if(obj[member.Key].GetType() == typeof(JObject)) {
MemoryTable _table = getTable((JObject)obj[member.Key]);
table[member.Key] = member.Value;
continue;
}
}
return table;
}
public MemoryDbTableIndex(MemoryTable table, string indexName)
{
this.table = table;
this.indexName = indexName;
}
/// <summary>
/// Convert to friendly object
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public MemoryTable JSONToTable(JObject jobject)
{
MemoryTable mt = new MemoryTable();
foreach (KeyValuePair<String, Newtonsoft.Json.Linq.JToken> obj in jobject)
{
if (obj.Value.GetType() == typeof(JObject))
{
MemoryTable _mt = this.JSONToTable((JObject)obj.Value);
mt[obj.Key] = _mt;
}
else
{
mt[obj.Key] = obj.Value;
}
}
return mt;
}
public object TableToNative(MemoryTable table)
{
return(ToLuaTable(table));
}