public TrimDatabase(TrimDatabaseOptions options) { _blockCache = options.BlockCache(); _options = options; if (!System.IO.Directory.Exists(options.DatabaseFolder)) { System.IO.Directory.CreateDirectory(options.DatabaseFolder); } _databaseFolder = options.DatabaseFolder; var unsorted = new UnsortedStorageLayer(1, _databaseFolder, _blockCache); _storageLayers.Add(unsorted); for (var i = 2; i <= _options.Levels; i++) { var filesAtLevel = (int)Math.Ceiling(_options.FirstLevelMaxFileCount * Math.Pow(_options.FirstLevelMaxFileCount, i - 2)); _storageLayers.Add(new SortedStorageLayer(i, _databaseFolder, _blockCache, _options.FileSize, filesAtLevel)); } if (!_options.OpenReadOnly) { _skipList = _options.MemoryTable(); } }
public void ReturnsRowsThatDifferInMultipleColumns() { var table1 = new MemoryTable(new[] { new[] { "A", "B" }, new[] { "1", "0" }, new[] { "1", "2" }, new[] { "1", "4" }, new[] { "1", "6" }, new[] { "1", "8" }, }); var table2 = new MemoryTable(new[] { new[] { "A", "B" }, new[] { "1", "2" }, new[] { "2", "0" }, new[] { "1", "0" }, }); var sut = new TableComparer(new[] { "A", "B" }); var result = sut.Compare(table1, table2); Assert.AreEqual(3, result.RowCount); CollectionAssert.AreEqual(new[] { "1", "4" }, result.Data[0]); CollectionAssert.AreEqual(new[] { "1", "6" }, result.Data[1]); CollectionAssert.AreEqual(new[] { "1", "8" }, result.Data[2]); }
public void TestJounalBroken() { using var js = new KeepOpenMemoryStream(); using (var journal = new Journal(js, new PlaneDBOptions())) { for (var i = 0; i < COUNT; ++i) { var v = i.ToString(); journal.Put(v, v + v + v); if (i % 10 == 0) { journal.Put("o" + v, v + v + v); } if (i % 30 == 0) { Assert.IsTrue(journal.Update("o" + v, v + v + v + v)); } else if (i % 20 == 0) { journal.Remove("o" + v); } } } js.SetLength(js.Length - 1); var table = new MemoryTable(new PlaneDBOptions()); Journal.ReplayOnto(js, new PlaneDBOptions(), table); }
public MemoryDbObjectQuery(MemoryTable table, string indexName) { this.table = table; query = new QueryOperation { IndexName = indexName }; }
/// <param name="location">Directory that will store the PlaneDB</param> /// <param name="mode">File mode to use, supported are: CreateNew, Open (existing), OpenOrCreate</param> /// <param name="options">Options to use, such as the transformer, cache settings, etc.</param> /// <summary>Opens or creates a new PlaneDB.</summary> public PlaneDB(DirectoryInfo location, FileMode mode, PlaneDBOptions options) { options.Validate(); Location = location; this.options = options.Clone(); blockCache = new BlockCache(options.BlockCacheCapacity); memoryTable = new MemoryTable(options); if (mode == FileMode.CreateNew || mode == FileMode.OpenOrCreate) { location.Create(); } state = new PlaneDBState(location, mode, options); ReopenSSTables(); if (tables.Count(i => i.Value.DiskSize < 524288) > 2) { MaybeMerge(true); } if (!options.ThreadSafe) { return; } mergeThread = new Thread(MergeLoop) { Priority = ThreadPriority.BelowNormal, Name = "Plane-Background-Merge" }; mergeThread.Start(); }
public void ThrowsWhenNoColumns() { var sut = new MemoryTable(new[] { new string[0], new string[0], }); }
public void TestJounalVarLength() { using var ms = new KeepOpenMemoryStream(); using (var journal = new Journal(ms, new PlaneDBOptions().EnableCompression())) { for (var i = 0; i < COUNT; ++i) { var v = i.ToString(); journal.Put(v, v + v + v); if (i % 10 == 0) { journal.Put("o" + v, v + v + v); } if (i % 30 == 0) { Assert.IsTrue(journal.Update("o" + v, v + v + v + v)); } else if (i % 20 == 0) { journal.Remove("o" + v); } } } var table = new MemoryTable(new PlaneDBOptions().EnableCompression()); Journal.ReplayOnto(ms, new PlaneDBOptions().EnableCompression(), table); for (var i = 0; i < COUNT; ++i) { var v = i.ToString(); Assert.IsTrue(table.ContainsKey(v, out _)); Assert.IsFalse(table.ContainsKey($"nope{v}", out _)); Assert.IsTrue(table.TryGet(v, out var s)); Assert.AreEqual(v + v + v, s); if (i % 30 == 0) { Assert.IsTrue(table.ContainsKey("o" + v, out _)); Assert.IsTrue(table.TryGet("o" + v, out var val)); Assert.AreEqual(v + v + v + v, val); } else if (i % 20 == 0) { Assert.IsTrue(table.ContainsKey("o" + v, out _)); Assert.IsTrue(table.TryGet("o" + v, out var val)); Assert.IsNull(val); } else if (i % 10 == 0) { Assert.IsTrue(table.ContainsKey("o" + v, out _)); Assert.IsTrue(table.TryGet("o" + v, out var val)); Assert.AreEqual(v + v + v, val); } } }
public void SingleRow() { var sut = new MemoryTable(new[] { new[] { "A", "B" }, new[] { "1", "2" }, }); Assert.AreEqual(1, sut.RowCount); }
public void ReturnsAllRowsWhenTheOtherTableIsEmpty() { var table1 = new MemoryTable(new[] { new[] { "A", "B" }, new[] { "1", "2" }, new[] { "3", "4" } }); var table2 = new MemoryTable(new[] { new[] { "A", "B" } }); var sut = new TableComparer(new[] { "A" }); var result = sut.Compare(table1, table2); Assert.AreEqual(2, result.RowCount); }
public void SingleColumn() { var sut = new MemoryTable(new[] { new[] { "A" }, new[] { "1" }, new[] { "2" }, }); Assert.AreEqual(1, sut.ColCount); }
public void MultipleColumns() { var sut = new MemoryTable(new[] { new[] { "A", "B", "C" }, new[] { "1", "2", "3" }, new[] { "4", "5", "6" }, }); Assert.AreEqual(3, sut.ColCount); }
public void SingleColumn() { var sut = new MemoryTable(new[] { new[] { "A" }, new[] { "1" }, new[] { "2" }, }); CollectionAssert.AreEqual(new[] { "A" }, sut.Columns); }
/// <summary> /// Invoke the function /// </summary> /// <param name="function"></param> /// <param name="arguments"></param> public object[] InvokeFunction(string function, MemoryTable table) { try { LuaFunction func = this.lua.GetFunction(function); return(func.Call(new Object[] { ToLuaTable(table) })); } catch (Exception e) { return(new Object[] { }); } }
public void MultipleRows() { var sut = new MemoryTable(new[] { new[] { "A", "B" }, new[] { "1", "2" }, new[] { "3", "4" }, new[] { "5", "6" }, }); Assert.AreEqual(3, sut.RowCount); }
/// <summary> /// Invoke the function /// </summary> /// <param name="function"></param> /// <param name="arguments"></param> public object[] InvokeFunction(string function, MemoryTable table) { try { LuaFunction func = this.lua.GetFunction(function); return func.Call(new Object[]{ToLuaTable(table)}); } catch (Exception e) { return new Object[] { }; } }
public void PrintsOutTheHeader() { var sut = new MemoryTable(new[] { new[] { "A", "B", "C" }, new[] { "1", "2", "3" }, new[] { "4", "5", "6" }, }); var output = new List <string>(); sut.Dump(new[] { "A", "C" }, output.Add); Assert.AreEqual(" A C", output[0]); }
private Enumerator GetInternalEnumerator(bool readValues) { var t = new List <IReadOnlyTable>(); if (!memoryTable.IsEmpty) { var mt = new MemoryTable(options); memoryTable.CopyTo(mt); t.Add(mt); } t.AddRange(tables.Select(i => i.Value)); return(new Enumerator(t.ToArray(), readValues, options.Comparer)); }
public void MultipleRowsAndColumns() { var sut = new MemoryTable(new[] { new[] { "A", "B", "C" }, new[] { "1", "2", "3" }, new[] { "4", "5", "6" }, }); Assert.AreEqual("1", sut.Data[0][0]); Assert.AreEqual("2", sut.Data[0][1]); Assert.AreEqual("3", sut.Data[0][2]); Assert.AreEqual("4", sut.Data[1][0]); Assert.AreEqual("5", sut.Data[1][1]); Assert.AreEqual("6", sut.Data[1][2]); }
public void PrintsOutTheData() { var sut = new MemoryTable(new[] { new[] { "A", "B", "C" }, new[] { "1", "2", "3" }, new[] { "4", "5", "6" }, }); var output = new List <string>(); sut.Dump(new[] { "A", "C" }, output.Add); Assert.AreEqual(3, output.Count); Assert.AreEqual("00001 1 3", output[1]); Assert.AreEqual("00002 4 6", output[2]); }
/// <summary>Gets the table with the specified name.</summary> /// <typeparam name="T"></typeparam> /// <param name="name">The name.</param> /// <returns></returns> public ITable <T> GetTable <T>(string name = null) where T : struct { var result = new MemoryTable <T>(); if (name == null) { name = result.Name; } if (HasTable(name)) { result.LoadTable(GetTable(name)); } return(result); }
/// <summary> /// Convert to friendly object /// </summary> /// <param name="data"></param> /// <returns></returns> public MemoryTable JSONToTable(JObject jobject) { MemoryTable mt = new MemoryTable(); foreach (KeyValuePair <String, Newtonsoft.Json.Linq.JToken> obj in jobject) { if (obj.Value.GetType() == typeof(JObject)) { MemoryTable _mt = this.JSONToTable((JObject)obj.Value); mt[obj.Key] = _mt; } else { mt[obj.Key] = obj.Value; } } return(mt); }
public void SetupTables() { var mockUserData = new[] { new User { Id = Guid.NewGuid(), Username = "******", Email = "*****@*****.**", Password = Storage.EncryptPassword("somepassword"), }, }; var mockGroupData = new[] { new Group { Id = 1, Name = "PMI51", Deleted = false } }; var mockGroupUserData = new[] { new GroupUser { GroupRef = 1, UserRef = mockUserData[0].Id } }; var mockUserRoleData = new[] { new UserRole { UserRef = mockUserData[0].Id, RoleRef = (int)Role.Teacher } }; var mockUsers = new MemoryTable <User>(mockUserData); var mockGroups = new MemoryTable <Group>(mockGroupData); var mockGroupUsers = new MemoryTable <GroupUser>(mockGroupUserData); var mockUserRoles = new MemoryTable <UserRole>(mockUserRoleData); MockDataContext.SetupGet(c => c.Users).Returns(mockUsers); MockDataContext.SetupGet(c => c.Groups).Returns(mockGroups); MockDataContext.SetupGet(c => c.GroupUsers).Returns(mockGroupUsers); MockDataContext.SetupGet(c => c.UserRoles).Returns(mockUserRoles); }
public void SetupTables() { var mockUserData = new[] { new User { Id = Guid.NewGuid(), Username = "******", Email = "*****@*****.**", Password = this.Storage.EncryptPassword("somepassword"), }, }; var mockGroupData = new[] { new Group { Id = 1, Name = "PMI51", Deleted = false } }; var mockGroupUserData = new[] { new GroupUser { GroupRef = 1, UserRef = mockUserData[0].Id } }; var mockUserRoleData = new[] { new UserRole { UserRef = mockUserData[0].Id, RoleRef = (int)Role.Teacher } }; var mockUserRatingsData = new[] { new UserTopicRating { UserId = mockUserData[0].Id, TopicId = 1, Rating = 5 } }; var mockUsers = new MemoryTable <User>(mockUserData); var mockGroups = new MemoryTable <Group>(mockGroupData); var mockGroupUsers = new MemoryTable <GroupUser>(mockGroupUserData); var mockUserRoles = new MemoryTable <UserRole>(mockUserRoleData); var mockUserRatings = new MemoryTable <UserTopicRating>(mockUserRatingsData); this.MockDataContext.SetupGet(c => c.Users).Returns(mockUsers); this.MockDataContext.SetupGet(c => c.Groups).Returns(mockGroups); this.MockDataContext.SetupGet(c => c.GroupUsers).Returns(mockGroupUsers); this.MockDataContext.SetupGet(c => c.UserRoles).Returns(mockUserRoles); this.MockDataContext.SetupGet(c => c.UserTopicRatings).Returns(mockUserRatings); this.MockLmsService.Setup(l => l.FindService <IUserService>().GetUsers()).Returns(this.DataContext.Users); }
public MemoryTable getTable(JObject obj) { MemoryTable table = new MemoryTable(); foreach (KeyValuePair <String, Newtonsoft.Json.Linq.JToken> member in obj) { if (obj[member.Key].GetType() == typeof(String)) { table[member.Key] = obj; } else if (obj[member.Key].GetType() == typeof(JObject)) { MemoryTable _table = getTable((JObject)obj[member.Key]); table[member.Key] = member.Value; continue; } } return(table); }
private async Task CanPutInOrder(MemoryTable memoryTable) { var loadedWords = CommonData.Words; foreach (var word in loadedWords) { var utf8 = Encoding.UTF8.GetBytes(word); var value = Encoding.UTF8.GetBytes($"VALUE={word}"); var result = memoryTable.Put(utf8, value); Assert.True(result); } foreach (var word in loadedWords) { var utf8 = Encoding.UTF8.GetBytes(word); var value = Encoding.UTF8.GetBytes($"VALUE={word}"); var result = memoryTable.TryGet(utf8, out _); Assert.Equal(SearchResult.Found, result); } }
private LuaTable ToLuaTable(MemoryTable table) { String name = "memory_table_" + new Random().Next().ToString(); lua.NewTable(name); LuaTable newTable = lua.GetTable(name); foreach (KeyValuePair <String, Object> obj in newTable) { if (obj.GetType() == typeof(MemoryTable)) { newTable[obj.Key] = ToLuaTable((MemoryTable)obj.Value); } else { newTable[obj.Key] = obj.Value; } } return(newTable); }
public TrimDatabase(Func <MemoryTable> inMemoryFunc, BlockCache blockCache, int levels, string databaseFolder, int targetFileSizeL2) { _blockCache = blockCache; if (!System.IO.Directory.Exists(databaseFolder)) { System.IO.Directory.CreateDirectory(databaseFolder); } _databaseFolder = databaseFolder; _inMemoryFunc = inMemoryFunc; var unsorted = new UnsortedStorageLayer(1, _databaseFolder, _blockCache); _storageLayers.Add(unsorted); for (var i = 2; i <= levels; i++) { _storageLayers.Add(new SortedStorageLayer(i, _databaseFolder, _blockCache, targetFileSizeL2)); } _skipList = _inMemoryFunc(); }
public async Task SaveMemoryTable(MemoryTable skipList) { using var fs = System.IO.File.Open(_fileName, System.IO.FileMode.CreateNew); var pipeWriter = PipeWriter.Create(fs); var iterator = skipList.GetEnumerator(); if (!iterator.MoveNext()) { throw new ArgumentOutOfRangeException("Empty iterator nothing to save"); } var itemCount = 0; WriteBlocks(pipeWriter, iterator, ref itemCount); await pipeWriter.FlushAsync(); WriteTOC(pipeWriter, _metaData.FirstKey.Span, _metaData.LastKey.Span, itemCount); await pipeWriter.FlushAsync(); await pipeWriter.CompleteAsync(); }
private void SkipListPutWorking(MemoryTable memoryTable) { var string1 = Encoding.UTF8.GetBytes("This is the first test string"); var string2 = Encoding.UTF8.GetBytes("This is the second test string"); var string3 = Encoding.UTF8.GetBytes("This is the missing test string"); var value1 = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; var value2 = new byte[] { 10, 11, 12, 13, 14, 15 }; _ = new byte[] { 16, 17, 18, 19, 20 }; memoryTable.Put(string1, value1); memoryTable.Put(string2, value2); var result = memoryTable.TryGet(string1, out var valueResult); Assert.Equal(SearchResult.Found, result); Assert.Equal(value1, valueResult.ToArray()); var result2 = memoryTable.TryGet(string2, out _); Assert.Equal(SearchResult.Found, result2); var result3 = memoryTable.TryGet(string3, out _); Assert.Equal(SearchResult.NotFound, result3); }
public void SetupTables() { var mockTagsData = new[] { new Tag { Id = 1, Name = "C++" }, new Tag { Id = 2, Name = "C++ STL" }, new Tag { Id = 3, Name = "C#" }, new Tag { Id = 4, Name = "C#.NET" } }; var mockTopicTagsData = new[] { new TopicTag { TopicId = 1, TagId = 1 }, new TopicTag { TopicId = 1, TagId = 2 }, new TopicTag { TopicId = 1, TagId = 4 }, new TopicTag { TopicId = 2, TagId = 1 }, new TopicTag { TopicId = 2, TagId = 2 }, new TopicTag { TopicId = 2, TagId = 4 } }; this.UserId = Guid.NewGuid(); var mockUserScoresData = new[] { new UserScore { Score = 80, TagId = 1, UserId = this.UserId }, new UserScore { Score = 90, TagId = 2, UserId = this.UserId }, new UserScore { Score = 60, TagId = 3, UserId = this.UserId } }; var mockTopicScoresData = new[] { new TopicScore { Score = 40, TagId = 1, TopicId = 1 }, new TopicScore { Score = 40, TagId = 2, TopicId = 1 }, new TopicScore { Score = 40, TagId = 4, TopicId = 1 } }; var mockTags = new MemoryTable<Tag>(mockTagsData); var mockTopicTags = new MemoryTable<TopicTag>(mockTopicTagsData); var mockUserScores = new MemoryTable<UserScore>(mockUserScoresData); var mockTopicScores = new MemoryTable<TopicScore>(mockTopicScoresData); this.MockDataContext.SetupGet(c => c.Tags).Returns(mockTags); this.MockDataContext.SetupGet(c => c.TopicTags).Returns(mockTopicTags); this.MockDataContext.SetupGet(c => c.UserScores).Returns(mockUserScores); this.MockDataContext.SetupGet(c => c.TopicScores).Returns(mockTopicScores); }
public void SetupTables() { var mockUserData = new[] { new User {Id = Guid.NewGuid(), Username = "******", Email = "*****@*****.**", Password = Storage.EncryptPassword("somepassword"), }, }; var mockGroupData = new[] { new Group {Id = 1, Name = "PMI51", Deleted = false} }; var mockGroupUserData = new[] { new GroupUser {GroupRef = 1, UserRef = mockUserData[0].Id} }; var mockUserRoleData = new[] { new UserRole {UserRef = mockUserData[0].Id, RoleRef = (int)Role.Teacher} }; var mockUsers = new MemoryTable<User>(mockUserData); var mockGroups = new MemoryTable<Group>(mockGroupData); var mockGroupUsers = new MemoryTable<GroupUser>(mockGroupUserData); var mockUserRoles = new MemoryTable<UserRole>(mockUserRoleData); MockDataContext.SetupGet(c => c.Users).Returns(mockUsers); MockDataContext.SetupGet(c => c.Groups).Returns(mockGroups); MockDataContext.SetupGet(c => c.GroupUsers).Returns(mockGroupUsers); MockDataContext.SetupGet(c => c.UserRoles).Returns(mockUserRoles); }
private LuaTable ToLuaTable(MemoryTable table) { String name = "memory_table_" + new Random().Next().ToString(); lua.NewTable(name); LuaTable newTable = lua.GetTable(name); foreach (KeyValuePair<String, Object> obj in newTable) { if (obj.GetType() == typeof(MemoryTable)) { newTable[obj.Key] = ToLuaTable((MemoryTable)obj.Value); } else { newTable[obj.Key] = obj.Value; } } return newTable; }
public object TableToNative(MemoryTable table) { return ToLuaTable(table); }
public MemoryTable getTable(JObject obj) { MemoryTable table = new MemoryTable(); foreach (KeyValuePair<String, Newtonsoft.Json.Linq.JToken> member in obj) { if (obj[member.Key].GetType() == typeof(String)) { table[member.Key] = obj; } else if(obj[member.Key].GetType() == typeof(JObject)) { MemoryTable _table = getTable((JObject)obj[member.Key]); table[member.Key] = member.Value; continue; } } return table; }
public MemoryDbTableIndex(MemoryTable table, string indexName) { this.table = table; this.indexName = indexName; }
/// <summary> /// Convert to friendly object /// </summary> /// <param name="data"></param> /// <returns></returns> public MemoryTable JSONToTable(JObject jobject) { MemoryTable mt = new MemoryTable(); foreach (KeyValuePair<String, Newtonsoft.Json.Linq.JToken> obj in jobject) { if (obj.Value.GetType() == typeof(JObject)) { MemoryTable _mt = this.JSONToTable((JObject)obj.Value); mt[obj.Key] = _mt; } else { mt[obj.Key] = obj.Value; } } return mt; }
public object TableToNative(MemoryTable table) { return(ToLuaTable(table)); }