public void DeleteBatch(HashSet <int> ids, TableInfo table_info, WriteBatchWithConstraints batch, Dictionary <string, int> trans_count_cache, Dictionary <string, KeyValuePair <byte[], HashSet <int> > > string_cache, Dictionary <string, Tuple <IndexNewData, IndexDeletedData, IndexChangedData> > memory_index_meta) { var existing_ids = new HashSet <int>(); var historic_columns = GetAllColumnsWithHistoric(table_info.TableNumber); foreach (var id in ids) { var key = MakeIndexKey(new IndexKeyInfo() { TableNumber = table_info.TableNumber, ColumnNumber = table_info.ColumnNumbers["Id"], Val = BitConverter.GetBytes(id).MyReverseNoCopy(), Id = id }); var index_val = leveld_db.Get(key); if (index_val == null) { continue; } existing_ids.Add(id); foreach (var column in historic_columns) { if (column.Item2 != LinqDbTypes.binary_ && column.Item2 != LinqDbTypes.string_) { var column_name = table_info.ColumnNumbers.Select(f => new { f.Key, f.Value }).Where(f => f.Value == column.Item1).FirstOrDefault(); IndexDeletedData index_deleted = null; if (column_name != null && memory_index_meta.ContainsKey(column_name.Key)) { index_deleted = memory_index_meta[column_name.Key].Item2; } DeleteDataColumn(batch, table_info.TableNumber, column.Item1, column.Item2, id, index_deleted); } else if (column.Item2 == LinqDbTypes.binary_) { DeleteBinaryColumn(batch, table_info.TableNumber, column.Item1, column.Item2, id); } else { DeleteStringColumn(batch, table_info.TableNumber, column.Item1, table_info, column.Item2, id, string_cache); } } } DecrementCount(table_info, existing_ids, batch, trans_count_cache); //WriteStringCacheToBatch(batch, string_cache, table_info, trans_phase_cache); }
public void UpdateBatch(UpdateInfo info, Dictionary <int, object> values, TableInfo table_info, WriteBatchWithConstraints batch, Dictionary <string, KeyValuePair <byte[], HashSet <int> > > string_cache, Dictionary <string, Tuple <IndexNewData, IndexDeletedData, IndexChangedData> > memory_index_meta) { foreach (var item in values) { var key = MakeIndexKey(new IndexKeyInfo() { TableNumber = info.TableNumber, ColumnNumber = table_info.ColumnNumbers["Id"], Val = BitConverter.GetBytes(item.Key).MyReverseNoCopy(), Id = item.Key }); var index_val = leveld_db.Get(key); if (index_val == null) { continue; } object value = item.Value; if (info.ColumnType == LinqDbTypes.string_) { SaveStringData(batch, (string)item.Value, info.ColumnName, info.TableInfo, item.Key, string_cache, false); } else if (info.ColumnType == LinqDbTypes.binary_) { SaveBinaryColumn(batch, value, info.ColumnName, info.TableInfo, item.Key, false); } else { IndexDeletedData index_deleted = null; IndexNewData index_new = null; IndexChangedData index_changed = null; if (memory_index_meta.ContainsKey(info.ColumnName)) { index_deleted = memory_index_meta[info.ColumnName].Item2; index_new = memory_index_meta[info.ColumnName].Item1; index_changed = memory_index_meta[info.ColumnName].Item3; } SaveDataColumn(batch, value, info.ColumnName, info.ColumnType, info.TableInfo, item.Key, false, index_new, index_changed); } } //WriteStringCacheToBatch(batch, string_cache, table_info, null); }
void DeleteDataColumn(WriteBatchWithConstraints batch, short TableNumber, short ColumnNumber, LinqDbTypes ColumnType, int id, IndexDeletedData index_deleted) { var key_info = new IndexKeyInfo() { ColumnNumber = ColumnNumber, TableNumber = TableNumber, ColumnType = ColumnType, Id = id }; var value_key = MakeValueKey(key_info); var old_val = leveld_db.Get(value_key); if (old_val == null) //maybe new column added and delete invoked { return; } bool is_old_negative = false; if (ValsEqual(old_val, NullConstant)) { is_old_negative = false; } else if (ColumnType == LinqDbTypes.double_ && BitConverter.ToDouble(old_val.MyReverseWithCopy(), 0) < 0) { is_old_negative = true; old_val = BitConverter.GetBytes((BitConverter.ToDouble(old_val.MyReverseWithCopy(), 0) * -1)).MyReverseNoCopy(); } else if (ColumnType == LinqDbTypes.int_ && BitConverter.ToInt32(old_val.MyReverseWithCopy(), 0) < 0) { is_old_negative = true; old_val = BitConverter.GetBytes((BitConverter.ToInt32(old_val.MyReverseWithCopy(), 0) * -1)).MyReverseNoCopy(); } byte[] index_key = null; if (is_old_negative) { index_key = MakeIndexKey(new IndexKeyInfo() { TableNumber = TableNumber, ColumnNumber = (short)(-1 * ColumnNumber), ColumnType = ColumnType, Val = old_val, Id = id }); } else { index_key = MakeIndexKey(new IndexKeyInfo() { TableNumber = TableNumber, ColumnNumber = ColumnNumber, ColumnType = ColumnType, Val = old_val, Id = id }); } batch.Delete(value_key); batch.Delete(index_key); //keep up the memory index, if there is one if (index_deleted != null) { index_deleted.Ids.Add(id); } }
public void MakeNewPropSnapshot(string type, string prop, string snapshot_id, IndexNewData index_new, IndexDeletedData index_deleted, IndexChangedData index_changed) { var latest_key = type + "|" + prop; var latest_snapshot_id = latest_snapshots[latest_key]; var key = type + "|" + prop + "|" + latest_snapshot_id; var index = indexes[key]; var ids_index = indexes[type + "|Id|" + latest_snapshots[type + "|Id"]]; var parts = new List <IndexPart>(); var new_index = new IndexGeneric() { ColumnName = index.ColumnName, ColumnType = index.ColumnType, GroupListMapping = index.GroupListMapping, IndexType = index.IndexType, Parts = parts, TypeName = index.TypeName }; bool has_changed_int = index_changed != null && index_changed.IntValues != null && index_changed.IntValues.Any(); bool has_changed_double = index_changed != null && index_changed.DoubleValues != null && index_changed.DoubleValues.Any(); bool has_deleted = index_deleted != null && index_deleted.Ids.Any(); //var dbloomsize = 100000; //var deleted_bloom = new List<bool>(dbloomsize); //for (int i = 0; i < dbloomsize; i++) //{ // deleted_bloom.Add(false); //} //if (has_deleted) //{ // foreach (var did in index_deleted.Ids) // { // deleted_bloom[did % dbloomsize] = true; // } //} //var changed_int_bloom = new List<bool>(dbloomsize); //for (int i = 0; i < dbloomsize; i++) //{ // changed_int_bloom.Add(false); //} //if (has_changed_int) //{ // foreach (var cid in index_changed.IntValues) // { // changed_int_bloom[cid.Key % dbloomsize] = true; // } //} //var changed_double_bloom = new List<bool>(dbloomsize); //for (int i = 0; i < dbloomsize; i++) //{ // changed_double_bloom.Add(false); //} //if (has_changed_double) //{ // foreach (var cid in index_changed.DoubleValues) // { // changed_double_bloom[cid.Key % dbloomsize] = true; // } //} switch (index.ColumnType) { case LinqDbTypes.int_: if (index.IndexType == IndexType.PropertyOnly) //property index only { for (int i = 0; i < index.Parts.Count; i++) { var p = index.Parts[i]; p.Ids = ids_index.Parts[i].IntValues; IndexPart np = null; HashSet <int> removed = null; int icount = p.IntValues.Count; for (int j = 0; j < icount; j++) { var cid = p.Ids[j]; if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid)) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), IntValues = new List <int>(p.IntValues) }; } np.IntValues[j] = index_changed.IntValues[cid]; } //if (j >= p.Ids.Count()) //{ // var a = 5; //} if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid)) { if (removed == null) { removed = new HashSet <int>(); } removed.Add((int)cid); } } if (removed != null) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), IntValues = new List <int>(p.IntValues) }; } var npp = new IndexPart() { Ids = new List <int>(), IntValues = new List <int>() }; for (int z = 0; z < np.Ids.Count; z++) { if (!removed.Contains((int)np.Ids[z])) { npp.Ids.Add(np.Ids[z]); npp.IntValues.Add(np.IntValues[z]); } } np = npp; } if (np != null) { p = np; //wow, finally! } parts.Add(p); } if (index_new != null) { IndexPart lastp = null; if (parts.Any()) { lastp = parts.LastOrDefault(); } if (lastp == null) { lastp = new IndexPart() { Ids = new List <int>(), IntValues = new List <int>() }; parts.Add(lastp); } else { lastp = new IndexPart() { Ids = new List <int>(lastp.Ids), IntValues = new List <int>(lastp.IntValues) }; parts[parts.Count() - 1] = lastp; } for (int i = 0; i < index_new.IntValues.Count; i++) { if (lastp.Ids.Count == 1000) { lastp = new IndexPart() { Ids = new List <int>(), IntValues = new List <int>() }; parts.Add(lastp); } lastp.IntValues.Add(index_new.IntValues[i]); lastp.Ids.Add(index_new.Ids[i]); } } for (int i = 0; i < index.Parts.Count(); i++) { index.Parts[i].Ids = null; } } else if (index.IndexType == IndexType.GroupOnly) //group index only { int map = 0; if (index.GroupListMapping.Any()) { map = index.GroupListMapping.Max(f => f.Value); map++; } for (int i = 0; i < index.Parts.Count; i++) { var p = index.Parts[i]; p.Ids = ids_index.Parts[i].IntValues; IndexPart np = null; HashSet <int> removed = null; int icount = p.GroupValues.Count; for (int j = 0; j < icount; j++) { int cid = p.Ids[j]; if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid)) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), GroupValues = new List <int>(p.GroupValues) }; } if (!index.GroupListMapping.ContainsKey(index_changed.IntValues[cid])) { index.GroupListMapping[index_changed.IntValues[cid]] = map; map++; } np.GroupValues[j] = index.GroupListMapping[index_changed.IntValues[cid]]; } if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid)) { if (removed == null) { removed = new HashSet <int>(); } removed.Add(cid); } } if (removed != null) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), GroupValues = new List <int>(p.GroupValues) }; } var npp = new IndexPart() { Ids = new List <int>(), GroupValues = new List <int>() }; for (int z = 0; z < np.Ids.Count; z++) { if (!removed.Contains((int)np.Ids[z])) { npp.Ids.Add(np.Ids[z]); npp.GroupValues.Add(np.GroupValues[z]); } } np = npp; } if (np != null) { p = np; //wow, finally! } parts.Add(p); } if (index_new != null) { IndexPart lastp = null; if (parts.Any()) { lastp = parts.LastOrDefault(); } if (lastp == null) { lastp = new IndexPart() { Ids = new List <int>(), GroupValues = new List <int>() }; parts.Add(lastp); } else { lastp = new IndexPart() { Ids = new List <int>(lastp.Ids), GroupValues = new List <int>(lastp.GroupValues) }; parts[parts.Count() - 1] = lastp; } for (int i = 0; i < index_new.IntValues.Count; i++) { if (lastp.Ids.Count == 1000) { lastp = new IndexPart() { Ids = new List <int>(), GroupValues = new List <int>() }; parts.Add(lastp); } if (!index.GroupListMapping.ContainsKey((int)index_new.IntValues[i])) { index.GroupListMapping[(int)index_new.IntValues[i]] = map; map++; } lastp.GroupValues.Add(index.GroupListMapping[(int)index_new.IntValues[i]]); lastp.Ids.Add(index_new.Ids[i]); } } for (int i = 0; i < index.Parts.Count(); i++) { index.Parts[i].Ids = null; } } else //both { int map = 0; if (index.GroupListMapping.Any()) { map = index.GroupListMapping.Max(f => f.Value); map++; } for (int i = 0; i < index.Parts.Count; i++) { var p = index.Parts[i]; p.Ids = ids_index.Parts[i].IntValues; IndexPart np = null; HashSet <int> removed = null; int icount = p.GroupValues.Count; for (int j = 0; j < icount; j++) { int cid = p.Ids[j]; if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid)) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), GroupValues = new List <int>(p.GroupValues), IntValues = new List <int>(p.IntValues) }; } if (!index.GroupListMapping.ContainsKey(index_changed.IntValues[cid])) { index.GroupListMapping[index_changed.IntValues[cid]] = map; map++; } np.GroupValues[j] = index.GroupListMapping[index_changed.IntValues[cid]]; np.IntValues[j] = index_changed.IntValues[cid]; } if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid)) { if (removed == null) { removed = new HashSet <int>(); } removed.Add(cid); } } if (removed != null) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), GroupValues = new List <int>(p.GroupValues), IntValues = new List <int>(p.IntValues) }; } var npp = new IndexPart() { Ids = new List <int>(), GroupValues = new List <int>(), IntValues = new List <int>() }; for (int z = 0; z < np.Ids.Count; z++) { if (!removed.Contains((int)np.Ids[z])) { npp.Ids.Add(np.Ids[z]); npp.GroupValues.Add(np.GroupValues[z]); npp.IntValues.Add(np.IntValues[z]); } } np = npp; } if (np != null) { p = np; //wow, finally! } parts.Add(p); } if (index_new != null) { IndexPart lastp = null; if (parts.Any()) { lastp = parts.LastOrDefault(); } if (lastp == null) { lastp = new IndexPart() { Ids = new List <int>(), GroupValues = new List <int>(), IntValues = new List <int>() }; parts.Add(lastp); } else { lastp = new IndexPart() { Ids = new List <int>(lastp.Ids), GroupValues = new List <int>(lastp.GroupValues), IntValues = new List <int>(lastp.IntValues) }; parts[parts.Count() - 1] = lastp; } for (int i = 0; i < index_new.IntValues.Count; i++) { if (lastp.Ids.Count == 1000) { lastp = new IndexPart() { Ids = new List <int>(), GroupValues = new List <int>(), IntValues = new List <int>() }; parts.Add(lastp); } if (!index.GroupListMapping.ContainsKey((int)index_new.IntValues[i])) { index.GroupListMapping[(int)index_new.IntValues[i]] = map; map++; } lastp.GroupValues.Add(index.GroupListMapping[(int)index_new.IntValues[i]]); lastp.IntValues.Add(index_new.IntValues[i]); lastp.Ids.Add(index_new.Ids[i]); } } for (int i = 0; i < index.Parts.Count(); i++) { index.Parts[i].Ids = null; } } break; case LinqDbTypes.double_: case LinqDbTypes.DateTime_: for (int i = 0; i < index.Parts.Count; i++) { var p = index.Parts[i]; p.Ids = ids_index.Parts[i].IntValues; IndexPart np = null; HashSet <int> removed = null; int icount = p.DoubleValues.Count; for (int j = 0; j < icount; j++) { int cid = p.Ids[j]; if (has_changed_double /*&& changed_double_bloom[cid % dbloomsize]*/ && index_changed.DoubleValues.ContainsKey(cid)) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), DoubleValues = new List <double>(p.DoubleValues) }; } np.DoubleValues[j] = index_changed.DoubleValues[cid]; } if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid)) { if (removed == null) { removed = new HashSet <int>(); } removed.Add(cid); } } if (removed != null) { if (np == null) { np = new IndexPart() { Ids = new List <int>(p.Ids), DoubleValues = new List <double>(p.DoubleValues) }; } var npp = new IndexPart() { Ids = new List <int>(), DoubleValues = new List <double>() }; for (int z = 0; z < np.Ids.Count; z++) { if (!removed.Contains((int)np.Ids[z])) { npp.Ids.Add(np.Ids[z]); npp.DoubleValues.Add(np.DoubleValues[z]); } } np = npp; } if (np != null) { p = np; //wow, finally! } parts.Add(p); } if (index_new != null) { IndexPart lastp = null; if (parts.Any()) { lastp = parts.LastOrDefault(); } if (lastp == null) { lastp = new IndexPart() { Ids = new List <int>(), DoubleValues = new List <double>() }; parts.Add(lastp); } else { lastp = new IndexPart() { Ids = new List <int>(lastp.Ids), DoubleValues = new List <double>(lastp.DoubleValues) }; parts[parts.Count() - 1] = lastp; } for (int i = 0; i < index_new.DoubleValues.Count; i++) { if (lastp.Ids.Count == 1000) { lastp = new IndexPart() { Ids = new List <int>(), DoubleValues = new List <double>() }; parts.Add(lastp); } lastp.DoubleValues.Add(index_new.DoubleValues[i]); lastp.Ids.Add(index_new.Ids[i]); } } for (int i = 0; i < index.Parts.Count(); i++) { index.Parts[i].Ids = null; } break; default: break; } //free old snapshots if ((DateTime.Now - last_cleanup[type + "|" + prop]).TotalMilliseconds > 30000) { var alive = snapshots_alive[type + "|" + prop]; var new_alive = new List <Tuple <bool, string> >(); //bool - schedueled for deletion foreach (var sn in alive) { if (!sn.Item1) { var nsn = new Tuple <bool, string>(true, sn.Item2); new_alive.Add(nsn); } else { IndexGeneric val = null; indexes.TryRemove(sn.Item2, out val); } } new_alive.Add(new Tuple <bool, string>(false, type + "|" + prop + "|" + snapshot_id)); snapshots_alive[type + "|" + prop] = new_alive; last_cleanup[type + "|" + prop] = DateTime.Now; } else { snapshots_alive[type + "|" + prop].Add(new Tuple <bool, string>(false, type + "|" + prop + "|" + snapshot_id)); } key = type + "|" + prop + "|" + snapshot_id; indexes[key] = new_index; latest_snapshots[latest_key] = snapshot_id; }