OrderedTmpResult OrderValuesByDescending(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool all, ReadOptions ro) { OrderedTmpResult res = new OrderedTmpResult(); res.Data = new Dictionary <int, int>(); res.Take = take; res.Skip = skip; var key = MakeIndexSearchKey(new IndexKeyInfo() { TableNumber = order_info.TableNumber, ColumnNumber = order_info.ColumnNumber, ColumnType = order_info.ColumnType, Val = PostEverything }); int count = 0; using (var it = leveld_db.NewIterator(null, ro)) { it.Seek(key); if (!it.Valid()) { return(res); } var k = it.Key(); var info = GetIndexKey(k); if (info.NotKey || info.TableNumber != order_info.TableNumber || info.ColumnNumber != order_info.ColumnNumber) { it.Prev(); if (!it.Valid()) { return(res); } k = it.Key(); if (k == null) { return(res); } info = GetIndexKey(k); } var contains = new ContainsInfo() { ids = res_set.ResIds }; bool has = false; if (!info.NotKey && info.TableNumber == order_info.TableNumber && info.ColumnNumber == order_info.ColumnNumber) { if (!all) { has = EfficientContains(contains, info.Id); if (contains.is_sorted) { res_set.ResIds = contains.ids; } } if (all || has) { if (res.Skip == null || res.Skip == 0) { res.Data[info.Id] = count; count++; if (res.Take != null && count == res.Take) { return(res); } } else { res.Skip--; } } } else { return(res); } while (true) { it.Prev(); if (!it.Valid()) { return(res); } k = it.Key(); if (k == null) { return(res); } info = GetIndexKey(k); if (!info.NotKey && info.TableNumber == order_info.TableNumber && info.ColumnNumber == order_info.ColumnNumber) { if (!all) { has = EfficientContains(contains, info.Id); } if (all || has) { if (res.Skip == null || res.Skip == 0) { res.Data[info.Id] = count; count++; if (res.Take != null && count == res.Take) { return(res); } } else { res.Skip--; } } } else { return(res); } } } }
Dictionary <int, int> OrderValuesByDescendingWithNegatives(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool all, ReadOptions ro) { var pos_res = OrderValuesByDescending(order_info, res_set, take, skip, all, ro); int?take_left = pos_res.Take == null ? null : (pos_res.Take - pos_res.Data.Count); if (take_left == null || take_left > 0) { order_info.ColumnNumber *= -1; var neg_res = OrderValues(order_info, res_set, take_left, pos_res.Skip, all, ro); order_info.ColumnNumber *= -1; var pos_count = pos_res.Data.Count(); foreach (var pv in neg_res.Data) { pos_res.Data[pv.Key] = pv.Value + pos_count; } } return(pos_res.Data); }
Dictionary <int, int> OrderSomeValuesById(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool desc, Dictionary <long, byte[]> cache, ReadOptions ro, TableInfo table_info) { Dictionary <int, int> res = new Dictionary <int, int>(); var ids = res_set.ResIds; if (!IsSorted(ids)) { ids = ids.OrderBy(f => f).ToList(); } if (!desc) { if (skip == null) { skip = 0; } int count = 0; for (int i = 0; i < ids.Count(); i++) { if (skip > 0) { skip--; continue; } if (take == null || take > 0) { if (take > 0) { take--; } res[ids[i]] = count; count++; } else if (take == 0) { break; } } } else { if (skip == null) { skip = 0; } int count = 0; for (int i = ids.Count() - 1; i >= 0; i--) { if (skip > 0) { skip--; continue; } if (take == null || take > 0) { if (take > 0) { take--; } res[ids[i]] = count; count++; } else if (take == 0) { break; } } } return(res); }
Dictionary <int, int> OrderSomeValues(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool desc, Dictionary <long, byte[]> cache, ReadOptions ro, TableInfo table_info) { if (order_info.ColumnName != "Id") { var resindex = OrderSomeValuesWithIndex(order_info, res_set, take, skip, desc, cache, ro, table_info); if (resindex != null) { return(resindex); } } else { return(OrderSomeValuesById(order_info, res_set, take, skip, desc, cache, ro, table_info)); } Dictionary <int, int> res = new Dictionary <int, int>(); List <KeyValuePair <object, int> > tmp_res = new List <KeyValuePair <object, int> >(res_set.ResIds.Count()); foreach (var r in res_set.ResIds) { if (order_info.ColumnName == "Id") { tmp_res.Add(new KeyValuePair <object, int>(r, r)); } else { byte[] val = null; //if (!GetFromCache(order_info.ColumnNumber, r, cache, out val)) //{ if (order_info.ColumnType == LinqDbTypes.string_) { var key = MakeStringValueKey(new IndexKeyInfo() { TableNumber = order_info.TableNumber, ColumnNumber = order_info.ColumnNumber, Id = r }); val = leveld_db.Get(key, null, ro); } else { var key = MakeValueKey(new IndexKeyInfo() { TableNumber = order_info.TableNumber, ColumnNumber = order_info.ColumnNumber, Id = r }); val = leveld_db.Get(key, null, ro); } //} if (order_info.ColumnType != LinqDbTypes.string_ && ValsEqual(val, NullConstant) || order_info.ColumnType == LinqDbTypes.string_ && ValsEqual(val, BinaryOrStringValuePrefix)) { tmp_res.Add(new KeyValuePair <object, int>(null, r)); } else { if (order_info.ColumnType == LinqDbTypes.int_) { tmp_res.Add(new KeyValuePair <object, int>(BitConverter.ToInt32(val.MyReverseWithCopy(), 0), r)); } else if (order_info.ColumnType == LinqDbTypes.double_ || order_info.ColumnType == LinqDbTypes.DateTime_) { tmp_res.Add(new KeyValuePair <object, int>(BitConverter.ToDouble(val.MyReverseWithCopy(), 0), r)); } else if (order_info.ColumnType == LinqDbTypes.string_) { tmp_res.Add(new KeyValuePair <object, int>(Encoding.Unicode.GetString(val), r)); } } } } List <KeyValuePair <object, int> > t = null; if (!desc) { t = tmp_res.OrderBy(f => f.Key).ToList(); } else { t = tmp_res.OrderByDescending(f => f.Key).ToList(); } if (skip != null) { t = t.Skip((int)skip).ToList(); } if (take != null) { t = t.Take((int)take).ToList(); } int count = 0; foreach (var v in t) { res[v.Value] = count; count++; } return(res); }
public Dictionary <int, List <object> > SelectGrouppedCommon(HashSet <int> distinct_groups, OperResult fres, List <Tuple <string, string> > aggregates, TableInfo table_info, short column_number, string column_name, ReadOptions ro, int count) { Dictionary <int, List <object> > cdata = new Dictionary <int, List <object> >(); //var max_id = 0; //List<byte> check_list = null; //int bloom_max = 2000000; //List<byte> bloom_int = null; List <int> groups = new List <int>(); var skey = MakeSnapshotKey(table_info.TableNumber, /*tree.GroupingInfo.ColumnNumber*/ column_number); var snapid = leveld_db.Get(skey, null, ro); if (snapid == null || !existing_indexes.ContainsKey(table_info.Name) || !existing_indexes[table_info.Name].Contains(column_name)) { throw new LinqDbException("Linqdb: group index on property " + /*tree.GroupingInfo.ColumnName*/ column_name + " does not exist."); } string snapshot_id = Encoding.UTF8.GetString(snapid); skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers["Id"]); snapid = leveld_db.Get(skey, null, ro); string id_snapshot_id = Encoding.UTF8.GetString(snapid); if (!indexes.ContainsKey(table_info.Name + "|" + /*tree.GroupingInfo.ColumnName*/ column_name + "|" + snapshot_id)) { throw new LinqDbException("Linqdb: group index on property " + /*tree.GroupingInfo.ColumnName*/ column_name + " does not exist."); } var group_index = indexes[table_info.Name + "|" + /*tree.GroupingInfo.ColumnName*/ column_name + "|" + snapshot_id]; if (group_index.IndexType == IndexType.PropertyOnly) { throw new LinqDbException("Linqdb: group index on property " + /*tree.GroupingInfo.ColumnName*/ column_name + " does not exist."); } var ids_index = indexes[table_info.Name + "|Id|" + id_snapshot_id]; if (!fres.All) { var tmp = new IndexGeneric() { ColumnName = group_index.ColumnName, ColumnType = group_index.ColumnType, TypeName = group_index.TypeName, GroupListMapping = new ConcurrentDictionary <int, int>(group_index.GroupListMapping), Parts = new List <IndexPart>() { new IndexPart() { GroupValues = new List <int>(count) } }, IndexType = group_index.IndexType }; tmp.Parts[0].GroupValues = EfficientContainsOneCall(ids_index, group_index, fres.ResIds, true, false).Item1; group_index = tmp; } var prop_indexes = new Dictionary <string, IndexGeneric>(); foreach (var agr in aggregates) { if (!string.IsNullOrEmpty(agr.Item2)) { var prop = agr.Item2; if (!prop_indexes.ContainsKey(prop)) { skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers[prop]); snapid = leveld_db.Get(skey, null, ro); if (snapid == null || !existing_indexes.ContainsKey(table_info.Name) || !existing_indexes[table_info.Name].Contains(prop)) { throw new LinqDbException("Linqdb: group index on property " + prop + " does not exist."); } snapshot_id = Encoding.UTF8.GetString(snapid); var tmpk = table_info.Name + "|" + prop + "|" + snapshot_id; if (!indexes.ContainsKey(tmpk)) { throw new LinqDbException("Linqdb: group index on property " + prop + " does not exist."); } prop_indexes[prop] = indexes[tmpk]; if (!fres.All) { var tmp = new IndexGeneric() { ColumnName = prop_indexes[prop].ColumnName, ColumnType = prop_indexes[prop].ColumnType, TypeName = prop_indexes[prop].TypeName, Parts = new List <IndexPart>() { new IndexPart() { IntValues = new List <int>(fres.ResIds.Count()), DoubleValues = new List <double>(fres.ResIds.Count()) } }, }; switch (tmp.ColumnType) { case LinqDbTypes.int_: tmp.Parts[0].IntValues = EfficientContainsOneCall(ids_index, prop_indexes[prop], fres.ResIds, false, false).Item1; break; case LinqDbTypes.double_: case LinqDbTypes.DateTime_: tmp.Parts[0].DoubleValues = EfficientContainsOneCall(ids_index, prop_indexes[prop], fres.ResIds, false, true).Item2; break; default: break; } prop_indexes[prop] = tmp; } } } } int max = group_index.GroupListMapping.Max(f => f.Value) + 1; var group_check_list = new List <bool>(max); for (int i = 0; i < max; i++) { group_check_list.Add(false); } for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var gr = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { group_check_list[gr[j]] = true; } } for (int i = 0; i < group_check_list.Count(); i++) { if (group_check_list[i]) { distinct_groups.Add(i); } } foreach (var group in distinct_groups) { cdata[group] = new List <object>(); } foreach (var agr in aggregates) { string prop = ""; if (!string.IsNullOrEmpty(agr.Item2)) { prop = agr.Item2; } if (agr.Item1.StartsWith("Key")) { foreach (var gr in cdata) { gr.Value.Add(group_index.GroupListMapping.Where(f => f.Value == gr.Key).First().Key); } } else if (agr.Item1.StartsWith("CountDistinct")) { int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1; int bloom_size = 100000; switch (prop_indexes[prop].ColumnType) { case LinqDbTypes.int_: var bloom_int = new List <List <bool> >(max_val); var last_ints = new List <List <int> >(max_val); for (int i = 0; i < max_val; i++) { var last_bloom = new List <bool>(bloom_size); bloom_int.Add(last_bloom); var last_int = new List <int>(bloom_size); last_ints.Add(last_int); for (int j = 0; j < bloom_size; j++) { last_bloom.Add(false); last_int.Add(0); } } var list_int = new List <HashSet <int> >(max_val); for (int i = 0; i < max_val; i++) { list_int.Add(new HashSet <int>()); } var int_values = prop_indexes[prop]; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var iv = int_values.Parts[i].IntValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int val = iv[j]; var bl = bloom_int[gv[j]]; var li = last_ints[gv[j]]; var hash = val % bloom_size; if (!bl[hash] || li[hash] != val) { list_int[gv[j]].Add(val); bl[hash] = true; li[hash] = val; } } } foreach (var gr in cdata) { gr.Value.Add(list_int[gr.Key].Count()); } break; case LinqDbTypes.double_: var bloom_double = new List <List <bool> >(max_val); var last_doubles = new List <List <double> >(max_val); for (int i = 0; i < max_val; i++) { var last_bloom = new List <bool>(bloom_size); bloom_double.Add(last_bloom); var last_double = new List <double>(bloom_size); last_doubles.Add(last_double); for (int j = 0; j < bloom_size; j++) { last_bloom.Add(false); last_double.Add(0); } } var list_double = new List <HashSet <double> >(max_val); for (int i = 0; i < max_val; i++) { list_double.Add(new HashSet <double>()); } var double_values = prop_indexes[prop]; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var dv = double_values.Parts[i].DoubleValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { double val = dv[j]; var bl = bloom_double[gv[j]]; var li = last_doubles[gv[j]]; var hash = Math.Abs(val.GetHashCode()) % bloom_size; if (!bl[hash] || li[hash] != val) { list_double[gv[j]].Add(val); bl[hash] = true; li[hash] = val; } } } foreach (var gr in cdata) { gr.Value.Add(list_double[gr.Key].Count()); } break; default: break; } } else if (agr.Item1.StartsWith("Count")) { int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1; var list = new List <int>(max_val); for (int i = 0; i < max_val; i++) { list.Add(0); } for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { list[gv[j]]++; } } foreach (var gr in cdata) { gr.Value.Add(list[(int)gr.Key]); } } else if (agr.Item1.StartsWith("Sum")) { int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1; var list_int = new List <int>(max_val); for (int i = 0; i < max_val; i++) { list_int.Add(0); } var list_double = new List <double>(max_val); for (int i = 0; i < max_val; i++) { list_double.Add(0); } switch (prop_indexes[prop].ColumnType) { case LinqDbTypes.int_: var int_values = prop_indexes[prop]; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var iv = int_values.Parts[i].IntValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { list_int[gv[j]] += iv[j]; } } foreach (var gr in cdata) { gr.Value.Add(list_int[gr.Key]); } break; case LinqDbTypes.double_: var double_values = prop_indexes[prop]; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var dv = double_values.Parts[i].DoubleValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { list_double[gv[j]] += dv[j]; } } foreach (var gr in cdata) { gr.Value.Add(list_double[gr.Key]); } break; default: break; } } else if (agr.Item1.StartsWith("Max")) { int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1; var list_int = new List <int?>(max_val); for (int i = 0; i < max_val; i++) { list_int.Add(null); } var list_double = new List <double?>(max_val); for (int i = 0; i < max_val; i++) { list_double.Add(null); } switch (prop_indexes[prop].ColumnType) { case LinqDbTypes.int_: var int_values = prop_indexes[prop]; int?igval = null; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var iv = int_values.Parts[i].IntValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int g = gv[j]; int ij = iv[j]; igval = list_int[g]; if (igval == null || igval < ij) { list_int[g] = ij; } } } foreach (var gr in cdata) { gr.Value.Add(list_int[gr.Key]); } break; case LinqDbTypes.double_: var double_values = prop_indexes[prop]; double?gval = null; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var dv = double_values.Parts[i].DoubleValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int g = gv[j]; double d = dv[j]; gval = list_double[g]; if (gval == null || gval < d) { list_double[g] = d; } } } foreach (var gr in cdata) { gr.Value.Add(list_double[gr.Key]); } break; default: break; } } else if (agr.Item1.StartsWith("Min")) { int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1; var list_int = new List <int?>(max_val); for (int i = 0; i < max_val; i++) { list_int.Add(null); } var list_double = new List <double?>(max_val); for (int i = 0; i < max_val; i++) { list_double.Add(null); } switch (prop_indexes[prop].ColumnType) { case LinqDbTypes.int_: var int_values = prop_indexes[prop]; int?igval = null; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var iv = int_values.Parts[i].IntValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int g = gv[j]; igval = list_int[g]; if (igval == null || igval > iv[j]) { list_int[g] = iv[j]; } } } foreach (var gr in cdata) { gr.Value.Add(list_int[gr.Key]); } break; case LinqDbTypes.double_: var double_values = prop_indexes[prop]; double?gval = null; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var dv = double_values.Parts[i].DoubleValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int g = gv[j]; gval = list_double[g]; if (gval == null || gval > dv[j]) { list_double[g] = dv[j]; } } } foreach (var gr in cdata) { gr.Value.Add(list_double[gr.Key]); } break; default: break; } } else if (agr.Item1.StartsWith("Average")) { int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1; var total_count = new List <int>(max_val); for (int i = 0; i < max_val; i++) { total_count.Add(0); } var list_int = new List <int>(max_val); for (int i = 0; i < max_val; i++) { list_int.Add(0); } var list_double = new List <double>(max_val); for (int i = 0; i < max_val; i++) { list_double.Add(0); } switch (prop_indexes[prop].ColumnType) { case LinqDbTypes.int_: var int_values = prop_indexes[prop]; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var iv = int_values.Parts[i].IntValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int g = gv[j]; list_int[g] += iv[j]; total_count[g]++; } } foreach (var gr in cdata) { if (total_count[gr.Key] != 0) { gr.Value.Add(list_int[gr.Key] / (double)total_count[gr.Key]); } else { gr.Value.Add(list_int[gr.Key]); } } break; case LinqDbTypes.double_: var double_values = prop_indexes[prop]; for (int i = 0; i < group_index.Parts.Count(); i++) { int icount = group_index.Parts[i].GroupValues.Count(); var dv = double_values.Parts[i].DoubleValues; var gv = group_index.Parts[i].GroupValues; for (int j = 0; j < icount; j++) { int g = gv[j]; list_double[g] += dv[j]; total_count[g]++; } } foreach (var gr in cdata) { if (total_count[gr.Key] != 0) { gr.Value.Add(list_double[gr.Key] / (double)total_count[gr.Key]); } else { gr.Value.Add(list_double[gr.Key]); } } break; default: break; } } else { throw new LinqDbException("Linqdb: group aggregation function " + agr.Item1 + " is not supported."); } } return(cdata); }