OrderedTmpResult OrderValuesByDescending(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool all, ReadOptions ro)
{
OrderedTmpResult res = new OrderedTmpResult();
res.Data = new Dictionary <int, int>();
res.Take = take;
res.Skip = skip;
var key = MakeIndexSearchKey(new IndexKeyInfo()
{
TableNumber = order_info.TableNumber,
ColumnNumber = order_info.ColumnNumber,
ColumnType = order_info.ColumnType,
Val = PostEverything
});
int count = 0;
using (var it = leveld_db.NewIterator(null, ro))
{
it.Seek(key);
if (!it.Valid())
{
return(res);
}
var k = it.Key();
var info = GetIndexKey(k);
if (info.NotKey || info.TableNumber != order_info.TableNumber || info.ColumnNumber != order_info.ColumnNumber)
{
it.Prev();
if (!it.Valid())
{
return(res);
}
k = it.Key();
if (k == null)
{
return(res);
}
info = GetIndexKey(k);
}
var contains = new ContainsInfo()
{
ids = res_set.ResIds
};
bool has = false;
if (!info.NotKey && info.TableNumber == order_info.TableNumber && info.ColumnNumber == order_info.ColumnNumber)
{
if (!all)
{
has = EfficientContains(contains, info.Id);
if (contains.is_sorted)
{
res_set.ResIds = contains.ids;
}
}
if (all || has)
{
if (res.Skip == null || res.Skip == 0)
{
res.Data[info.Id] = count;
count++;
if (res.Take != null && count == res.Take)
{
return(res);
}
}
else
{
res.Skip--;
}
}
}
else
{
return(res);
}
while (true)
{
it.Prev();
if (!it.Valid())
{
return(res);
}
k = it.Key();
if (k == null)
{
return(res);
}
info = GetIndexKey(k);
if (!info.NotKey && info.TableNumber == order_info.TableNumber && info.ColumnNumber == order_info.ColumnNumber)
{
if (!all)
{
has = EfficientContains(contains, info.Id);
}
if (all || has)
{
if (res.Skip == null || res.Skip == 0)
{
res.Data[info.Id] = count;
count++;
if (res.Take != null && count == res.Take)
{
return(res);
}
}
else
{
res.Skip--;
}
}
}
else
{
return(res);
}
}
}
}
Dictionary <int, int> OrderValuesByDescendingWithNegatives(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool all, ReadOptions ro)
{
var pos_res = OrderValuesByDescending(order_info, res_set, take, skip, all, ro);
int?take_left = pos_res.Take == null ? null : (pos_res.Take - pos_res.Data.Count);
if (take_left == null || take_left > 0)
{
order_info.ColumnNumber *= -1;
var neg_res = OrderValues(order_info, res_set, take_left, pos_res.Skip, all, ro);
order_info.ColumnNumber *= -1;
var pos_count = pos_res.Data.Count();
foreach (var pv in neg_res.Data)
{
pos_res.Data[pv.Key] = pv.Value + pos_count;
}
}
return(pos_res.Data);
}
Dictionary <int, int> OrderSomeValuesById(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool desc, Dictionary <long, byte[]> cache, ReadOptions ro, TableInfo table_info)
{
Dictionary <int, int> res = new Dictionary <int, int>();
var ids = res_set.ResIds;
if (!IsSorted(ids))
{
ids = ids.OrderBy(f => f).ToList();
}
if (!desc)
{
if (skip == null)
{
skip = 0;
}
int count = 0;
for (int i = 0; i < ids.Count(); i++)
{
if (skip > 0)
{
skip--;
continue;
}
if (take == null || take > 0)
{
if (take > 0)
{
take--;
}
res[ids[i]] = count;
count++;
}
else if (take == 0)
{
break;
}
}
}
else
{
if (skip == null)
{
skip = 0;
}
int count = 0;
for (int i = ids.Count() - 1; i >= 0; i--)
{
if (skip > 0)
{
skip--;
continue;
}
if (take == null || take > 0)
{
if (take > 0)
{
take--;
}
res[ids[i]] = count;
count++;
}
else if (take == 0)
{
break;
}
}
}
return(res);
}
Dictionary <int, int> OrderSomeValues(OrderByInfo order_info, OperResult res_set, int?take, int?skip, bool desc, Dictionary <long, byte[]> cache, ReadOptions ro, TableInfo table_info)
{
if (order_info.ColumnName != "Id")
{
var resindex = OrderSomeValuesWithIndex(order_info, res_set, take, skip, desc, cache, ro, table_info);
if (resindex != null)
{
return(resindex);
}
}
else
{
return(OrderSomeValuesById(order_info, res_set, take, skip, desc, cache, ro, table_info));
}
Dictionary <int, int> res = new Dictionary <int, int>();
List <KeyValuePair <object, int> > tmp_res = new List <KeyValuePair <object, int> >(res_set.ResIds.Count());
foreach (var r in res_set.ResIds)
{
if (order_info.ColumnName == "Id")
{
tmp_res.Add(new KeyValuePair <object, int>(r, r));
}
else
{
byte[] val = null;
//if (!GetFromCache(order_info.ColumnNumber, r, cache, out val))
//{
if (order_info.ColumnType == LinqDbTypes.string_)
{
var key = MakeStringValueKey(new IndexKeyInfo()
{
TableNumber = order_info.TableNumber,
ColumnNumber = order_info.ColumnNumber,
Id = r
});
val = leveld_db.Get(key, null, ro);
}
else
{
var key = MakeValueKey(new IndexKeyInfo()
{
TableNumber = order_info.TableNumber,
ColumnNumber = order_info.ColumnNumber,
Id = r
});
val = leveld_db.Get(key, null, ro);
}
//}
if (order_info.ColumnType != LinqDbTypes.string_ && ValsEqual(val, NullConstant) ||
order_info.ColumnType == LinqDbTypes.string_ && ValsEqual(val, BinaryOrStringValuePrefix))
{
tmp_res.Add(new KeyValuePair <object, int>(null, r));
}
else
{
if (order_info.ColumnType == LinqDbTypes.int_)
{
tmp_res.Add(new KeyValuePair <object, int>(BitConverter.ToInt32(val.MyReverseWithCopy(), 0), r));
}
else if (order_info.ColumnType == LinqDbTypes.double_ || order_info.ColumnType == LinqDbTypes.DateTime_)
{
tmp_res.Add(new KeyValuePair <object, int>(BitConverter.ToDouble(val.MyReverseWithCopy(), 0), r));
}
else if (order_info.ColumnType == LinqDbTypes.string_)
{
tmp_res.Add(new KeyValuePair <object, int>(Encoding.Unicode.GetString(val), r));
}
}
}
}
List <KeyValuePair <object, int> > t = null;
if (!desc)
{
t = tmp_res.OrderBy(f => f.Key).ToList();
}
else
{
t = tmp_res.OrderByDescending(f => f.Key).ToList();
}
if (skip != null)
{
t = t.Skip((int)skip).ToList();
}
if (take != null)
{
t = t.Take((int)take).ToList();
}
int count = 0;
foreach (var v in t)
{
res[v.Value] = count;
count++;
}
return(res);
}
public Dictionary <int, List <object> > SelectGrouppedCommon(HashSet <int> distinct_groups, OperResult fres, List <Tuple <string, string> > aggregates, TableInfo table_info, short column_number, string column_name, ReadOptions ro, int count)
{
Dictionary <int, List <object> > cdata = new Dictionary <int, List <object> >();
//var max_id = 0;
//List<byte> check_list = null;
//int bloom_max = 2000000;
//List<byte> bloom_int = null;
List <int> groups = new List <int>();
var skey = MakeSnapshotKey(table_info.TableNumber, /*tree.GroupingInfo.ColumnNumber*/ column_number);
var snapid = leveld_db.Get(skey, null, ro);
if (snapid == null || !existing_indexes.ContainsKey(table_info.Name) || !existing_indexes[table_info.Name].Contains(column_name))
{
throw new LinqDbException("Linqdb: group index on property " + /*tree.GroupingInfo.ColumnName*/ column_name + " does not exist.");
}
string snapshot_id = Encoding.UTF8.GetString(snapid);
skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers["Id"]);
snapid = leveld_db.Get(skey, null, ro);
string id_snapshot_id = Encoding.UTF8.GetString(snapid);
if (!indexes.ContainsKey(table_info.Name + "|" + /*tree.GroupingInfo.ColumnName*/ column_name + "|" + snapshot_id))
{
throw new LinqDbException("Linqdb: group index on property " + /*tree.GroupingInfo.ColumnName*/ column_name + " does not exist.");
}
var group_index = indexes[table_info.Name + "|" + /*tree.GroupingInfo.ColumnName*/ column_name + "|" + snapshot_id];
if (group_index.IndexType == IndexType.PropertyOnly)
{
throw new LinqDbException("Linqdb: group index on property " + /*tree.GroupingInfo.ColumnName*/ column_name + " does not exist.");
}
var ids_index = indexes[table_info.Name + "|Id|" + id_snapshot_id];
if (!fres.All)
{
var tmp = new IndexGeneric()
{
ColumnName = group_index.ColumnName,
ColumnType = group_index.ColumnType,
TypeName = group_index.TypeName,
GroupListMapping = new ConcurrentDictionary <int, int>(group_index.GroupListMapping),
Parts = new List <IndexPart>()
{
new IndexPart()
{
GroupValues = new List <int>(count)
}
},
IndexType = group_index.IndexType
};
tmp.Parts[0].GroupValues = EfficientContainsOneCall(ids_index, group_index, fres.ResIds, true, false).Item1;
group_index = tmp;
}
var prop_indexes = new Dictionary <string, IndexGeneric>();
foreach (var agr in aggregates)
{
if (!string.IsNullOrEmpty(agr.Item2))
{
var prop = agr.Item2;
if (!prop_indexes.ContainsKey(prop))
{
skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers[prop]);
snapid = leveld_db.Get(skey, null, ro);
if (snapid == null || !existing_indexes.ContainsKey(table_info.Name) || !existing_indexes[table_info.Name].Contains(prop))
{
throw new LinqDbException("Linqdb: group index on property " + prop + " does not exist.");
}
snapshot_id = Encoding.UTF8.GetString(snapid);
var tmpk = table_info.Name + "|" + prop + "|" + snapshot_id;
if (!indexes.ContainsKey(tmpk))
{
throw new LinqDbException("Linqdb: group index on property " + prop + " does not exist.");
}
prop_indexes[prop] = indexes[tmpk];
if (!fres.All)
{
var tmp = new IndexGeneric()
{
ColumnName = prop_indexes[prop].ColumnName,
ColumnType = prop_indexes[prop].ColumnType,
TypeName = prop_indexes[prop].TypeName,
Parts = new List <IndexPart>()
{
new IndexPart()
{
IntValues = new List <int>(fres.ResIds.Count()), DoubleValues = new List <double>(fres.ResIds.Count())
}
},
};
switch (tmp.ColumnType)
{
case LinqDbTypes.int_:
tmp.Parts[0].IntValues = EfficientContainsOneCall(ids_index, prop_indexes[prop], fres.ResIds, false, false).Item1;
break;
case LinqDbTypes.double_:
case LinqDbTypes.DateTime_:
tmp.Parts[0].DoubleValues = EfficientContainsOneCall(ids_index, prop_indexes[prop], fres.ResIds, false, true).Item2;
break;
default:
break;
}
prop_indexes[prop] = tmp;
}
}
}
}
int max = group_index.GroupListMapping.Max(f => f.Value) + 1;
var group_check_list = new List <bool>(max);
for (int i = 0; i < max; i++)
{
group_check_list.Add(false);
}
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var gr = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
group_check_list[gr[j]] = true;
}
}
for (int i = 0; i < group_check_list.Count(); i++)
{
if (group_check_list[i])
{
distinct_groups.Add(i);
}
}
foreach (var group in distinct_groups)
{
cdata[group] = new List <object>();
}
foreach (var agr in aggregates)
{
string prop = "";
if (!string.IsNullOrEmpty(agr.Item2))
{
prop = agr.Item2;
}
if (agr.Item1.StartsWith("Key"))
{
foreach (var gr in cdata)
{
gr.Value.Add(group_index.GroupListMapping.Where(f => f.Value == gr.Key).First().Key);
}
}
else if (agr.Item1.StartsWith("CountDistinct"))
{
int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1;
int bloom_size = 100000;
switch (prop_indexes[prop].ColumnType)
{
case LinqDbTypes.int_:
var bloom_int = new List <List <bool> >(max_val);
var last_ints = new List <List <int> >(max_val);
for (int i = 0; i < max_val; i++)
{
var last_bloom = new List <bool>(bloom_size);
bloom_int.Add(last_bloom);
var last_int = new List <int>(bloom_size);
last_ints.Add(last_int);
for (int j = 0; j < bloom_size; j++)
{
last_bloom.Add(false);
last_int.Add(0);
}
}
var list_int = new List <HashSet <int> >(max_val);
for (int i = 0; i < max_val; i++)
{
list_int.Add(new HashSet <int>());
}
var int_values = prop_indexes[prop];
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var iv = int_values.Parts[i].IntValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int val = iv[j];
var bl = bloom_int[gv[j]];
var li = last_ints[gv[j]];
var hash = val % bloom_size;
if (!bl[hash] || li[hash] != val)
{
list_int[gv[j]].Add(val);
bl[hash] = true;
li[hash] = val;
}
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_int[gr.Key].Count());
}
break;
case LinqDbTypes.double_:
var bloom_double = new List <List <bool> >(max_val);
var last_doubles = new List <List <double> >(max_val);
for (int i = 0; i < max_val; i++)
{
var last_bloom = new List <bool>(bloom_size);
bloom_double.Add(last_bloom);
var last_double = new List <double>(bloom_size);
last_doubles.Add(last_double);
for (int j = 0; j < bloom_size; j++)
{
last_bloom.Add(false);
last_double.Add(0);
}
}
var list_double = new List <HashSet <double> >(max_val);
for (int i = 0; i < max_val; i++)
{
list_double.Add(new HashSet <double>());
}
var double_values = prop_indexes[prop];
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var dv = double_values.Parts[i].DoubleValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
double val = dv[j];
var bl = bloom_double[gv[j]];
var li = last_doubles[gv[j]];
var hash = Math.Abs(val.GetHashCode()) % bloom_size;
if (!bl[hash] || li[hash] != val)
{
list_double[gv[j]].Add(val);
bl[hash] = true;
li[hash] = val;
}
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_double[gr.Key].Count());
}
break;
default:
break;
}
}
else if (agr.Item1.StartsWith("Count"))
{
int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1;
var list = new List <int>(max_val);
for (int i = 0; i < max_val; i++)
{
list.Add(0);
}
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
list[gv[j]]++;
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list[(int)gr.Key]);
}
}
else if (agr.Item1.StartsWith("Sum"))
{
int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1;
var list_int = new List <int>(max_val);
for (int i = 0; i < max_val; i++)
{
list_int.Add(0);
}
var list_double = new List <double>(max_val);
for (int i = 0; i < max_val; i++)
{
list_double.Add(0);
}
switch (prop_indexes[prop].ColumnType)
{
case LinqDbTypes.int_:
var int_values = prop_indexes[prop];
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var iv = int_values.Parts[i].IntValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
list_int[gv[j]] += iv[j];
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_int[gr.Key]);
}
break;
case LinqDbTypes.double_:
var double_values = prop_indexes[prop];
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var dv = double_values.Parts[i].DoubleValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
list_double[gv[j]] += dv[j];
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_double[gr.Key]);
}
break;
default:
break;
}
}
else if (agr.Item1.StartsWith("Max"))
{
int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1;
var list_int = new List <int?>(max_val);
for (int i = 0; i < max_val; i++)
{
list_int.Add(null);
}
var list_double = new List <double?>(max_val);
for (int i = 0; i < max_val; i++)
{
list_double.Add(null);
}
switch (prop_indexes[prop].ColumnType)
{
case LinqDbTypes.int_:
var int_values = prop_indexes[prop];
int?igval = null;
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var iv = int_values.Parts[i].IntValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int g = gv[j];
int ij = iv[j];
igval = list_int[g];
if (igval == null || igval < ij)
{
list_int[g] = ij;
}
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_int[gr.Key]);
}
break;
case LinqDbTypes.double_:
var double_values = prop_indexes[prop];
double?gval = null;
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var dv = double_values.Parts[i].DoubleValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int g = gv[j];
double d = dv[j];
gval = list_double[g];
if (gval == null || gval < d)
{
list_double[g] = d;
}
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_double[gr.Key]);
}
break;
default:
break;
}
}
else if (agr.Item1.StartsWith("Min"))
{
int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1;
var list_int = new List <int?>(max_val);
for (int i = 0; i < max_val; i++)
{
list_int.Add(null);
}
var list_double = new List <double?>(max_val);
for (int i = 0; i < max_val; i++)
{
list_double.Add(null);
}
switch (prop_indexes[prop].ColumnType)
{
case LinqDbTypes.int_:
var int_values = prop_indexes[prop];
int?igval = null;
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var iv = int_values.Parts[i].IntValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int g = gv[j];
igval = list_int[g];
if (igval == null || igval > iv[j])
{
list_int[g] = iv[j];
}
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_int[gr.Key]);
}
break;
case LinqDbTypes.double_:
var double_values = prop_indexes[prop];
double?gval = null;
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var dv = double_values.Parts[i].DoubleValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int g = gv[j];
gval = list_double[g];
if (gval == null || gval > dv[j])
{
list_double[g] = dv[j];
}
}
}
foreach (var gr in cdata)
{
gr.Value.Add(list_double[gr.Key]);
}
break;
default:
break;
}
}
else if (agr.Item1.StartsWith("Average"))
{
int max_val = group_index.GroupListMapping.Max(f => f.Value) + 1;
var total_count = new List <int>(max_val);
for (int i = 0; i < max_val; i++)
{
total_count.Add(0);
}
var list_int = new List <int>(max_val);
for (int i = 0; i < max_val; i++)
{
list_int.Add(0);
}
var list_double = new List <double>(max_val);
for (int i = 0; i < max_val; i++)
{
list_double.Add(0);
}
switch (prop_indexes[prop].ColumnType)
{
case LinqDbTypes.int_:
var int_values = prop_indexes[prop];
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var iv = int_values.Parts[i].IntValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int g = gv[j];
list_int[g] += iv[j];
total_count[g]++;
}
}
foreach (var gr in cdata)
{
if (total_count[gr.Key] != 0)
{
gr.Value.Add(list_int[gr.Key] / (double)total_count[gr.Key]);
}
else
{
gr.Value.Add(list_int[gr.Key]);
}
}
break;
case LinqDbTypes.double_:
var double_values = prop_indexes[prop];
for (int i = 0; i < group_index.Parts.Count(); i++)
{
int icount = group_index.Parts[i].GroupValues.Count();
var dv = double_values.Parts[i].DoubleValues;
var gv = group_index.Parts[i].GroupValues;
for (int j = 0; j < icount; j++)
{
int g = gv[j];
list_double[g] += dv[j];
total_count[g]++;
}
}
foreach (var gr in cdata)
{
if (total_count[gr.Key] != 0)
{
gr.Value.Add(list_double[gr.Key] / (double)total_count[gr.Key]);
}
else
{
gr.Value.Add(list_double[gr.Key]);
}
}
break;
default:
break;
}
}
else
{
throw new LinqDbException("Linqdb: group aggregation function " + agr.Item1 + " is not supported.");
}
}
return(cdata);
}
