public void UpdateBatch(UpdateInfo info, Dictionary <int, object> values, TableInfo table_info, WriteBatchWithConstraints batch, Dictionary <string, KeyValuePair <byte[], HashSet <int> > > string_cache, Dictionary <string, Tuple <IndexNewData, IndexDeletedData, IndexChangedData> > memory_index_meta)
{
foreach (var item in values)
{
var key = MakeIndexKey(new IndexKeyInfo()
{
TableNumber = info.TableNumber,
ColumnNumber = table_info.ColumnNumbers["Id"],
Val = BitConverter.GetBytes(item.Key).MyReverseNoCopy(),
Id = item.Key
});
var index_val = leveld_db.Get(key);
if (index_val == null)
{
continue;
}
object value = item.Value;
if (info.ColumnType == LinqDbTypes.string_)
{
SaveStringData(batch, (string)item.Value, info.ColumnName, info.TableInfo, item.Key, string_cache, false);
}
else if (info.ColumnType == LinqDbTypes.binary_)
{
SaveBinaryColumn(batch, value, info.ColumnName, info.TableInfo, item.Key, false);
}
else
{
IndexDeletedData index_deleted = null;
IndexNewData index_new = null;
IndexChangedData index_changed = null;
if (memory_index_meta.ContainsKey(info.ColumnName))
{
index_deleted = memory_index_meta[info.ColumnName].Item2;
index_new = memory_index_meta[info.ColumnName].Item1;
index_changed = memory_index_meta[info.ColumnName].Item3;
}
SaveDataColumn(batch, value, info.ColumnName, info.ColumnType, info.TableInfo, item.Key, false, index_new, index_changed);
}
}
//WriteStringCacheToBatch(batch, string_cache, table_info, null);
}
public void MakeNewPropSnapshot(string type, string prop, string snapshot_id, IndexNewData index_new, IndexDeletedData index_deleted, IndexChangedData index_changed)
{
var latest_key = type + "|" + prop;
var latest_snapshot_id = latest_snapshots[latest_key];
var key = type + "|" + prop + "|" + latest_snapshot_id;
var index = indexes[key];
var ids_index = indexes[type + "|Id|" + latest_snapshots[type + "|Id"]];
var parts = new List <IndexPart>();
var new_index = new IndexGeneric()
{
ColumnName = index.ColumnName,
ColumnType = index.ColumnType,
GroupListMapping = index.GroupListMapping,
IndexType = index.IndexType,
Parts = parts,
TypeName = index.TypeName
};
bool has_changed_int = index_changed != null && index_changed.IntValues != null && index_changed.IntValues.Any();
bool has_changed_double = index_changed != null && index_changed.DoubleValues != null && index_changed.DoubleValues.Any();
bool has_deleted = index_deleted != null && index_deleted.Ids.Any();
//var dbloomsize = 100000;
//var deleted_bloom = new List<bool>(dbloomsize);
//for (int i = 0; i < dbloomsize; i++)
//{
// deleted_bloom.Add(false);
//}
//if (has_deleted)
//{
// foreach (var did in index_deleted.Ids)
// {
// deleted_bloom[did % dbloomsize] = true;
// }
//}
//var changed_int_bloom = new List<bool>(dbloomsize);
//for (int i = 0; i < dbloomsize; i++)
//{
// changed_int_bloom.Add(false);
//}
//if (has_changed_int)
//{
// foreach (var cid in index_changed.IntValues)
// {
// changed_int_bloom[cid.Key % dbloomsize] = true;
// }
//}
//var changed_double_bloom = new List<bool>(dbloomsize);
//for (int i = 0; i < dbloomsize; i++)
//{
// changed_double_bloom.Add(false);
//}
//if (has_changed_double)
//{
// foreach (var cid in index_changed.DoubleValues)
// {
// changed_double_bloom[cid.Key % dbloomsize] = true;
// }
//}
switch (index.ColumnType)
{
case LinqDbTypes.int_:
if (index.IndexType == IndexType.PropertyOnly) //property index only
{
for (int i = 0; i < index.Parts.Count; i++)
{
var p = index.Parts[i];
p.Ids = ids_index.Parts[i].IntValues;
IndexPart np = null;
HashSet <int> removed = null;
int icount = p.IntValues.Count;
for (int j = 0; j < icount; j++)
{
var cid = p.Ids[j];
if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid))
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
IntValues = new List <int>(p.IntValues)
};
}
np.IntValues[j] = index_changed.IntValues[cid];
}
//if (j >= p.Ids.Count())
//{
// var a = 5;
//}
if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
{
if (removed == null)
{
removed = new HashSet <int>();
}
removed.Add((int)cid);
}
}
if (removed != null)
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
IntValues = new List <int>(p.IntValues)
};
}
var npp = new IndexPart()
{
Ids = new List <int>(),
IntValues = new List <int>()
};
for (int z = 0; z < np.Ids.Count; z++)
{
if (!removed.Contains((int)np.Ids[z]))
{
npp.Ids.Add(np.Ids[z]);
npp.IntValues.Add(np.IntValues[z]);
}
}
np = npp;
}
if (np != null)
{
p = np; //wow, finally!
}
parts.Add(p);
}
if (index_new != null)
{
IndexPart lastp = null;
if (parts.Any())
{
lastp = parts.LastOrDefault();
}
if (lastp == null)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
IntValues = new List <int>()
};
parts.Add(lastp);
}
else
{
lastp = new IndexPart()
{
Ids = new List <int>(lastp.Ids),
IntValues = new List <int>(lastp.IntValues)
};
parts[parts.Count() - 1] = lastp;
}
for (int i = 0; i < index_new.IntValues.Count; i++)
{
if (lastp.Ids.Count == 1000)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
IntValues = new List <int>()
};
parts.Add(lastp);
}
lastp.IntValues.Add(index_new.IntValues[i]);
lastp.Ids.Add(index_new.Ids[i]);
}
}
for (int i = 0; i < index.Parts.Count(); i++)
{
index.Parts[i].Ids = null;
}
}
else if (index.IndexType == IndexType.GroupOnly) //group index only
{
int map = 0;
if (index.GroupListMapping.Any())
{
map = index.GroupListMapping.Max(f => f.Value);
map++;
}
for (int i = 0; i < index.Parts.Count; i++)
{
var p = index.Parts[i];
p.Ids = ids_index.Parts[i].IntValues;
IndexPart np = null;
HashSet <int> removed = null;
int icount = p.GroupValues.Count;
for (int j = 0; j < icount; j++)
{
int cid = p.Ids[j];
if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid))
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
GroupValues = new List <int>(p.GroupValues)
};
}
if (!index.GroupListMapping.ContainsKey(index_changed.IntValues[cid]))
{
index.GroupListMapping[index_changed.IntValues[cid]] = map;
map++;
}
np.GroupValues[j] = index.GroupListMapping[index_changed.IntValues[cid]];
}
if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
{
if (removed == null)
{
removed = new HashSet <int>();
}
removed.Add(cid);
}
}
if (removed != null)
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
GroupValues = new List <int>(p.GroupValues)
};
}
var npp = new IndexPart()
{
Ids = new List <int>(),
GroupValues = new List <int>()
};
for (int z = 0; z < np.Ids.Count; z++)
{
if (!removed.Contains((int)np.Ids[z]))
{
npp.Ids.Add(np.Ids[z]);
npp.GroupValues.Add(np.GroupValues[z]);
}
}
np = npp;
}
if (np != null)
{
p = np; //wow, finally!
}
parts.Add(p);
}
if (index_new != null)
{
IndexPart lastp = null;
if (parts.Any())
{
lastp = parts.LastOrDefault();
}
if (lastp == null)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
GroupValues = new List <int>()
};
parts.Add(lastp);
}
else
{
lastp = new IndexPart()
{
Ids = new List <int>(lastp.Ids),
GroupValues = new List <int>(lastp.GroupValues)
};
parts[parts.Count() - 1] = lastp;
}
for (int i = 0; i < index_new.IntValues.Count; i++)
{
if (lastp.Ids.Count == 1000)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
GroupValues = new List <int>()
};
parts.Add(lastp);
}
if (!index.GroupListMapping.ContainsKey((int)index_new.IntValues[i]))
{
index.GroupListMapping[(int)index_new.IntValues[i]] = map;
map++;
}
lastp.GroupValues.Add(index.GroupListMapping[(int)index_new.IntValues[i]]);
lastp.Ids.Add(index_new.Ids[i]);
}
}
for (int i = 0; i < index.Parts.Count(); i++)
{
index.Parts[i].Ids = null;
}
}
else //both
{
int map = 0;
if (index.GroupListMapping.Any())
{
map = index.GroupListMapping.Max(f => f.Value);
map++;
}
for (int i = 0; i < index.Parts.Count; i++)
{
var p = index.Parts[i];
p.Ids = ids_index.Parts[i].IntValues;
IndexPart np = null;
HashSet <int> removed = null;
int icount = p.GroupValues.Count;
for (int j = 0; j < icount; j++)
{
int cid = p.Ids[j];
if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid))
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
GroupValues = new List <int>(p.GroupValues),
IntValues = new List <int>(p.IntValues)
};
}
if (!index.GroupListMapping.ContainsKey(index_changed.IntValues[cid]))
{
index.GroupListMapping[index_changed.IntValues[cid]] = map;
map++;
}
np.GroupValues[j] = index.GroupListMapping[index_changed.IntValues[cid]];
np.IntValues[j] = index_changed.IntValues[cid];
}
if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
{
if (removed == null)
{
removed = new HashSet <int>();
}
removed.Add(cid);
}
}
if (removed != null)
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
GroupValues = new List <int>(p.GroupValues),
IntValues = new List <int>(p.IntValues)
};
}
var npp = new IndexPart()
{
Ids = new List <int>(),
GroupValues = new List <int>(),
IntValues = new List <int>()
};
for (int z = 0; z < np.Ids.Count; z++)
{
if (!removed.Contains((int)np.Ids[z]))
{
npp.Ids.Add(np.Ids[z]);
npp.GroupValues.Add(np.GroupValues[z]);
npp.IntValues.Add(np.IntValues[z]);
}
}
np = npp;
}
if (np != null)
{
p = np; //wow, finally!
}
parts.Add(p);
}
if (index_new != null)
{
IndexPart lastp = null;
if (parts.Any())
{
lastp = parts.LastOrDefault();
}
if (lastp == null)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
GroupValues = new List <int>(),
IntValues = new List <int>()
};
parts.Add(lastp);
}
else
{
lastp = new IndexPart()
{
Ids = new List <int>(lastp.Ids),
GroupValues = new List <int>(lastp.GroupValues),
IntValues = new List <int>(lastp.IntValues)
};
parts[parts.Count() - 1] = lastp;
}
for (int i = 0; i < index_new.IntValues.Count; i++)
{
if (lastp.Ids.Count == 1000)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
GroupValues = new List <int>(),
IntValues = new List <int>()
};
parts.Add(lastp);
}
if (!index.GroupListMapping.ContainsKey((int)index_new.IntValues[i]))
{
index.GroupListMapping[(int)index_new.IntValues[i]] = map;
map++;
}
lastp.GroupValues.Add(index.GroupListMapping[(int)index_new.IntValues[i]]);
lastp.IntValues.Add(index_new.IntValues[i]);
lastp.Ids.Add(index_new.Ids[i]);
}
}
for (int i = 0; i < index.Parts.Count(); i++)
{
index.Parts[i].Ids = null;
}
}
break;
case LinqDbTypes.double_:
case LinqDbTypes.DateTime_:
for (int i = 0; i < index.Parts.Count; i++)
{
var p = index.Parts[i];
p.Ids = ids_index.Parts[i].IntValues;
IndexPart np = null;
HashSet <int> removed = null;
int icount = p.DoubleValues.Count;
for (int j = 0; j < icount; j++)
{
int cid = p.Ids[j];
if (has_changed_double /*&& changed_double_bloom[cid % dbloomsize]*/ && index_changed.DoubleValues.ContainsKey(cid))
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
DoubleValues = new List <double>(p.DoubleValues)
};
}
np.DoubleValues[j] = index_changed.DoubleValues[cid];
}
if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
{
if (removed == null)
{
removed = new HashSet <int>();
}
removed.Add(cid);
}
}
if (removed != null)
{
if (np == null)
{
np = new IndexPart()
{
Ids = new List <int>(p.Ids),
DoubleValues = new List <double>(p.DoubleValues)
};
}
var npp = new IndexPart()
{
Ids = new List <int>(),
DoubleValues = new List <double>()
};
for (int z = 0; z < np.Ids.Count; z++)
{
if (!removed.Contains((int)np.Ids[z]))
{
npp.Ids.Add(np.Ids[z]);
npp.DoubleValues.Add(np.DoubleValues[z]);
}
}
np = npp;
}
if (np != null)
{
p = np; //wow, finally!
}
parts.Add(p);
}
if (index_new != null)
{
IndexPart lastp = null;
if (parts.Any())
{
lastp = parts.LastOrDefault();
}
if (lastp == null)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
DoubleValues = new List <double>()
};
parts.Add(lastp);
}
else
{
lastp = new IndexPart()
{
Ids = new List <int>(lastp.Ids),
DoubleValues = new List <double>(lastp.DoubleValues)
};
parts[parts.Count() - 1] = lastp;
}
for (int i = 0; i < index_new.DoubleValues.Count; i++)
{
if (lastp.Ids.Count == 1000)
{
lastp = new IndexPart()
{
Ids = new List <int>(),
DoubleValues = new List <double>()
};
parts.Add(lastp);
}
lastp.DoubleValues.Add(index_new.DoubleValues[i]);
lastp.Ids.Add(index_new.Ids[i]);
}
}
for (int i = 0; i < index.Parts.Count(); i++)
{
index.Parts[i].Ids = null;
}
break;
default:
break;
}
//free old snapshots
if ((DateTime.Now - last_cleanup[type + "|" + prop]).TotalMilliseconds > 30000)
{
var alive = snapshots_alive[type + "|" + prop];
var new_alive = new List <Tuple <bool, string> >(); //bool - schedueled for deletion
foreach (var sn in alive)
{
if (!sn.Item1)
{
var nsn = new Tuple <bool, string>(true, sn.Item2);
new_alive.Add(nsn);
}
else
{
IndexGeneric val = null;
indexes.TryRemove(sn.Item2, out val);
}
}
new_alive.Add(new Tuple <bool, string>(false, type + "|" + prop + "|" + snapshot_id));
snapshots_alive[type + "|" + prop] = new_alive;
last_cleanup[type + "|" + prop] = DateTime.Now;
}
else
{
snapshots_alive[type + "|" + prop].Add(new Tuple <bool, string>(false, type + "|" + prop + "|" + snapshot_id));
}
key = type + "|" + prop + "|" + snapshot_id;
indexes[key] = new_index;
latest_snapshots[latest_key] = snapshot_id;
}