Ejemplo n.º 1
0
        public void CreateGroupByIndexString(string table_name, string group_name, string value_name)
        {
            var _write_lock = GetTableWriteLock(table_name);

            lock (_write_lock)
            {
                using (var snapshot = leveld_db.CreateSnapshot())
                {
                    var table_info = GetTableInfo(table_name);
                    if (table_info.Columns[value_name] == LinqDbTypes.binary_ || table_info.Columns[value_name] == LinqDbTypes.string_)
                    {
                        throw new LinqDbException("Linqdb: Property type is not supported as memory index: " + value_name);
                    }
                    if (table_info.Columns[group_name] != LinqDbTypes.int_)
                    {
                        throw new LinqDbException("Linqdb: Property type is not supported as group by column: " + group_name);
                    }

                    string     snapshot_id = Ldb.GetNewSpnapshotId();
                    var        ro          = new ReadOptions().SetSnapshot(snapshot);
                    int        total       = GetTableRowCount(table_info, ro);
                    List <int> ids         = null;
                    if (!existing_indexes.ContainsKey(table_name) || !existing_indexes[table_name].Contains("Id"))
                    {
                        IndexGeneric index = new IndexGeneric()
                        {
                            ColumnName = "Id",
                            ColumnType = LinqDbTypes.int_,
                            TypeName   = table_name,
                            Parts      = new List <IndexPart>(),
                            IndexType  = IndexType.PropertyOnly
                        };
                        ids = ReadAllIds(table_info, ro, total);
                        int       counter = 0;
                        IndexPart cpart   = null;
                        foreach (var id in ids)
                        {
                            if (counter % 1000 == 0)
                            {
                                cpart = new IndexPart()
                                {
                                    IntValues = new List <int>(1000)
                                };
                                index.Parts.Add(cpart);
                            }
                            cpart.IntValues.Add(id);
                            counter++;
                        }
                        if (!existing_indexes.ContainsKey(table_name))
                        {
                            existing_indexes[table_name] = new HashSet <string>()
                            {
                                "Id"
                            };
                        }
                        else
                        {
                            existing_indexes[table_name].Add("Id");
                        }
                        indexes[table_name + "|Id|" + snapshot_id] = index;
                        latest_snapshots[table_name + "|Id"]       = snapshot_id;
                        snapshots_alive.TryAdd(table_name + "|Id", new List <Tuple <bool, string> >()
                        {
                            new Tuple <bool, string>(false, table_name + "|Id|" + snapshot_id)
                        });
                        last_cleanup.TryAdd(table_name + "|Id", DateTime.Now);

                        //
                        var skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers["Id"]);
                        leveld_db.Put(skey, Encoding.UTF8.GetBytes(snapshot_id));
                    }
                    else
                    {
                        ids = new List <int>(total);
                        var skey           = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers["Id"]);
                        var snapid         = leveld_db.Get(skey, null, ro);
                        var id_snapshot_id = Encoding.UTF8.GetString(snapid);
                        var index          = indexes[table_name + "|Id|" + id_snapshot_id];
                        for (int i = 0; i < index.Parts.Count(); i++)
                        {
                            ids.AddRange(index.Parts[i].IntValues);
                        }
                    }


                    if (!existing_indexes[table_name].Contains(group_name))
                    {
                        IndexGeneric index = new IndexGeneric()
                        {
                            ColumnName       = group_name,
                            ColumnType       = LinqDbTypes.int_,
                            TypeName         = table_name,
                            GroupListMapping = new ConcurrentDictionary <int, int>(),
                            Parts            = new List <IndexPart>(),
                            IndexType        = IndexType.GroupOnly
                        };
                        int totalex = 0;
                        if ((ids.Any() ? ids.Max() : 0) < 250000000)
                        {
                            var ivalues = ReadAllIntValuesList(index.ColumnName, table_info, ro, ids.Any() ? ids.Max() : 0, out totalex);
                            if (totalex != ids.Count())
                            {
                                throw new LinqDbException("Linqdb: column " + index.ColumnName + " has gaps in data. Prior to building an index gaps must be updated with values. (" + totalex + " != " + ids.Count() + ")");
                            }
                            int map = 0;
                            foreach (var id in ids)
                            {
                                if (!index.GroupListMapping.ContainsKey(ivalues[id]))
                                {
                                    index.GroupListMapping[ivalues[id]] = map;
                                    map++;
                                }
                            }

                            var latest_snapshot_id = latest_snapshots[table_name + "|Id"];
                            var id_index           = indexes[table_name + "|Id|" + latest_snapshot_id];
                            for (int j = 0; j < id_index.Parts.Count(); j++)
                            {
                                var part     = id_index.Parts[j];
                                var new_part = new IndexPart()
                                {
                                    GroupValues = new List <int>(part.IntValues.Count())
                                };
                                index.Parts.Add(new_part);
                                for (int k = 0; k < part.IntValues.Count(); k++)
                                {
                                    var ival = ivalues[part.IntValues[k]];
                                    var val  = index.GroupListMapping[ival];
                                    new_part.GroupValues.Add(val);
                                }
                            }
                        }
                        else
                        {
                            var ivalues = ReadAllIntValuesDic(index.ColumnName, table_info, ro, ids.Any() ? ids.Max() : 0, total, out totalex);
                            if (totalex != ids.Count())
                            {
                                throw new LinqDbException("Linqdb: column " + index.ColumnName + " has gaps in data. Prior to building an index gaps must be updated with values. (" + totalex + " != " + ids.Count() + ")");
                            }
                            int map = 0;
                            foreach (var id in ids)
                            {
                                if (!index.GroupListMapping.ContainsKey(ivalues[id]))
                                {
                                    index.GroupListMapping[ivalues[id]] = map;
                                    map++;
                                }
                            }

                            var latest_snapshot_id = latest_snapshots[table_name + "|Id"];
                            var id_index           = indexes[table_name + "|Id|" + latest_snapshot_id];
                            for (int j = 0; j < id_index.Parts.Count(); j++)
                            {
                                var part     = id_index.Parts[j];
                                var new_part = new IndexPart()
                                {
                                    GroupValues = new List <int>(part.IntValues.Count())
                                };
                                index.Parts.Add(new_part);
                                for (int k = 0; k < part.IntValues.Count(); k++)
                                {
                                    var ival = ivalues[part.IntValues[k]];
                                    var val  = index.GroupListMapping[ival];
                                    new_part.GroupValues.Add(val);
                                }
                            }
                        }

                        existing_indexes[table_name].Add(group_name);
                        indexes[table_name + "|" + group_name + "|" + snapshot_id] = index;
                        latest_snapshots[table_name + "|" + group_name]            = snapshot_id;
                        snapshots_alive.TryAdd(table_name + "|" + group_name, new List <Tuple <bool, string> >()
                        {
                            new Tuple <bool, string>(false, table_name + "|" + group_name + "|" + snapshot_id)
                        });
                        last_cleanup.TryAdd(table_name + "|" + group_name, DateTime.Now);

                        //
                        var skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers[group_name]);
                        leveld_db.Put(skey, Encoding.UTF8.GetBytes(snapshot_id));
                    }
                    else //could be property index, we need a group one
                    {
                        var latest_snapshot_id = latest_snapshots[table_name + "|" + group_name];
                        var index = indexes[table_name + "|" + group_name + "|" + latest_snapshot_id];
                        if (index.IndexType == IndexType.PropertyOnly)
                        {
                            index.GroupListMapping = new ConcurrentDictionary <int, int>();
                            int map = 0;
                            for (int i = 0; i < index.Parts.Count(); i++)
                            {
                                int icount = index.Parts[i].IntValues.Count();
                                index.Parts[i].GroupValues = new List <int>(icount);
                                var gv = index.Parts[i].GroupValues;
                                for (int j = 0; j < icount; j++)
                                {
                                    var val  = index.Parts[i].IntValues[j];
                                    var ival = (int)val;
                                    if (!index.GroupListMapping.ContainsKey(ival))
                                    {
                                        index.GroupListMapping[ival] = map;
                                        map++;
                                    }
                                    gv.Add(index.GroupListMapping[ival]);
                                }
                            }
                            index.IndexType = IndexType.Both;
                        }
                    }

                    if (!existing_indexes[table_name].Contains(value_name))
                    {
                        IndexGeneric index = new IndexGeneric()
                        {
                            ColumnName = value_name,
                            TypeName   = table_name,
                            Parts      = new List <IndexPart>(),
                            IndexType  = IndexType.PropertyOnly
                        };

                        if (ids == null)
                        {
                            ids = ReadAllValues("Id", table_info, new OperResult()
                            {
                                All = true
                            }, ro, new ReadByteCount()
                            {
                                read_size_bytes = -Int32.MaxValue
                            }, total).Item1;
                        }


                        var latest_snapshot_id = latest_snapshots[table_name + "|Id"];
                        switch (table_info.Columns[value_name])
                        {
                        case LinqDbTypes.int_:
                            index.ColumnType = LinqDbTypes.int_;
                            int totalex = 0;
                            if ((ids.Any() ? ids.Max() : 0) < 250000000)
                            {
                                var ivalues = ReadAllIntValuesList(index.ColumnName, table_info, ro, ids.Any() ? ids.Max() : 0, out totalex);
                                if (totalex != ids.Count())
                                {
                                    throw new LinqDbException("Linqdb: column " + index.ColumnName + " has gaps in data. Prior to building an index gaps must be updated with values. (" + totalex + " != " + ids.Count() + ")");
                                }

                                var id_index = indexes[table_name + "|Id|" + latest_snapshot_id];
                                for (int j = 0; j < id_index.Parts.Count(); j++)
                                {
                                    var part     = id_index.Parts[j];
                                    var new_part = new IndexPart()
                                    {
                                        IntValues = new List <int>(part.IntValues.Count())
                                    };
                                    index.Parts.Add(new_part);
                                    for (int k = 0; k < part.IntValues.Count(); k++)
                                    {
                                        var val = ivalues[(int)part.IntValues[k]];
                                        new_part.IntValues.Add(val);
                                    }
                                }
                            }
                            else
                            {
                                var ivalues = ReadAllIntValuesDic(index.ColumnName, table_info, ro, ids.Any() ? ids.Max() : 0, ids.Count(), out totalex);
                                if (totalex != ids.Count())
                                {
                                    throw new LinqDbException("Linqdb: column " + index.ColumnName + " has gaps in data. Prior to building an index gaps must be updated with values. (" + totalex + " != " + ids.Count() + ")");
                                }

                                var id_index = indexes[table_name + "|Id|" + latest_snapshot_id];
                                for (int j = 0; j < id_index.Parts.Count(); j++)
                                {
                                    var part     = id_index.Parts[j];
                                    var new_part = new IndexPart()
                                    {
                                        IntValues = new List <int>(part.IntValues.Count())
                                    };
                                    index.Parts.Add(new_part);
                                    for (int k = 0; k < part.IntValues.Count(); k++)
                                    {
                                        var val = ivalues[(int)part.IntValues[k]];
                                        new_part.IntValues.Add(val);
                                    }
                                }
                            }
                            break;

                        case LinqDbTypes.double_:
                        case LinqDbTypes.DateTime_:
                            index.ColumnType = LinqDbTypes.double_;
                            int totalexd = 0;
                            if ((ids.Any() ? ids.Max() : 0) < 250000000)
                            {
                                var dvalues = ReadAllDoubleValuesList(index.ColumnName, table_info, ro, ids.Any() ? ids.Max() : 0, out totalexd);
                                if (totalexd != ids.Count())
                                {
                                    throw new LinqDbException("Linqdb: column " + index.ColumnName + " has gaps in data. Prior to building an index gaps must be updated with values. (" + totalexd + " != " + ids.Count() + ")");
                                }

                                var id_index = indexes[table_name + "|Id|" + latest_snapshot_id];
                                for (int j = 0; j < id_index.Parts.Count(); j++)
                                {
                                    var part     = id_index.Parts[j];
                                    var new_part = new IndexPart()
                                    {
                                        DoubleValues = new List <double>(part.IntValues.Count())
                                    };
                                    index.Parts.Add(new_part);
                                    for (int k = 0; k < part.IntValues.Count(); k++)
                                    {
                                        var val = dvalues[(int)part.IntValues[k]];
                                        new_part.DoubleValues.Add(val);
                                    }
                                }
                            }
                            else
                            {
                                var dvalues = ReadAllDoubleValuesDic(index.ColumnName, table_info, ro, ids.Any() ? ids.Max() : 0, ids.Count(), out totalexd);
                                if (totalexd != ids.Count())
                                {
                                    throw new LinqDbException("Linqdb: column " + index.ColumnName + " has gaps in data. Prior to building an index gaps must be updated with values. (" + totalexd + " != " + ids.Count() + ")");
                                }

                                var id_index = indexes[table_name + "|Id|" + latest_snapshot_id];
                                for (int j = 0; j < id_index.Parts.Count(); j++)
                                {
                                    var part     = id_index.Parts[j];
                                    var new_part = new IndexPart()
                                    {
                                        DoubleValues = new List <double>(part.IntValues.Count())
                                    };
                                    index.Parts.Add(new_part);
                                    for (int k = 0; k < part.IntValues.Count(); k++)
                                    {
                                        var val = dvalues[(int)part.IntValues[k]];
                                        new_part.DoubleValues.Add(val);
                                    }
                                }
                            }
                            break;
                        }

                        existing_indexes[table_name].Add(value_name);
                        indexes[table_name + "|" + value_name + "|" + snapshot_id] = index;
                        latest_snapshots[table_name + "|" + value_name]            = snapshot_id;
                        snapshots_alive.TryAdd(table_name + "|" + value_name, new List <Tuple <bool, string> >()
                        {
                            new Tuple <bool, string>(false, table_name + "|" + value_name + "|" + snapshot_id)
                        });
                        last_cleanup.TryAdd(table_name + "|" + value_name, DateTime.Now);

                        //
                        var skey = MakeSnapshotKey(table_info.TableNumber, table_info.ColumnNumbers[value_name]);
                        leveld_db.Put(skey, Encoding.UTF8.GetBytes(snapshot_id));
                    }
                }
            }
            GC.Collect();
            GC.WaitForPendingFinalizers();
        }
Ejemplo n.º 2
0
        public void MakeNewPropSnapshot(string type, string prop, string snapshot_id, IndexNewData index_new, IndexDeletedData index_deleted, IndexChangedData index_changed)
        {
            var latest_key         = type + "|" + prop;
            var latest_snapshot_id = latest_snapshots[latest_key];
            var key       = type + "|" + prop + "|" + latest_snapshot_id;
            var index     = indexes[key];
            var ids_index = indexes[type + "|Id|" + latest_snapshots[type + "|Id"]];
            var parts     = new List <IndexPart>();
            var new_index = new IndexGeneric()
            {
                ColumnName       = index.ColumnName,
                ColumnType       = index.ColumnType,
                GroupListMapping = index.GroupListMapping,
                IndexType        = index.IndexType,
                Parts            = parts,
                TypeName         = index.TypeName
            };
            bool has_changed_int    = index_changed != null && index_changed.IntValues != null && index_changed.IntValues.Any();
            bool has_changed_double = index_changed != null && index_changed.DoubleValues != null && index_changed.DoubleValues.Any();
            bool has_deleted        = index_deleted != null && index_deleted.Ids.Any();

            //var dbloomsize = 100000;
            //var deleted_bloom = new List<bool>(dbloomsize);
            //for (int i = 0; i < dbloomsize; i++)
            //{
            //    deleted_bloom.Add(false);
            //}
            //if (has_deleted)
            //{
            //    foreach (var did in index_deleted.Ids)
            //    {
            //        deleted_bloom[did % dbloomsize] = true;
            //    }
            //}
            //var changed_int_bloom = new List<bool>(dbloomsize);
            //for (int i = 0; i < dbloomsize; i++)
            //{
            //    changed_int_bloom.Add(false);
            //}
            //if (has_changed_int)
            //{
            //    foreach (var cid in index_changed.IntValues)
            //    {
            //        changed_int_bloom[cid.Key % dbloomsize] = true;
            //    }
            //}
            //var changed_double_bloom = new List<bool>(dbloomsize);
            //for (int i = 0; i < dbloomsize; i++)
            //{
            //    changed_double_bloom.Add(false);
            //}
            //if (has_changed_double)
            //{
            //    foreach (var cid in index_changed.DoubleValues)
            //    {
            //        changed_double_bloom[cid.Key % dbloomsize] = true;
            //    }
            //}

            switch (index.ColumnType)
            {
            case LinqDbTypes.int_:
                if (index.IndexType == IndexType.PropertyOnly)     //property index only
                {
                    for (int i = 0; i < index.Parts.Count; i++)
                    {
                        var p = index.Parts[i];
                        p.Ids = ids_index.Parts[i].IntValues;
                        IndexPart     np      = null;
                        HashSet <int> removed = null;
                        int           icount  = p.IntValues.Count;
                        for (int j = 0; j < icount; j++)
                        {
                            var cid = p.Ids[j];
                            if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid))
                            {
                                if (np == null)
                                {
                                    np = new IndexPart()
                                    {
                                        Ids       = new List <int>(p.Ids),
                                        IntValues = new List <int>(p.IntValues)
                                    };
                                }
                                np.IntValues[j] = index_changed.IntValues[cid];
                            }
                            //if (j >= p.Ids.Count())
                            //{
                            //    var a = 5;
                            //}
                            if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
                            {
                                if (removed == null)
                                {
                                    removed = new HashSet <int>();
                                }
                                removed.Add((int)cid);
                            }
                        }
                        if (removed != null)
                        {
                            if (np == null)
                            {
                                np = new IndexPart()
                                {
                                    Ids       = new List <int>(p.Ids),
                                    IntValues = new List <int>(p.IntValues)
                                };
                            }
                            var npp = new IndexPart()
                            {
                                Ids       = new List <int>(),
                                IntValues = new List <int>()
                            };
                            for (int z = 0; z < np.Ids.Count; z++)
                            {
                                if (!removed.Contains((int)np.Ids[z]))
                                {
                                    npp.Ids.Add(np.Ids[z]);
                                    npp.IntValues.Add(np.IntValues[z]);
                                }
                            }
                            np = npp;
                        }
                        if (np != null)
                        {
                            p = np;     //wow, finally!
                        }
                        parts.Add(p);
                    }
                    if (index_new != null)
                    {
                        IndexPart lastp = null;
                        if (parts.Any())
                        {
                            lastp = parts.LastOrDefault();
                        }
                        if (lastp == null)
                        {
                            lastp = new IndexPart()
                            {
                                Ids       = new List <int>(),
                                IntValues = new List <int>()
                            };
                            parts.Add(lastp);
                        }
                        else
                        {
                            lastp = new IndexPart()
                            {
                                Ids       = new List <int>(lastp.Ids),
                                IntValues = new List <int>(lastp.IntValues)
                            };
                            parts[parts.Count() - 1] = lastp;
                        }
                        for (int i = 0; i < index_new.IntValues.Count; i++)
                        {
                            if (lastp.Ids.Count == 1000)
                            {
                                lastp = new IndexPart()
                                {
                                    Ids       = new List <int>(),
                                    IntValues = new List <int>()
                                };
                                parts.Add(lastp);
                            }
                            lastp.IntValues.Add(index_new.IntValues[i]);
                            lastp.Ids.Add(index_new.Ids[i]);
                        }
                    }
                    for (int i = 0; i < index.Parts.Count(); i++)
                    {
                        index.Parts[i].Ids = null;
                    }
                }
                else if (index.IndexType == IndexType.GroupOnly)     //group index only
                {
                    int map = 0;
                    if (index.GroupListMapping.Any())
                    {
                        map = index.GroupListMapping.Max(f => f.Value);
                        map++;
                    }
                    for (int i = 0; i < index.Parts.Count; i++)
                    {
                        var p = index.Parts[i];
                        p.Ids = ids_index.Parts[i].IntValues;
                        IndexPart     np      = null;
                        HashSet <int> removed = null;
                        int           icount  = p.GroupValues.Count;
                        for (int j = 0; j < icount; j++)
                        {
                            int cid = p.Ids[j];
                            if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid))
                            {
                                if (np == null)
                                {
                                    np = new IndexPart()
                                    {
                                        Ids         = new List <int>(p.Ids),
                                        GroupValues = new List <int>(p.GroupValues)
                                    };
                                }
                                if (!index.GroupListMapping.ContainsKey(index_changed.IntValues[cid]))
                                {
                                    index.GroupListMapping[index_changed.IntValues[cid]] = map;
                                    map++;
                                }
                                np.GroupValues[j] = index.GroupListMapping[index_changed.IntValues[cid]];
                            }
                            if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
                            {
                                if (removed == null)
                                {
                                    removed = new HashSet <int>();
                                }
                                removed.Add(cid);
                            }
                        }
                        if (removed != null)
                        {
                            if (np == null)
                            {
                                np = new IndexPart()
                                {
                                    Ids         = new List <int>(p.Ids),
                                    GroupValues = new List <int>(p.GroupValues)
                                };
                            }
                            var npp = new IndexPart()
                            {
                                Ids         = new List <int>(),
                                GroupValues = new List <int>()
                            };
                            for (int z = 0; z < np.Ids.Count; z++)
                            {
                                if (!removed.Contains((int)np.Ids[z]))
                                {
                                    npp.Ids.Add(np.Ids[z]);
                                    npp.GroupValues.Add(np.GroupValues[z]);
                                }
                            }
                            np = npp;
                        }
                        if (np != null)
                        {
                            p = np;     //wow, finally!
                        }
                        parts.Add(p);
                    }
                    if (index_new != null)
                    {
                        IndexPart lastp = null;
                        if (parts.Any())
                        {
                            lastp = parts.LastOrDefault();
                        }
                        if (lastp == null)
                        {
                            lastp = new IndexPart()
                            {
                                Ids         = new List <int>(),
                                GroupValues = new List <int>()
                            };
                            parts.Add(lastp);
                        }
                        else
                        {
                            lastp = new IndexPart()
                            {
                                Ids         = new List <int>(lastp.Ids),
                                GroupValues = new List <int>(lastp.GroupValues)
                            };
                            parts[parts.Count() - 1] = lastp;
                        }
                        for (int i = 0; i < index_new.IntValues.Count; i++)
                        {
                            if (lastp.Ids.Count == 1000)
                            {
                                lastp = new IndexPart()
                                {
                                    Ids         = new List <int>(),
                                    GroupValues = new List <int>()
                                };
                                parts.Add(lastp);
                            }
                            if (!index.GroupListMapping.ContainsKey((int)index_new.IntValues[i]))
                            {
                                index.GroupListMapping[(int)index_new.IntValues[i]] = map;
                                map++;
                            }
                            lastp.GroupValues.Add(index.GroupListMapping[(int)index_new.IntValues[i]]);
                            lastp.Ids.Add(index_new.Ids[i]);
                        }
                    }
                    for (int i = 0; i < index.Parts.Count(); i++)
                    {
                        index.Parts[i].Ids = null;
                    }
                }
                else     //both
                {
                    int map = 0;
                    if (index.GroupListMapping.Any())
                    {
                        map = index.GroupListMapping.Max(f => f.Value);
                        map++;
                    }
                    for (int i = 0; i < index.Parts.Count; i++)
                    {
                        var p = index.Parts[i];
                        p.Ids = ids_index.Parts[i].IntValues;
                        IndexPart     np      = null;
                        HashSet <int> removed = null;
                        int           icount  = p.GroupValues.Count;
                        for (int j = 0; j < icount; j++)
                        {
                            int cid = p.Ids[j];
                            if (has_changed_int /*&& changed_int_bloom[cid % dbloomsize]*/ && index_changed.IntValues.ContainsKey(cid))
                            {
                                if (np == null)
                                {
                                    np = new IndexPart()
                                    {
                                        Ids         = new List <int>(p.Ids),
                                        GroupValues = new List <int>(p.GroupValues),
                                        IntValues   = new List <int>(p.IntValues)
                                    };
                                }
                                if (!index.GroupListMapping.ContainsKey(index_changed.IntValues[cid]))
                                {
                                    index.GroupListMapping[index_changed.IntValues[cid]] = map;
                                    map++;
                                }
                                np.GroupValues[j] = index.GroupListMapping[index_changed.IntValues[cid]];
                                np.IntValues[j]   = index_changed.IntValues[cid];
                            }
                            if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
                            {
                                if (removed == null)
                                {
                                    removed = new HashSet <int>();
                                }
                                removed.Add(cid);
                            }
                        }
                        if (removed != null)
                        {
                            if (np == null)
                            {
                                np = new IndexPart()
                                {
                                    Ids         = new List <int>(p.Ids),
                                    GroupValues = new List <int>(p.GroupValues),
                                    IntValues   = new List <int>(p.IntValues)
                                };
                            }
                            var npp = new IndexPart()
                            {
                                Ids         = new List <int>(),
                                GroupValues = new List <int>(),
                                IntValues   = new List <int>()
                            };
                            for (int z = 0; z < np.Ids.Count; z++)
                            {
                                if (!removed.Contains((int)np.Ids[z]))
                                {
                                    npp.Ids.Add(np.Ids[z]);
                                    npp.GroupValues.Add(np.GroupValues[z]);
                                    npp.IntValues.Add(np.IntValues[z]);
                                }
                            }
                            np = npp;
                        }
                        if (np != null)
                        {
                            p = np;     //wow, finally!
                        }
                        parts.Add(p);
                    }
                    if (index_new != null)
                    {
                        IndexPart lastp = null;
                        if (parts.Any())
                        {
                            lastp = parts.LastOrDefault();
                        }
                        if (lastp == null)
                        {
                            lastp = new IndexPart()
                            {
                                Ids         = new List <int>(),
                                GroupValues = new List <int>(),
                                IntValues   = new List <int>()
                            };
                            parts.Add(lastp);
                        }
                        else
                        {
                            lastp = new IndexPart()
                            {
                                Ids         = new List <int>(lastp.Ids),
                                GroupValues = new List <int>(lastp.GroupValues),
                                IntValues   = new List <int>(lastp.IntValues)
                            };
                            parts[parts.Count() - 1] = lastp;
                        }
                        for (int i = 0; i < index_new.IntValues.Count; i++)
                        {
                            if (lastp.Ids.Count == 1000)
                            {
                                lastp = new IndexPart()
                                {
                                    Ids         = new List <int>(),
                                    GroupValues = new List <int>(),
                                    IntValues   = new List <int>()
                                };
                                parts.Add(lastp);
                            }
                            if (!index.GroupListMapping.ContainsKey((int)index_new.IntValues[i]))
                            {
                                index.GroupListMapping[(int)index_new.IntValues[i]] = map;
                                map++;
                            }
                            lastp.GroupValues.Add(index.GroupListMapping[(int)index_new.IntValues[i]]);
                            lastp.IntValues.Add(index_new.IntValues[i]);
                            lastp.Ids.Add(index_new.Ids[i]);
                        }
                    }
                    for (int i = 0; i < index.Parts.Count(); i++)
                    {
                        index.Parts[i].Ids = null;
                    }
                }
                break;

            case LinqDbTypes.double_:
            case LinqDbTypes.DateTime_:
                for (int i = 0; i < index.Parts.Count; i++)
                {
                    var p = index.Parts[i];
                    p.Ids = ids_index.Parts[i].IntValues;
                    IndexPart     np      = null;
                    HashSet <int> removed = null;
                    int           icount  = p.DoubleValues.Count;
                    for (int j = 0; j < icount; j++)
                    {
                        int cid = p.Ids[j];
                        if (has_changed_double /*&& changed_double_bloom[cid % dbloomsize]*/ && index_changed.DoubleValues.ContainsKey(cid))
                        {
                            if (np == null)
                            {
                                np = new IndexPart()
                                {
                                    Ids          = new List <int>(p.Ids),
                                    DoubleValues = new List <double>(p.DoubleValues)
                                };
                            }
                            np.DoubleValues[j] = index_changed.DoubleValues[cid];
                        }
                        if (has_deleted /*&& deleted_bloom[cid % dbloomsize]*/ && index_deleted.Ids.Contains(cid))
                        {
                            if (removed == null)
                            {
                                removed = new HashSet <int>();
                            }
                            removed.Add(cid);
                        }
                    }
                    if (removed != null)
                    {
                        if (np == null)
                        {
                            np = new IndexPart()
                            {
                                Ids          = new List <int>(p.Ids),
                                DoubleValues = new List <double>(p.DoubleValues)
                            };
                        }
                        var npp = new IndexPart()
                        {
                            Ids          = new List <int>(),
                            DoubleValues = new List <double>()
                        };
                        for (int z = 0; z < np.Ids.Count; z++)
                        {
                            if (!removed.Contains((int)np.Ids[z]))
                            {
                                npp.Ids.Add(np.Ids[z]);
                                npp.DoubleValues.Add(np.DoubleValues[z]);
                            }
                        }
                        np = npp;
                    }
                    if (np != null)
                    {
                        p = np;     //wow, finally!
                    }
                    parts.Add(p);
                }
                if (index_new != null)
                {
                    IndexPart lastp = null;
                    if (parts.Any())
                    {
                        lastp = parts.LastOrDefault();
                    }
                    if (lastp == null)
                    {
                        lastp = new IndexPart()
                        {
                            Ids          = new List <int>(),
                            DoubleValues = new List <double>()
                        };
                        parts.Add(lastp);
                    }
                    else
                    {
                        lastp = new IndexPart()
                        {
                            Ids          = new List <int>(lastp.Ids),
                            DoubleValues = new List <double>(lastp.DoubleValues)
                        };
                        parts[parts.Count() - 1] = lastp;
                    }
                    for (int i = 0; i < index_new.DoubleValues.Count; i++)
                    {
                        if (lastp.Ids.Count == 1000)
                        {
                            lastp = new IndexPart()
                            {
                                Ids          = new List <int>(),
                                DoubleValues = new List <double>()
                            };
                            parts.Add(lastp);
                        }
                        lastp.DoubleValues.Add(index_new.DoubleValues[i]);
                        lastp.Ids.Add(index_new.Ids[i]);
                    }
                }
                for (int i = 0; i < index.Parts.Count(); i++)
                {
                    index.Parts[i].Ids = null;
                }
                break;

            default:
                break;
            }

            //free old snapshots
            if ((DateTime.Now - last_cleanup[type + "|" + prop]).TotalMilliseconds > 30000)
            {
                var alive     = snapshots_alive[type + "|" + prop];
                var new_alive = new List <Tuple <bool, string> >(); //bool - schedueled for deletion
                foreach (var sn in alive)
                {
                    if (!sn.Item1)
                    {
                        var nsn = new Tuple <bool, string>(true, sn.Item2);
                        new_alive.Add(nsn);
                    }
                    else
                    {
                        IndexGeneric val = null;
                        indexes.TryRemove(sn.Item2, out val);
                    }
                }
                new_alive.Add(new Tuple <bool, string>(false, type + "|" + prop + "|" + snapshot_id));
                snapshots_alive[type + "|" + prop] = new_alive;
                last_cleanup[type + "|" + prop]    = DateTime.Now;
            }
            else
            {
                snapshots_alive[type + "|" + prop].Add(new Tuple <bool, string>(false, type + "|" + prop + "|" + snapshot_id));
            }

            key          = type + "|" + prop + "|" + snapshot_id;
            indexes[key] = new_index;
            latest_snapshots[latest_key] = snapshot_id;
        }