Пример #1
0
        private void MigrateMDBToCH_Click(object sender, RoutedEventArgs e)
        {
            WaitScreen.Splash();
            string    tableName = MDBTablesComboBox.SelectedItem.ToString();
            Stopwatch stopwatch = Stopwatch.StartNew();

            CSVToClickHouse.CHConn(() => GetData(tableName));
            stopwatch.Stop();
            var elapsedBulkMs = stopwatch.ElapsedMilliseconds;

            Timer.Content = "Migration time : " + elapsedBulkMs + "ms";

            NumberOfRecords.Content = "Records : " + (lineCount);

            Logs xd = new DBUplader.Logs();

            xd.AddLog("--------------------------------------------");
            xd.AddLog("Migrate MongoDB to ClickHouse");
            xd.AddLog("       Data name : " + tableName);
            xd.AddLog("       Bulk time : " + elapsedBulkMs + "ms");
            xd.AddLog("Records inserted : " + lineCount);
        }
Пример #2
0
        public void GetData(string tableName)
        {
            WaitScreen.Splash();
            this._Types = null;
            this._Names = null;
            var db = new MongoClient("mongodb://localhost:27017").GetDatabase("test").GetCollection <BsonDocument>(tableName);

            var count    = db.CountDocuments(FilterDefinition <BsonDocument> .Empty);
            var stepSize = 1000;

            for (int i = 0; i < Math.Ceiling((double)count / stepSize); i++)
            {
                var list   = db.Find(new BsonDocument()).Skip(i * stepSize).Limit(stepSize).ToList();
                var values = this.getValues(list);
                if (i == 0)
                {
                    values = values.ToList();
                    CSVToClickHouse.CreateCHTable(this._Names, this._Types, tableName);
                }
                CSVToClickHouse.InsertValues(values, tableName, this._Names);
            }
            lineCount = (int)count;
        }
Пример #3
0
        private IEnumerable <object> getValues(List <BsonDocument> tab)
        {
            if (tab.Any())
            {
                var fItems = tab[0].Values.Where(p => p.BsonType != BsonType.ObjectId).Select(p => p.ToString()).ToArray();

                if (_Types == null)
                {
                    _Names = tab[0].Names.Where(p => p != "_id").ToList();
                    _Types = CSVToClickHouse.CheckType(fItems);
                }
                foreach (var t in tab)
                {
                    object[] tabResult = new object[_Types.Count];
                    for (int i = 0; i < _Names.Count; i++)
                    {
                        var v = CSVToClickHouse.GetValue(_Types[i], t.GetValue(_Names[i]).ToString());
                        tabResult[i] = v;
                    }
                    yield return(tabResult);
                }
            }
        }
Пример #4
0
        private void CSVToCH_Click(object sender, RoutedEventArgs e)
        {
            CSVToClickHouse win = new CSVToClickHouse();

            win.Show();
        }