private void MigrateMDBToCH_Click(object sender, RoutedEventArgs e) { WaitScreen.Splash(); string tableName = MDBTablesComboBox.SelectedItem.ToString(); Stopwatch stopwatch = Stopwatch.StartNew(); CSVToClickHouse.CHConn(() => GetData(tableName)); stopwatch.Stop(); var elapsedBulkMs = stopwatch.ElapsedMilliseconds; Timer.Content = "Migration time : " + elapsedBulkMs + "ms"; NumberOfRecords.Content = "Records : " + (lineCount); Logs xd = new DBUplader.Logs(); xd.AddLog("--------------------------------------------"); xd.AddLog("Migrate MongoDB to ClickHouse"); xd.AddLog(" Data name : " + tableName); xd.AddLog(" Bulk time : " + elapsedBulkMs + "ms"); xd.AddLog("Records inserted : " + lineCount); }
public void GetData(string tableName) { WaitScreen.Splash(); this._Types = null; this._Names = null; var db = new MongoClient("mongodb://localhost:27017").GetDatabase("test").GetCollection <BsonDocument>(tableName); var count = db.CountDocuments(FilterDefinition <BsonDocument> .Empty); var stepSize = 1000; for (int i = 0; i < Math.Ceiling((double)count / stepSize); i++) { var list = db.Find(new BsonDocument()).Skip(i * stepSize).Limit(stepSize).ToList(); var values = this.getValues(list); if (i == 0) { values = values.ToList(); CSVToClickHouse.CreateCHTable(this._Names, this._Types, tableName); } CSVToClickHouse.InsertValues(values, tableName, this._Names); } lineCount = (int)count; }
private IEnumerable <object> getValues(List <BsonDocument> tab) { if (tab.Any()) { var fItems = tab[0].Values.Where(p => p.BsonType != BsonType.ObjectId).Select(p => p.ToString()).ToArray(); if (_Types == null) { _Names = tab[0].Names.Where(p => p != "_id").ToList(); _Types = CSVToClickHouse.CheckType(fItems); } foreach (var t in tab) { object[] tabResult = new object[_Types.Count]; for (int i = 0; i < _Names.Count; i++) { var v = CSVToClickHouse.GetValue(_Types[i], t.GetValue(_Names[i]).ToString()); tabResult[i] = v; } yield return(tabResult); } } }
private void CSVToCH_Click(object sender, RoutedEventArgs e) { CSVToClickHouse win = new CSVToClickHouse(); win.Show(); }