private void MigrateMDBToCH_Click(object sender, RoutedEventArgs e)
{
WaitScreen.Splash();
string tableName = MDBTablesComboBox.SelectedItem.ToString();
Stopwatch stopwatch = Stopwatch.StartNew();
CSVToClickHouse.CHConn(() => GetData(tableName));
stopwatch.Stop();
var elapsedBulkMs = stopwatch.ElapsedMilliseconds;
Timer.Content = "Migration time : " + elapsedBulkMs + "ms";
NumberOfRecords.Content = "Records : " + (lineCount);
Logs xd = new DBUplader.Logs();
xd.AddLog("--------------------------------------------");
xd.AddLog("Migrate MongoDB to ClickHouse");
xd.AddLog(" Data name : " + tableName);
xd.AddLog(" Bulk time : " + elapsedBulkMs + "ms");
xd.AddLog("Records inserted : " + lineCount);
}
public void GetData(string tableName)
{
WaitScreen.Splash();
this._Types = null;
this._Names = null;
var db = new MongoClient("mongodb://localhost:27017").GetDatabase("test").GetCollection <BsonDocument>(tableName);
var count = db.CountDocuments(FilterDefinition <BsonDocument> .Empty);
var stepSize = 1000;
for (int i = 0; i < Math.Ceiling((double)count / stepSize); i++)
{
var list = db.Find(new BsonDocument()).Skip(i * stepSize).Limit(stepSize).ToList();
var values = this.getValues(list);
if (i == 0)
{
values = values.ToList();
CSVToClickHouse.CreateCHTable(this._Names, this._Types, tableName);
}
CSVToClickHouse.InsertValues(values, tableName, this._Names);
}
lineCount = (int)count;
}
private IEnumerable <object> getValues(List <BsonDocument> tab)
{
if (tab.Any())
{
var fItems = tab[0].Values.Where(p => p.BsonType != BsonType.ObjectId).Select(p => p.ToString()).ToArray();
if (_Types == null)
{
_Names = tab[0].Names.Where(p => p != "_id").ToList();
_Types = CSVToClickHouse.CheckType(fItems);
}
foreach (var t in tab)
{
object[] tabResult = new object[_Types.Count];
for (int i = 0; i < _Names.Count; i++)
{
var v = CSVToClickHouse.GetValue(_Types[i], t.GetValue(_Names[i]).ToString());
tabResult[i] = v;
}
yield return(tabResult);
}
}
}
private void CSVToCH_Click(object sender, RoutedEventArgs e)
{
CSVToClickHouse win = new CSVToClickHouse();
win.Show();
}
