private bool post_check_data(workspace wspace) { if (profile.Remaped_IdTables.Count != configed_tables.Values.Count) { return(false); } bool all_exist = true; foreach (var v in profile.Remaped_IdTables) { if (!configed_tables.Keys.Contains(v.table_name.table_name)) { all_exist = false; break; } } if (!all_exist) { return(false); } foreach (var v in profile.Remaped_IdTables) { var tbl_count = new cmd_wraper.table_cmd(curr_conn, v.table_name, wspace.TargetDb).get_record_count(); if (tbl_count != v.Records.Count) { return(false); } } return(true); }
bool post_check_data(workspace wspace) { if (profile.Remaped_IdNameTables.Count != configed_tables.Count) { return(false); } foreach (var v in profile.Remaped_IdNameTables) { if (!configed_tables.ContainsKey(v.table_name.table_name)) { return(false); } } foreach (var v in configed_tables) { bool bexist = false; foreach (var kvp in profile.Remaped_IdNameTables) { if (kvp.table_name.table_name == v.Key) { bexist = true; break; } } if (!bexist) { return(false); } } return(true); }
void post_check_data(workspace ws) { // just check count & 2 sample if (cached.Remaped_Avatars.Count != cached.All_Avatars.Count) { throw new Exception("post check failed,rename avatar table"); } var renamed_cache = new List <AvatarInfo>(); foreach (var kvp in cached.Remaped_Avatars) { if (kvp.new_name != kvp.old_name) { renamed_cache.Add(kvp); } } // todo: random check for (int i = 0; i < 2 & i < renamed_cache.Count; ++i) { var ai = renamed_cache[i]; var anr = new cog_wraper.avatar_name_reader(ai.new_dbid); new cog_wraper.ogtable_reader <cog_wraper.avatar_name_reader>(anr, curr_conn, ws.TargetDb, avatar_table_name).load_summary(); if (anr.Value != ai.new_name) { throw new Exception("post check failed,avatar new name"); } } }
bool post_check_data(workspace wspace) { if (configed_columns.Count == profile.RemapedScriptsTable.Count) { return(true); } return(false); }
public void post_check_data(workspace wspace) { var merged_count = new cmd_wraper.table_cmd(current_connection, avatar_table_name, wspace.TargetDb).get_record_count(); if (merged_count != profile.All_Avatars.Count) { throw new Exception("post_check failed,avatar table"); } }
private void load_fakekey_dbidname_cache(workspace wspace) { fakekey2keyname = new Dictionary <string, KeyValuePair <ulong, string> >(); foreach (var nava in profile.Remaped_Avatars) { var fakekey = safeformat.get_keystr_from_dbid_serverid(nava.old_dbid, nava.serverid); fakekey2keyname.Add(fakekey, new KeyValuePair <ulong, string>(nava.new_dbid, nava.new_name)); } }
private void load_fakekey_dbid_cache(workspace wspace) { fakekey2dbid = new Dictionary <string, ulong>(); foreach (var nava in profile.Remaped_Avatars) { var fakekey = safeformat.get_keystr_from_dbid_serverid(nava.old_dbid, nava.serverid); fakekey2dbid.Add(fakekey, nava.new_dbid); } }
public void post_check_data(workspace wspace) { var merged_count = new cmd_wraper.table_cmd(curr_conn, account_table_name, wspace.TargetDb).get_record_count(); if (merged_count != cached.Remaped_Accounts.Count) { throw new Exception("post_check failed,remap account table"); } }
public void main(workspace wspace) { var files = System.IO.Directory.GetFiles(System.Environment.CurrentDirectory, "*.dat"); foreach (var f in files) { System.IO.File.Delete(f); } }
public void process(workspace wspace) { if (System.IO.File.Exists(file_data_name)) { // load load_from_data_file(wspace); } else { create_data_alonewith_save_db_file(wspace); } post_check_data(wspace); }
void create_alonewith_database_operation(workspace wspace) { var dst_tables = new List <string>(); foreach (var v in configed_tables) { var tbl_name = new data.tbname(v); new cmd_wraper.table_cmd(curr_conn, tbl_name, wspace.TargetDb).drop_table(true); new cnl_wrapper.nltable_cmd(curr_conn, wspace.TargetDb, tbl_name).clone_from(wspace.SchemaDatabase); dst_tables.Add(v); } profile.Simple_Tables = dst_tables; // save to disk new cpe_wraper.serialize_op <List <string> >().save(file_data_name, profile.Simple_Tables); }
bool post_check_data(workspace wspace) { foreach (var v in configed_tables) { if (!profile.One_Tables.Contains(v)) { return(false); } } foreach (var v in profile.One_Tables) { if (!configed_tables.Contains(v)) { return(false); } } return(true); }
public void process(workspace wspace) { if (System.IO.File.Exists(file_data_name)) { // load load_from_data_file(wspace); } else { var watch = new System.Diagnostics.Stopwatch(); watch.Start(); // create create_alonewith_database_operation(wspace); watch.Stop(); Console.WriteLine("avatar table use time : " + watch.ElapsedMilliseconds / 1000); } post_check_data(wspace); }
public void create_alonewith_database_operation(workspace wspace) { new cmd_wraper.table_cmd(current_connection, avatar_table_name, wspace.TargetDb).drop_table(true); new cnl_wrapper.nltable_cmd(current_connection, wspace.TargetDb, avatar_table_name).copy_from(wspace.SchemaDatabase); // fill adaptor foreach (var db in wspace.SourceDbs) { new cnl_wrapper.nltable_cmd(current_connection, wspace.TargetDb, avatar_table_name).fill_from(db); } // load all to cache var avatar_reader = new cog_wraper.avatar_summary_reader(); new cog_wraper.ogtable_reader <cog_wraper.avatar_summary_reader>(avatar_reader, current_connection, wspace.TargetDb, avatar_table_name).load_summary(); var datas = avatar_reader.Values; profile.All_Avatars = datas; // save cache to disk new cpe_wraper.serialize_op <List <AvatarInfo> >().save(file_data_name, datas); }
public void process(workspace wspace) { load_config_file(); if (System.IO.File.Exists(file_data_name)) { // load load_from_data_file(wspace); } else { // create create_alonewith_database_operation(wspace); } if (!post_check_data(wspace)) { create_alonewith_database_operation(wspace); } if (!post_check_data(wspace)) { throw new Exception("post check fail,simpe dropall tables"); } }
public void main(workspace wspace) { var dr = new diagram_rawdata(); var watch = new System.Diagnostics.Stopwatch(); watch.Start(); foreach (var v in wspace.SourceDbs) { var tables = new cmd_wraper.database_cmd(data.database.Instance, v).show_tables(); foreach (var t in tables) { if (t.StartsWith(@"tbl_")) { var tbl = new data.tbname(t); new check_table(data.database.Instance, tbl, v, dr).search_table(); } } } watch.Stop(); Console.WriteLine("read database used seconds:" + watch.ElapsedMilliseconds / 1000); dr.save_to_disk("checked_tables.xml"); }
public void process(workspace wspace) { load_config_file(); load_fakekey_dbidname_cache(wspace); if (System.IO.File.Exists(file_data_name)) { // load load_from_data_file(wspace); } else { create_alonewith_database_operation(wspace); } if (!post_check_data(wspace)) { create_alonewith_database_operation(wspace); } if (!post_check_data(wspace)) { throw new Exception("post check fail,remap id name tables"); } }
public void post_check_data(workspace wspace) { var merged_count = new cmd_wraper.table_cmd(conn, account_table_name, wspace.TargetDb).get_record_count(); if (merged_count != profile.All_Account.Count) { // may, we have deleted some accounts var accounts = new HashSet <string>(); foreach (var v in profile.All_Account) { if (accounts.Contains(v.old_name)) { continue; } accounts.Add(v.old_name); } if (accounts.Count != merged_count) { throw new Exception("post_check failed,account table"); } } }
void load_from_data_file(workspace wspace) { profile.RemapedScriptsTable = new cpe_wraper.serialize_op <List <ScriptIdInfoSet> >().load(file_data_name); }
private void create_alonewith_database_operation(workspace wspace) { var all_data = new List <TableIdInfoSet>(); foreach (var cfgtbl in configed_tables) { var table_name = new data.tbname(cfgtbl.Key); // create table new cmd_wraper.table_cmd(curr_conn, table_name, wspace.TargetDb).drop_table(true); new cnl_wrapper.nltable_cmd(curr_conn, wspace.TargetDb, table_name).copy_from(wspace.SchemaDatabase); // fill data foreach (var v in wspace.SourceDbs) { new cnl_wrapper.nltable_cmd(curr_conn, wspace.TargetDb, table_name).fill_from(v); } } // load datas foreach (var cfgtbl in configed_tables) { var table_set = new TableIdInfoSet(); table_set.table_name = new data.tbname(cfgtbl.Key); table_set.Desc = new TableIdInfoDesc(); table_set.Desc.columns = new List <string>(); foreach (var key_name in cfgtbl.Value) { table_set.Desc.columns.Add(key_name); } var tir = new cog_wraper.table_idremap_reader(table_set.Desc); new cmd_wraper.table_cmd(curr_conn, new data.tbname(cfgtbl.Key), wspace.TargetDb).load_records(tir); table_set.Records = tir.Values; all_data.Add(table_set); } // remap ids foreach (var tis in all_data) { var desc = tis.Desc; foreach (var record in tis.Records) { for (int i = 0; i < record.values.Count; ++i) { var curr_v = record.values[i]; var server_id = record.serverid; if (curr_v > 0) { var new_id = get_dbid_by_fakekey(curr_v, server_id); record.values[i] = new_id; } else { record.values[i] = 0; } } } } Action <TableIdInfoDesc, List <TableIdInfo>, data.tbname> act = (x, y, z) => { var rbw = new cog_wraper.remapid_batch_writer(x, y); new cmd_wraper.table_cmd(curr_conn, z, wspace.TargetDb).update_record(rbw); }; List <TableIdInfo> wait_for_saving = new List <TableIdInfo>(); int page_size = 1000; // change it if sql text is too long // save all ids foreach (var tis in all_data) { var desc = tis.Desc; foreach (var ti in tis.Records) { // todo batch update wait_for_saving.Add(ti); if (wait_for_saving.Count > page_size) { // save batch act(desc, wait_for_saving, tis.table_name); wait_for_saving.Clear(); } } if (wait_for_saving.Count > 0) { // save batch act(desc, wait_for_saving, tis.table_name); } } // data to loacal disk profile.Remaped_IdTables = all_data; new cpe_wraper.serialize_op <List <TableIdInfoSet> >().save(file_data_name, profile.Remaped_IdTables); }
public void load_from_data_file(workspace wspace) { profile.All_Avatars = new cpe_wraper.serialize_op <List <AvatarInfo> >().load(file_data_name); }
void load_from_data_file(workspace wspace) { profile.One_Tables = new cpe_wraper.serialize_op <List <string> >().load(file_data_name); }
public void process(workspace wspace) { var cmd = new cmd_wraper.database_cmd(current_connection, wspace.TargetDb); cmd.create_database(false); }
void create_alonewith_database_operation(workspace wspace) { var all_data = new List <TableIdNameInfoSet>(); foreach (var cfgtbl in configed_tables) { var table_name = new data.tbname(cfgtbl.Key); // drop , create new cmd_wraper.table_cmd(curr_conn, table_name, wspace.TargetDb).drop_table(true); new cnl_wrapper.nltable_cmd(curr_conn, wspace.TargetDb, table_name).copy_from(wspace.SchemaDatabase); // fill data foreach (var v in wspace.SourceDbs) { new cnl_wrapper.nltable_cmd(curr_conn, wspace.TargetDb, table_name).fill_from(v); } } // load data foreach (var cfgtbl in configed_tables) { var table_set = new TableIdNameInfoSet(); table_set.table_name = new data.tbname(cfgtbl.Key); table_set.Desc = new TableIdNameDesc(); table_set.Desc.left_name = cfgtbl.Value.Key; table_set.Desc.right_name = cfgtbl.Value.Value; var tinr = new cog_wraper.table_idname_remap_reader(table_set.Desc); new cmd_wraper.table_cmd(curr_conn, new data.tbname(cfgtbl.Key), wspace.TargetDb).load_records(tinr); table_set.Records = tinr.Values; all_data.Add(table_set); } // remap id / name foreach (var tins in all_data) { var desc = tins.Desc; for (int i = 0; i < tins.Records.Count; ++i) { var data = tins.Records[i]; var kvp = get_info_by_fakekey(data.record_id, data.serverid); tins.Records[i].record_id = kvp.Key; tins.Records[i].record_name = kvp.Value; } } // save all Action <TableIdNameDesc, List <TableIdNameInfo>, data.tbname> act = (x, y, z) => { var rinw = new cog_wraper.remap_idname_batch_writer(x, y); new cmd_wraper.table_cmd(curr_conn, z, wspace.TargetDb).update_record(rinw); }; var wait_for_saving = new List <TableIdNameInfo>(); int page_size = 1000; foreach (var tins in all_data) { var desc = tins.Desc; foreach (var v in tins.Records) { wait_for_saving.Add(v); if (wait_for_saving.Count > page_size) { act(desc, wait_for_saving, tins.table_name); wait_for_saving.Clear(); } } if (wait_for_saving.Count > 0) { act(desc, wait_for_saving, tins.table_name); wait_for_saving.Clear(); } } // data to local disk profile.Remaped_IdNameTables = all_data; new cpe_wraper.serialize_op <List <TableIdNameInfoSet> >().save(file_data_name, profile.Remaped_IdNameTables); }
void load_from_data_file(workspace wspace) { profile.Remaped_IdNameTables = new cpe_wraper.serialize_op <List <TableIdNameInfoSet> >().load(file_data_name); }
// how to rename // 1. rolename + server_id // 2. if exist, try rolename + auto_inc id void create_data_alonewith_save_db_file(workspace ws) { // clear context old_names.Clear(); conficted_names.Clear(); renamed_names.Clear(); old_name_to_avatars.Clear(); var all_avatar = cached.All_Avatars; // cache names & conficted names foreach (var v in all_avatar) { if (!old_names.Contains(v.old_name)) { old_names.Add(v.old_name); } else { conficted_names.Add(v.old_name); } if (!old_name_to_avatars.ContainsKey(v.old_name)) { old_name_to_avatars.Add(v.old_name, new List <AvatarInfo>()); } old_name_to_avatars[v.old_name].Add(v); } // rename foreach (var conficted in conficted_names) { var avatars = GetAvatarsInfoByOldName(conficted); foreach (var ava in avatars) { bool bRenamed = false; var try_new_name = ava.old_name + ws.GetSourceDbIndex(ava.serverid).ToString(); if (!old_names.Contains(try_new_name)) { old_names.Add(try_new_name); renamed_names.Add(ava.new_dbid, try_new_name); bRenamed = true; } if (!bRenamed) { int cur_idx = ws.SourceDbs.Count + 1; for (; cur_idx < 100; ++cur_idx) { try_new_name = ava.old_name + cur_idx.ToString(); if (!old_names.Contains(try_new_name)) { old_names.Add(try_new_name); renamed_names.Add(ava.new_dbid, try_new_name); bRenamed = true; break; } } } if (!bRenamed) { throw new Exception("too many rename tries,give up"); } } } old_names.Clear(); // apply new name var target_avatars = new List <AvatarInfo>(); foreach (var kvp in cached.All_Avatars) { var p = kvp.deep_copy(); if (renamed_names.ContainsKey(p.new_dbid)) { p.new_name = renamed_names[p.new_dbid]; } else { p.new_name = p.old_name; } target_avatars.Add(p); if (old_names.Contains(p.new_name)) { throw new Exception("rename failed"); } else { old_names.Add(p.new_name); } } cached.Remaped_Avatars = target_avatars; var watcher = new System.Diagnostics.Stopwatch(); watcher.Start(); // save to db int page_count = 1000; var avatars_for_save = new List <AvatarInfo>(); Action <List <AvatarInfo> > action = (o) => { var anw = new cog_wraper.avatar_name_batch_writer(avatars_for_save); new cog_wraper.ogtable_writer <cog_wraper.avatar_name_batch_writer>(anw, curr_conn, ws.TargetDb, avatar_table_name).save_summary(); }; foreach (var kvp in target_avatars) { if (renamed_names.ContainsKey(kvp.new_dbid)) { avatars_for_save.Add(kvp); if (avatars_for_save.Count >= page_count) { action(avatars_for_save); avatars_for_save.Clear(); } } } if (avatars_for_save.Count > 0) { action(avatars_for_save); avatars_for_save.Clear(); } watcher.Stop(); Console.WriteLine("save new names : " + watcher.ElapsedMilliseconds / 1000); // save to disk new cpe_wraper.serialize_op <List <AvatarInfo> >().save(file_data_name, cached.Remaped_Avatars); }
public void load_from_data_file(workspace wspace) { cached.Remaped_Accounts = new cpe_wraper.serialize_op <List <AccountInfo> >().load(file_data_name); }
void create_alonewith_database_operation(workspace wspace) { var all_data = new List <ScriptIdInfoSet>(); // load all foreach (var cfgtbl in configed_columns) { var tbl_name = new data.tbname(cfgtbl.Key); var column_name = cfgtbl.Value; var siis = new ScriptIdInfoSet(); siis.table_name = tbl_name; siis.Desc = new ScriptIdInfoDesc(); siis.Desc.column_name = cfgtbl.Value; // load data var sir = new cog_wraper.script_id_reader(siis.Desc); new cmd_wraper.table_cmd(curr_conn, tbl_name, wspace.TargetDb).load_records(sir); siis.Records = sir.Values; all_data.Add(siis); } // remap all(why not in same line? face to human is better than face to cpu) foreach (var s in all_data) { var desc = s.Desc; var table_name = s.table_name.table_name; var column_name = desc.column_name; var patterns = new cqx_wraper.qxtable_loader().load_pattern_by_script_id(config_file_name, table_name, column_name); foreach (var p in s.Records) { p.new_info = p.old_info; foreach (var pattern in patterns) { var oldinfo = p.new_info; var p1 = pattern.Key; var p2 = pattern.Value; var matches = System.Text.RegularExpressions.Regex.Matches(p.new_info, p1); foreach (System.Text.RegularExpressions.Match m in matches) { var secondinfo = m.Value; var m2 = System.Text.RegularExpressions.Regex.Matches(m.Value, p2); if (m2.Count > 1) { throw new Exception("too complex match,try a special method"); } var m3 = UInt64.Parse(((System.Text.RegularExpressions.Match)m2[0]).Value); var serverinfo = p.serverid; var m4 = get_dbid_by_fakekey(m3, serverinfo); var replaced_info = secondinfo.Replace(m3.ToString(), m4.ToString()); oldinfo = oldinfo.Replace(secondinfo, replaced_info); } p.new_info = oldinfo; } } } Action <ScriptIdInfoDesc, List <ScriptIdInfo>, data.tbname> act = (x, y, z) => { var siw = new cog_wraper.script_id_writer(x, y); new cmd_wraper.table_cmd(curr_conn, z, wspace.TargetDb).update_record(siw); }; // save to db all int page_size = 1000; foreach (var s in all_data) { var wait_for_saving = new List <ScriptIdInfo>(); foreach (var p in s.Records) { wait_for_saving.Add(p); if (wait_for_saving.Count > page_size) { act(s.Desc, wait_for_saving, s.table_name); wait_for_saving.Clear(); } } if (wait_for_saving.Count > 0) { act(s.Desc, wait_for_saving, s.table_name); wait_for_saving.Clear(); } } // save to local disk profile.RemapedScriptsTable = all_data; new cpe_wraper.serialize_op <List <ScriptIdInfoSet> >().save(file_data_name, profile.RemapedScriptsTable); }
// how to remap // 1. avatar dbid // 2. avatar name // 3. account merge public void create_data_alonewith_save_db_file(workspace ws) { // load avatar info cache var dbids_to_new = new Dictionary <string, UInt64>(); var dbids_to_name = new Dictionary <string, string>(); var dbid_to_avatar = new Dictionary <UInt64, AvatarInfo>(); foreach (var ava_info in cached.Remaped_Avatars) { var fake_key = safeformat.get_keystr_from_dbid_serverid(ava_info.old_dbid, ava_info.serverid); dbids_to_new.Add(fake_key, ava_info.new_dbid); dbids_to_name.Add(fake_key, ava_info.new_name); dbid_to_avatar.Add(ava_info.new_dbid, ava_info); } // load all accounts var avatarsInfo_all = new Dictionary <string, List <string> >(); avatarsInfo_all.Clear(); foreach (var acci in cached.All_Account) { if (!avatarsInfo_all.ContainsKey(acci.old_name)) { avatarsInfo_all.Add(acci.old_name, new List <string>()); } avatarsInfo_all[acci.old_name].Add(acci.old_avatarinfo); } // first round,changed dbid & name var accounts = new List <AccountInfo>(); foreach (var acc in cached.All_Account) { var p = acc.deep_copy(); var avatarsInfo = new scriptdata.AvatarsInfo(); avatarsInfo.LoadFromString(p.old_avatarinfo); // change id & name foreach (var ava in avatarsInfo.AllAvatars) { var old_id = ava.id; var server_id = p.serverid; var fake_key = safeformat.get_keystr_from_dbid_serverid(old_id, server_id); var new_id = dbids_to_new[fake_key]; var new_name = dbids_to_name[fake_key]; ava.id = new_id; ava.name = new_name; } p.new_avatarinfo = avatarsInfo.SaveToString(); accounts.Add(p); } var keeps_acc_ids = new HashSet <UInt64>(); var merged_acc_ids = new HashSet <UInt64>(); var new_accounts = new Dictionary <string, AccountInfo>(); foreach (var acc in accounts) { if (new_accounts.ContainsKey(acc.old_name)) { // merge var old_acc = new_accounts[acc.old_name]; var cur_acc = acc; var c1 = new scriptdata.AvatarsInfo(); c1.LoadFromString(old_acc.new_avatarinfo); var c2 = new scriptdata.AvatarsInfo(); c2.LoadFromString(cur_acc.new_avatarinfo); c1.Merge(c2); old_acc.new_avatarinfo = c1.SaveToString(); merged_acc_ids.Add(acc.new_dbid); } else { // simple add new_accounts.Add(acc.old_name, acc); keeps_acc_ids.Add(acc.new_dbid); } } accounts.Clear(); foreach (var kvp in new_accounts) { accounts.Add(kvp.Value); } cached.Remaped_Accounts = accounts; var watcher = new System.Diagnostics.Stopwatch(); watcher.Start(); // save to db foreach (var dbid in merged_acc_ids) { var ard = new cog_wraper.account_record_deletor(dbid); new cog_wraper.ogtable_deletor <cog_wraper.account_record_deletor>(ard, curr_conn, ws.TargetDb, account_table_name).delete_summary(); } var count = cached.Remaped_Accounts.Count; int page_count = 1000; for (var i = 0; i < count; ++i) { var little_set = new List <AccountInfo>(); var start = i * page_count; int j = start; for (; j < start + page_count && j < count; ++j) { little_set.Add(cached.Remaped_Accounts[j]); } if (little_set.Count == 0) { break; } var aaw = new cog_wraper.account_avatarsinfo_batch_writer(little_set); new cog_wraper.ogtable_writer <cog_wraper.account_avatarsinfo_batch_writer>(aaw, curr_conn, ws.TargetDb, account_table_name).save_summary(); } watcher.Stop(); Console.WriteLine("save new accounts : " + watcher.ElapsedMilliseconds / 1000); // save to disk new cpe_wraper.serialize_op <List <AccountInfo> >().save(file_data_name, cached.Remaped_Accounts); }