public void Process_Export_Queue_Item() { //System.Console.WriteLine ("{0} check_for_changes_job.Process_Export_Queue_Item: started", System.DateTime.Now); List <export_queue_item> result = new List <export_queue_item> (); var get_curl = new cURL("GET", null, Program.config_couchdb_url + "/export_queue/_all_docs?include_docs=true", null, this.user_name, this.password); string responseFromServer = get_curl.execute(); IDictionary <string, object> response_result = Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject> (responseFromServer) as IDictionary <string, object>; IList <object> enumerable_rows = response_result ["rows"] as IList <object>; foreach (IDictionary <string, object> enumerable_item in enumerable_rows) { IDictionary <string, object> doc_item = enumerable_item ["doc"] as IDictionary <string, object>; if ( doc_item ["status"] != null && doc_item ["status"].ToString().StartsWith("In Queue...", StringComparison.OrdinalIgnoreCase)) { export_queue_item item = new export_queue_item(); item._id = doc_item ["_id"].ToString(); item._rev = doc_item ["_rev"].ToString(); item._deleted = doc_item.ContainsKey("_deleted") ? doc_item["_deleted"] as bool?: null; item.date_created = doc_item ["date_created"] as DateTime?; item.created_by = doc_item.ContainsKey("created_by") && doc_item ["created_by"] != null ? doc_item ["created_by"].ToString() : null; item.date_last_updated = doc_item ["date_last_updated"] as DateTime?; item.last_updated_by = doc_item.ContainsKey("last_updated_by") && doc_item ["last_updated_by"] != null ? doc_item ["last_updated_by"].ToString() : null; item.file_name = doc_item ["file_name"] != null ? doc_item ["file_name"].ToString() : null; item.export_type = doc_item ["export_type"] != null ? doc_item ["export_type"].ToString() : null; item.status = doc_item ["status"] != null ? doc_item ["status"].ToString() : null; result.Add(item); } } if (result.Count > 0) { if (result.Count > 1) { var comparer = Comparer <export_queue_item> .Create ( (x, y) => x.date_created.Value.CompareTo(y.date_created.Value) ); result.Sort(comparer); } export_queue_item item_to_process = result [0]; item_to_process.date_last_updated = new DateTime?(); //item_to_process.last_updated_by = g_uid; List <string> args = new List <string>(); args.Add("exporter:exporter"); args.Add("user_name:" + this.user_name); args.Add("password:"******"database_url:" + this.couch_db_url); args.Add("item_file_name:" + item_to_process.file_name); args.Add("item_id:" + item_to_process._id); if (item_to_process.export_type.StartsWith("core csv", StringComparison.OrdinalIgnoreCase)) { item_to_process.status = "Creating Export..."; item_to_process.last_updated_by = "mmria-server"; item_to_process.date_last_updated = DateTime.Now; Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore; string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings); var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password); responseFromServer = set_curl.execute(); try { mmria.server.model.actor.ScheduleInfoMessage scheduleInfo = new actor.ScheduleInfoMessage ( Configuration["mmria_settings:cron_schedule"], Configuration["mmria_settings:couchdb_url"], Configuration["mmria_settings:timer_user_name"], Configuration["mmria_settings:timer_password"], Configuration["mmria_settings:export_directory"] ); mmria.server.util.core_element_exporter core_element_exporter = new mmria.server.util.core_element_exporter(scheduleInfo); core_element_exporter.Execute(args.ToArray()); } catch (Exception ex) { System.Console.WriteLine(ex); } } else if (item_to_process.export_type.StartsWith("all csv", StringComparison.OrdinalIgnoreCase)) { item_to_process.status = "Creating Export..."; item_to_process.last_updated_by = "mmria-server"; item_to_process.date_last_updated = DateTime.Now; Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore; string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings); var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password); responseFromServer = set_curl.execute(); mmria.server.model.actor.ScheduleInfoMessage scheduleInfo = new actor.ScheduleInfoMessage ( Configuration["mmria_settings:cron_schedule"], Configuration["mmria_settings:couchdb_url"], Configuration["mmria_settings:timer_user_name"], Configuration["mmria_settings:timer_password"], Configuration["mmria_settings:export_directory"] ); try { mmria.server.util.mmrds_exporter mmrds_exporter = new mmria.server.util.mmrds_exporter(scheduleInfo); mmrds_exporter.Execute(args.ToArray()); } catch (Exception ex) { System.Console.WriteLine(ex); } } else if (item_to_process.export_type.StartsWith("cdc csv", StringComparison.OrdinalIgnoreCase)) { item_to_process.status = "Creating Export..."; item_to_process.last_updated_by = "mmria-server"; item_to_process.date_last_updated = DateTime.Now; Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore; string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings); var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password); responseFromServer = set_curl.execute(); args.Add("is_cdc_de_identified:true"); try { mmria.server.model.actor.ScheduleInfoMessage scheduleInfo = new actor.ScheduleInfoMessage ( Configuration["mmria_settings:cron_schedule"], Configuration["mmria_settings:couchdb_url"], Configuration["mmria_settings:timer_user_name"], Configuration["mmria_settings:timer_password"], Configuration["mmria_settings:export_directory"] ); mmria.server.util.mmrds_exporter mmrds_exporter = new mmria.server.util.mmrds_exporter(scheduleInfo); mmrds_exporter.Execute(args.ToArray()); } catch (Exception ex) { System.Console.WriteLine(ex); } } } }
public void Process_Export_Queue_Delete() { //System.Console.WriteLine ("{0} check_for_changes_job.Process_Export_Queue_Delete: started", System.DateTime.Now); List <export_queue_item> result = new List <export_queue_item> (); var get_curl = new cURL("GET", null, Program.config_couchdb_url + "/export_queue/_all_docs?include_docs=true", null, this.user_name, this.password); string responseFromServer = get_curl.execute(); IDictionary <string, object> response_result = Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject> (responseFromServer) as IDictionary <string, object>; IList <object> enumerable_rows = response_result ["rows"] as IList <object>; foreach (IDictionary <string, object> enumerable_item in enumerable_rows) { IDictionary <string, object> doc_item = enumerable_item ["doc"] as IDictionary <string, object>; if ( doc_item ["status"] != null && doc_item ["status"].ToString().StartsWith("Deleted", StringComparison.OrdinalIgnoreCase)) { export_queue_item item = new export_queue_item(); item._id = doc_item ["_id"].ToString(); item._rev = doc_item ["_rev"].ToString(); item._deleted = doc_item.ContainsKey("_deleted") ? doc_item["_deleted"] as bool?: null; item.date_created = doc_item ["date_created"] as DateTime?; item.created_by = doc_item.ContainsKey("created_by") && doc_item ["created_by"] != null ? doc_item ["created_by"].ToString() : null; item.date_last_updated = doc_item ["date_last_updated"] as DateTime?; item.last_updated_by = doc_item.ContainsKey("last_updated_by") && doc_item["last_updated_by"] != null ? doc_item ["last_updated_by"].ToString() : null; item.file_name = doc_item ["file_name"] != null ? doc_item ["file_name"].ToString() : null; item.export_type = doc_item ["export_type"] != null ? doc_item ["export_type"].ToString() : null; item.status = doc_item ["status"] != null ? doc_item ["status"].ToString() : null; result.Add(item); } } if (result.Count > 0) { if (result.Count > 1) { var comparer = Comparer <export_queue_item> .Create ( (x, y) => x.date_created.Value.CompareTo(y.date_created.Value) ); result.Sort(comparer); } export_queue_item item_to_process = result [0]; try { string item_directory_name = item_to_process.file_name.Substring(0, item_to_process.file_name.LastIndexOf(".")); string export_directory = System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings ["export_directory"], item_directory_name); try { if (System.IO.Directory.Exists(export_directory)) { System.IO.Directory.Delete(export_directory, true); } } catch (Exception Ex) { // do nothing for now System.Console.WriteLine("check_for_changes_job.Process_Export_Queue_Delete: Unable to Delete Directory {0}", export_directory); } string file_path = System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings ["export_directory"], item_to_process.file_name); try { if (System.IO.File.Exists(file_path)) { System.IO.File.Delete(file_path); } } catch (Exception Ex) { // do nothing for now System.Console.WriteLine("Program.Process_Export_Queue_Delete: Unable to Delete File {0}", file_path); } item_to_process.status = "expunged"; item_to_process.last_updated_by = "mmria-server"; Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore; string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings); var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password); responseFromServer = get_curl.execute(); } catch (Exception ex) { // do nothing for now } } }
public void Execute(string[] args) { if (args.Length > 1) { for (var i = 1; i < args.Length; i++) { string arg = args[i]; int index = arg.IndexOf(':'); string val = arg.Substring(index + 1, arg.Length - (index + 1)).Trim(new char[] { '\"' }); if (arg.ToLower().StartsWith("auth_token")) { this.auth_token = val; } else if (arg.ToLower().StartsWith("user_name")) { this.user_name = val; } else if (arg.ToLower().StartsWith("password")) { this.password = val; } else if (arg.ToLower().StartsWith("database_url")) { this.database_url = val; } else if (arg.ToLower().StartsWith("database")) { this.database_path = val; } else if (arg.ToLower().StartsWith("item_file_name")) { this.item_file_name = val; this.item_directory_name = this.item_file_name.Substring(0, this.item_file_name.LastIndexOf(".")); } else if (arg.ToLower().StartsWith("item_id")) { this.item_id = val; } } } string core_file_name = "core_mmria_export.csv"; if (string.IsNullOrWhiteSpace(this.database_url)) { this.database_url = System.Configuration.ConfigurationManager.AppSettings["couchdb_url"]; if (string.IsNullOrWhiteSpace(this.database_url)) { System.Console.WriteLine("missing database_url"); System.Console.WriteLine(" form database:[file path]"); System.Console.WriteLine(" example database:http://localhost:5984"); System.Console.WriteLine(" mmria.exe export user_name:user1 password:secret url:http://localhost:12345 database_url:http://localhost:5984"); return; } } if (string.IsNullOrWhiteSpace(this.user_name)) { System.Console.WriteLine("missing user_name"); System.Console.WriteLine(" form user_name:[user_name]"); System.Console.WriteLine(" example user_name:user1"); System.Console.WriteLine(" mmria.exe export user_name:user1 password:secret url:http://localhost:12345"); return; } if (string.IsNullOrWhiteSpace(this.password)) { System.Console.WriteLine("missing password"); System.Console.WriteLine(" form password:[password]"); System.Console.WriteLine(" example password:secret"); System.Console.WriteLine(" mmria.exe export user_name:user1 password:secret url:http://localhost:12345"); return; } string export_directory = System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings["export_directory"], this.item_directory_name); if (!System.IO.Directory.Exists(export_directory)) { System.IO.Directory.CreateDirectory(export_directory); } string URL = this.database_url + "/mmrds/_all_docs"; string urlParameters = "?include_docs=true"; cURL document_curl = new cURL("GET", null, URL + urlParameters, null, this.user_name, this.password); dynamic all_cases = Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject>(document_curl.execute()); string metadata_url = this.database_url + "/metadata/2016-06-12T13:49:24.759Z"; cURL metadata_curl = new cURL("GET", null, metadata_url, null, this.user_name, this.password); mmria.common.metadata.app metadata = Newtonsoft.Json.JsonConvert.DeserializeObject <mmria.common.metadata.app>(metadata_curl.execute()); /* * foreach (KeyValuePair<string, object> kvp in all_cases) * { * System.Console.WriteLine(kvp.Key); * } */ System.Collections.Generic.Dictionary <string, int> path_to_int_map = new Dictionary <string, int>(); System.Collections.Generic.Dictionary <string, string> path_to_file_name_map = new Dictionary <string, string>(); System.Collections.Generic.Dictionary <string, mmria.common.metadata.node> path_to_node_map = new Dictionary <string, mmria.common.metadata.node>(); System.Collections.Generic.Dictionary <string, string> path_to_grid_map = new Dictionary <string, string>(); System.Collections.Generic.Dictionary <string, string> path_to_multi_form_map = new Dictionary <string, string>(); System.Collections.Generic.Dictionary <string, string> grid_to_multi_form_map = new Dictionary <string, string>(); System.Collections.Generic.HashSet <string> path_to_flat_map = new System.Collections.Generic.HashSet <string>(); System.Collections.Generic.Dictionary <string, WriteCSV> path_to_csv_writer = new Dictionary <string, WriteCSV>(); generate_path_map (metadata, "", core_file_name, "", path_to_int_map, path_to_file_name_map, path_to_node_map, path_to_grid_map, path_to_multi_form_map, false, grid_to_multi_form_map, false, path_to_flat_map ); System.Collections.Generic.HashSet <string> flat_grid_set = new HashSet <string>(StringComparer.OrdinalIgnoreCase); System.Collections.Generic.HashSet <string> mutiform_grid_set = new HashSet <string>(StringComparer.OrdinalIgnoreCase); foreach (KeyValuePair <string, string> kvp in path_to_grid_map) { if (grid_to_multi_form_map.ContainsKey(kvp.Key)) { mutiform_grid_set.Add(kvp.Value); } } foreach (KeyValuePair <string, string> kvp in path_to_grid_map) { if ( !mutiform_grid_set.Contains(kvp.Value) && !flat_grid_set.Contains(kvp.Value) ) { flat_grid_set.Add(kvp.Value); } } /* * System.Collections.Generic.HashSet<string> mutiform_set = new HashSet<string>(StringComparer.OrdinalIgnoreCase); * foreach (KeyValuePair<string, string> kvp in path_to_multi_form_map) * { * if (!mutiform_set.Contains(kvp.Value)) * { * mutiform_set.Add(kvp.Value); * } * }*/ path_to_csv_writer.Add(core_file_name, new WriteCSV(core_file_name, this.item_directory_name)); int stream_file_count = 0; /* * foreach (string file_name in path_to_file_name_map.Select(kvp => kvp.Value).Distinct()) * { * path_to_csv_writer.Add(file_name, new WriteCSV(file_name)); * Console.WriteLine(file_name); * stream_file_count++; * }*/ //Console.WriteLine("stream_file_count: {0}", stream_file_count); create_header_row ( path_to_int_map, path_to_flat_map, path_to_node_map, path_to_csv_writer[core_file_name].Table, true, false, false ); dynamic all_cases_rows; if (this.is_offline_mode) { all_cases_rows = all_cases; } else { all_cases_rows = all_cases.rows; } foreach (System.Dynamic.ExpandoObject case_row in all_cases_rows) { IDictionary <string, object> case_doc; if (this.is_offline_mode) { case_doc = case_row as IDictionary <string, object>; } else { case_doc = ((IDictionary <string, object>)case_row)["doc"] as IDictionary <string, object>; } //IDictionary<string, object> case_doc = ((IDictionary<string, object>)case_row)["doc"] as IDictionary<string, object>; //IDictionary<string, object> case_doc = case_row as IDictionary<string, object>; if (case_doc["_id"].ToString().StartsWith("_design", StringComparison.InvariantCultureIgnoreCase)) { continue; } System.Data.DataRow row = path_to_csv_writer[core_file_name].Table.NewRow(); string mmria_case_id = case_doc["_id"].ToString(); row["_id"] = mmria_case_id; List <string> ordered_column_list = get_core_element_list(); foreach (string path in ordered_column_list) { if ( !path_to_node_map.ContainsKey(path) || !row.Table.Columns.Contains(convert_path_to_field_name(path)) || path_to_node_map[path].type.ToLower() == "app" || path_to_node_map[path].type.ToLower() == "form" || path_to_node_map[path].type.ToLower() == "group" ) { continue; } //System.Console.WriteLine("path {0}", path); dynamic val = get_value(case_doc as IDictionary <string, object>, path); /* * if (path_to_int_map[path].ToString("X") == "41") * { * System.Console.Write("pause"); * } */ try { switch (path_to_node_map [path].type.ToLower()) { case "number": if (val != null && (!string.IsNullOrWhiteSpace(val.ToString()))) { row [convert_path_to_field_name(path)] = val; } break; case "list": if ( path_to_node_map [path].is_multiselect != null && path_to_node_map [path].is_multiselect == true ) { IList <object> temp = val as IList <object>; if (temp != null && temp.Count > 0) { row [convert_path_to_field_name(path)] = string.Join("|", temp); } } else { if (val != null) { row [convert_path_to_field_name(path)] = val; } } break; //case "date": case "datetime": case "time": if (val != null) { row [convert_path_to_field_name(path)] = val; } break; default: if (val != null) { row [convert_path_to_field_name(path)] = val; } break; } } catch (Exception ex) { } } path_to_csv_writer[core_file_name].Table.Rows.Add(row); } Dictionary <string, string> int_to_path_map = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); foreach (KeyValuePair <string, int> ptn in path_to_int_map) { //int_to_path_map.Add(ptn.Value.ToString("X"), ptn.Key); string key = convert_path_to_field_name(ptn.Key); if (int_to_path_map.ContainsKey(key)) { int_to_path_map.Add("_" + ptn.Value.ToString("X"), ptn.Key); } else { int_to_path_map.Add(key, ptn.Key); } } WriteCSV mapping_document = new WriteCSV("core_field_mapping.csv", this.item_directory_name); System.Data.DataColumn column = null; column = new System.Data.DataColumn("file_name", typeof(string)); mapping_document.Table.Columns.Add(column); column = new System.Data.DataColumn("mmria_path", typeof(string)); mapping_document.Table.Columns.Add(column); column = new System.Data.DataColumn("mmria_prompt", typeof(string)); mapping_document.Table.Columns.Add(column); column = new System.Data.DataColumn("column_name", typeof(string)); mapping_document.Table.Columns.Add(column); foreach (KeyValuePair <string, WriteCSV> kvp in path_to_csv_writer) { foreach (System.Data.DataColumn table_column in kvp.Value.Table.Columns) { System.Data.DataRow mapping_row = mapping_document.Table.NewRow(); mapping_row["file_name"] = kvp.Key; if (int_to_path_map.ContainsKey(table_column.ColumnName)) { string path = int_to_path_map [table_column.ColumnName]; mapping_row["mmria_path"] = path; mapping_row ["mmria_prompt"] = path_to_node_map [path].prompt; } else { switch (table_column.ColumnName) { case "_record_index": case "_parent_index": default: mapping_row["mmria_path"] = table_column.ColumnName; break; } } mapping_row["column_name"] = table_column.ColumnName; mapping_document.Table.Rows.Add(mapping_row); } kvp.Value.WriteToStream(); } mapping_document.WriteToStream(); mmria.server.util.cFolderCompressor folder_compressor = new mmria.server.util.cFolderCompressor(); folder_compressor.Compress ( System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings["export_directory"], this.item_file_name), null, // string password System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings["export_directory"], this.item_directory_name) ); var get_item_curl = new cURL("GET", null, Program.config_couchdb_url + "/export_queue/" + this.item_id, null, this.user_name, this.password); string responseFromServer = get_item_curl.execute(); export_queue_item export_queue_item = Newtonsoft.Json.JsonConvert.DeserializeObject <export_queue_item> (responseFromServer); export_queue_item.status = "Download"; Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore; string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(export_queue_item, settings); var set_item_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + export_queue_item._id, object_string, this.user_name, this.password); responseFromServer = set_item_curl.execute(); Console.WriteLine("{0} Export Finished.", System.DateTime.Now); }