public void Process_Export_Queue_Item()
{
//System.Console.WriteLine ("{0} check_for_changes_job.Process_Export_Queue_Item: started", System.DateTime.Now);
List <export_queue_item> result = new List <export_queue_item> ();
var get_curl = new cURL("GET", null, Program.config_couchdb_url + "/export_queue/_all_docs?include_docs=true", null, this.user_name, this.password);
string responseFromServer = get_curl.execute();
IDictionary <string, object> response_result = Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject> (responseFromServer) as IDictionary <string, object>;
IList <object> enumerable_rows = response_result ["rows"] as IList <object>;
foreach (IDictionary <string, object> enumerable_item in enumerable_rows)
{
IDictionary <string, object> doc_item = enumerable_item ["doc"] as IDictionary <string, object>;
if (
doc_item ["status"] != null &&
doc_item ["status"].ToString().StartsWith("In Queue...", StringComparison.OrdinalIgnoreCase))
{
export_queue_item item = new export_queue_item();
item._id = doc_item ["_id"].ToString();
item._rev = doc_item ["_rev"].ToString();
item._deleted = doc_item.ContainsKey("_deleted") ? doc_item["_deleted"] as bool?: null;
item.date_created = doc_item ["date_created"] as DateTime?;
item.created_by = doc_item.ContainsKey("created_by") && doc_item ["created_by"] != null ? doc_item ["created_by"].ToString() : null;
item.date_last_updated = doc_item ["date_last_updated"] as DateTime?;
item.last_updated_by = doc_item.ContainsKey("last_updated_by") && doc_item ["last_updated_by"] != null ? doc_item ["last_updated_by"].ToString() : null;
item.file_name = doc_item ["file_name"] != null ? doc_item ["file_name"].ToString() : null;
item.export_type = doc_item ["export_type"] != null ? doc_item ["export_type"].ToString() : null;
item.status = doc_item ["status"] != null ? doc_item ["status"].ToString() : null;
result.Add(item);
}
}
if (result.Count > 0)
{
if (result.Count > 1)
{
var comparer = Comparer <export_queue_item> .Create
(
(x, y) => x.date_created.Value.CompareTo(y.date_created.Value)
);
result.Sort(comparer);
}
export_queue_item item_to_process = result [0];
item_to_process.date_last_updated = new DateTime?();
//item_to_process.last_updated_by = g_uid;
List <string> args = new List <string>();
args.Add("exporter:exporter");
args.Add("user_name:" + this.user_name);
args.Add("password:"******"database_url:" + this.couch_db_url);
args.Add("item_file_name:" + item_to_process.file_name);
args.Add("item_id:" + item_to_process._id);
if (item_to_process.export_type.StartsWith("core csv", StringComparison.OrdinalIgnoreCase))
{
item_to_process.status = "Creating Export...";
item_to_process.last_updated_by = "mmria-server";
item_to_process.date_last_updated = DateTime.Now;
Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings();
settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore;
string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings);
var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password);
responseFromServer = set_curl.execute();
try
{
mmria.server.model.actor.ScheduleInfoMessage scheduleInfo = new actor.ScheduleInfoMessage
(
Configuration["mmria_settings:cron_schedule"],
Configuration["mmria_settings:couchdb_url"],
Configuration["mmria_settings:timer_user_name"],
Configuration["mmria_settings:timer_password"],
Configuration["mmria_settings:export_directory"]
);
mmria.server.util.core_element_exporter core_element_exporter = new mmria.server.util.core_element_exporter(scheduleInfo);
core_element_exporter.Execute(args.ToArray());
}
catch (Exception ex)
{
System.Console.WriteLine(ex);
}
}
else if (item_to_process.export_type.StartsWith("all csv", StringComparison.OrdinalIgnoreCase))
{
item_to_process.status = "Creating Export...";
item_to_process.last_updated_by = "mmria-server";
item_to_process.date_last_updated = DateTime.Now;
Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings();
settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore;
string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings);
var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password);
responseFromServer = set_curl.execute();
mmria.server.model.actor.ScheduleInfoMessage scheduleInfo = new actor.ScheduleInfoMessage
(
Configuration["mmria_settings:cron_schedule"],
Configuration["mmria_settings:couchdb_url"],
Configuration["mmria_settings:timer_user_name"],
Configuration["mmria_settings:timer_password"],
Configuration["mmria_settings:export_directory"]
);
try
{
mmria.server.util.mmrds_exporter mmrds_exporter = new mmria.server.util.mmrds_exporter(scheduleInfo);
mmrds_exporter.Execute(args.ToArray());
}
catch (Exception ex)
{
System.Console.WriteLine(ex);
}
}
else if (item_to_process.export_type.StartsWith("cdc csv", StringComparison.OrdinalIgnoreCase))
{
item_to_process.status = "Creating Export...";
item_to_process.last_updated_by = "mmria-server";
item_to_process.date_last_updated = DateTime.Now;
Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings();
settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore;
string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings);
var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password);
responseFromServer = set_curl.execute();
args.Add("is_cdc_de_identified:true");
try
{
mmria.server.model.actor.ScheduleInfoMessage scheduleInfo = new actor.ScheduleInfoMessage
(
Configuration["mmria_settings:cron_schedule"],
Configuration["mmria_settings:couchdb_url"],
Configuration["mmria_settings:timer_user_name"],
Configuration["mmria_settings:timer_password"],
Configuration["mmria_settings:export_directory"]
);
mmria.server.util.mmrds_exporter mmrds_exporter = new mmria.server.util.mmrds_exporter(scheduleInfo);
mmrds_exporter.Execute(args.ToArray());
}
catch (Exception ex)
{
System.Console.WriteLine(ex);
}
}
}
}
public void Process_Export_Queue_Delete()
{
//System.Console.WriteLine ("{0} check_for_changes_job.Process_Export_Queue_Delete: started", System.DateTime.Now);
List <export_queue_item> result = new List <export_queue_item> ();
var get_curl = new cURL("GET", null, Program.config_couchdb_url + "/export_queue/_all_docs?include_docs=true", null, this.user_name, this.password);
string responseFromServer = get_curl.execute();
IDictionary <string, object> response_result = Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject> (responseFromServer) as IDictionary <string, object>;
IList <object> enumerable_rows = response_result ["rows"] as IList <object>;
foreach (IDictionary <string, object> enumerable_item in enumerable_rows)
{
IDictionary <string, object> doc_item = enumerable_item ["doc"] as IDictionary <string, object>;
if (
doc_item ["status"] != null &&
doc_item ["status"].ToString().StartsWith("Deleted", StringComparison.OrdinalIgnoreCase))
{
export_queue_item item = new export_queue_item();
item._id = doc_item ["_id"].ToString();
item._rev = doc_item ["_rev"].ToString();
item._deleted = doc_item.ContainsKey("_deleted") ? doc_item["_deleted"] as bool?: null;
item.date_created = doc_item ["date_created"] as DateTime?;
item.created_by = doc_item.ContainsKey("created_by") && doc_item ["created_by"] != null ? doc_item ["created_by"].ToString() : null;
item.date_last_updated = doc_item ["date_last_updated"] as DateTime?;
item.last_updated_by = doc_item.ContainsKey("last_updated_by") && doc_item["last_updated_by"] != null ? doc_item ["last_updated_by"].ToString() : null;
item.file_name = doc_item ["file_name"] != null ? doc_item ["file_name"].ToString() : null;
item.export_type = doc_item ["export_type"] != null ? doc_item ["export_type"].ToString() : null;
item.status = doc_item ["status"] != null ? doc_item ["status"].ToString() : null;
result.Add(item);
}
}
if (result.Count > 0)
{
if (result.Count > 1)
{
var comparer = Comparer <export_queue_item> .Create
(
(x, y) => x.date_created.Value.CompareTo(y.date_created.Value)
);
result.Sort(comparer);
}
export_queue_item item_to_process = result [0];
try
{
string item_directory_name = item_to_process.file_name.Substring(0, item_to_process.file_name.LastIndexOf("."));
string export_directory = System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings ["export_directory"], item_directory_name);
try
{
if (System.IO.Directory.Exists(export_directory))
{
System.IO.Directory.Delete(export_directory, true);
}
}
catch (Exception Ex)
{
// do nothing for now
System.Console.WriteLine("check_for_changes_job.Process_Export_Queue_Delete: Unable to Delete Directory {0}", export_directory);
}
string file_path = System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings ["export_directory"], item_to_process.file_name);
try
{
if (System.IO.File.Exists(file_path))
{
System.IO.File.Delete(file_path);
}
}
catch (Exception Ex)
{
// do nothing for now
System.Console.WriteLine("Program.Process_Export_Queue_Delete: Unable to Delete File {0}", file_path);
}
item_to_process.status = "expunged";
item_to_process.last_updated_by = "mmria-server";
Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings();
settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore;
string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(item_to_process, settings);
var set_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + item_to_process._id, object_string, this.user_name, this.password);
responseFromServer = get_curl.execute();
}
catch (Exception ex)
{
// do nothing for now
}
}
}
public void Execute(string[] args)
{
if (args.Length > 1)
{
for (var i = 1; i < args.Length; i++)
{
string arg = args[i];
int index = arg.IndexOf(':');
string val = arg.Substring(index + 1, arg.Length - (index + 1)).Trim(new char[] { '\"' });
if (arg.ToLower().StartsWith("auth_token"))
{
this.auth_token = val;
}
else if (arg.ToLower().StartsWith("user_name"))
{
this.user_name = val;
}
else if (arg.ToLower().StartsWith("password"))
{
this.password = val;
}
else if (arg.ToLower().StartsWith("database_url"))
{
this.database_url = val;
}
else if (arg.ToLower().StartsWith("database"))
{
this.database_path = val;
}
else if (arg.ToLower().StartsWith("item_file_name"))
{
this.item_file_name = val;
this.item_directory_name = this.item_file_name.Substring(0, this.item_file_name.LastIndexOf("."));
}
else if (arg.ToLower().StartsWith("item_id"))
{
this.item_id = val;
}
}
}
string core_file_name = "core_mmria_export.csv";
if (string.IsNullOrWhiteSpace(this.database_url))
{
this.database_url = System.Configuration.ConfigurationManager.AppSettings["couchdb_url"];
if (string.IsNullOrWhiteSpace(this.database_url))
{
System.Console.WriteLine("missing database_url");
System.Console.WriteLine(" form database:[file path]");
System.Console.WriteLine(" example database:http://localhost:5984");
System.Console.WriteLine(" mmria.exe export user_name:user1 password:secret url:http://localhost:12345 database_url:http://localhost:5984");
return;
}
}
if (string.IsNullOrWhiteSpace(this.user_name))
{
System.Console.WriteLine("missing user_name");
System.Console.WriteLine(" form user_name:[user_name]");
System.Console.WriteLine(" example user_name:user1");
System.Console.WriteLine(" mmria.exe export user_name:user1 password:secret url:http://localhost:12345");
return;
}
if (string.IsNullOrWhiteSpace(this.password))
{
System.Console.WriteLine("missing password");
System.Console.WriteLine(" form password:[password]");
System.Console.WriteLine(" example password:secret");
System.Console.WriteLine(" mmria.exe export user_name:user1 password:secret url:http://localhost:12345");
return;
}
string export_directory = System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings["export_directory"], this.item_directory_name);
if (!System.IO.Directory.Exists(export_directory))
{
System.IO.Directory.CreateDirectory(export_directory);
}
string URL = this.database_url + "/mmrds/_all_docs";
string urlParameters = "?include_docs=true";
cURL document_curl = new cURL("GET", null, URL + urlParameters, null, this.user_name, this.password);
dynamic all_cases = Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject>(document_curl.execute());
string metadata_url = this.database_url + "/metadata/2016-06-12T13:49:24.759Z";
cURL metadata_curl = new cURL("GET", null, metadata_url, null, this.user_name, this.password);
mmria.common.metadata.app metadata = Newtonsoft.Json.JsonConvert.DeserializeObject <mmria.common.metadata.app>(metadata_curl.execute());
/*
* foreach (KeyValuePair<string, object> kvp in all_cases)
* {
* System.Console.WriteLine(kvp.Key);
* }
*/
System.Collections.Generic.Dictionary <string, int> path_to_int_map = new Dictionary <string, int>();
System.Collections.Generic.Dictionary <string, string> path_to_file_name_map = new Dictionary <string, string>();
System.Collections.Generic.Dictionary <string, mmria.common.metadata.node> path_to_node_map = new Dictionary <string, mmria.common.metadata.node>();
System.Collections.Generic.Dictionary <string, string> path_to_grid_map = new Dictionary <string, string>();
System.Collections.Generic.Dictionary <string, string> path_to_multi_form_map = new Dictionary <string, string>();
System.Collections.Generic.Dictionary <string, string> grid_to_multi_form_map = new Dictionary <string, string>();
System.Collections.Generic.HashSet <string> path_to_flat_map = new System.Collections.Generic.HashSet <string>();
System.Collections.Generic.Dictionary <string, WriteCSV> path_to_csv_writer = new Dictionary <string, WriteCSV>();
generate_path_map
(metadata, "", core_file_name, "",
path_to_int_map,
path_to_file_name_map,
path_to_node_map,
path_to_grid_map,
path_to_multi_form_map,
false,
grid_to_multi_form_map,
false,
path_to_flat_map
);
System.Collections.Generic.HashSet <string> flat_grid_set = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
System.Collections.Generic.HashSet <string> mutiform_grid_set = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair <string, string> kvp in path_to_grid_map)
{
if (grid_to_multi_form_map.ContainsKey(kvp.Key))
{
mutiform_grid_set.Add(kvp.Value);
}
}
foreach (KeyValuePair <string, string> kvp in path_to_grid_map)
{
if (
!mutiform_grid_set.Contains(kvp.Value) &&
!flat_grid_set.Contains(kvp.Value)
)
{
flat_grid_set.Add(kvp.Value);
}
}
/*
* System.Collections.Generic.HashSet<string> mutiform_set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
* foreach (KeyValuePair<string, string> kvp in path_to_multi_form_map)
* {
* if (!mutiform_set.Contains(kvp.Value))
* {
* mutiform_set.Add(kvp.Value);
* }
* }*/
path_to_csv_writer.Add(core_file_name, new WriteCSV(core_file_name, this.item_directory_name));
int stream_file_count = 0;
/*
* foreach (string file_name in path_to_file_name_map.Select(kvp => kvp.Value).Distinct())
* {
* path_to_csv_writer.Add(file_name, new WriteCSV(file_name));
* Console.WriteLine(file_name);
* stream_file_count++;
* }*/
//Console.WriteLine("stream_file_count: {0}", stream_file_count);
create_header_row
(
path_to_int_map,
path_to_flat_map,
path_to_node_map,
path_to_csv_writer[core_file_name].Table,
true,
false,
false
);
dynamic all_cases_rows;
if (this.is_offline_mode)
{
all_cases_rows = all_cases;
}
else
{
all_cases_rows = all_cases.rows;
}
foreach (System.Dynamic.ExpandoObject case_row in all_cases_rows)
{
IDictionary <string, object> case_doc;
if (this.is_offline_mode)
{
case_doc = case_row as IDictionary <string, object>;
}
else
{
case_doc = ((IDictionary <string, object>)case_row)["doc"] as IDictionary <string, object>;
}
//IDictionary<string, object> case_doc = ((IDictionary<string, object>)case_row)["doc"] as IDictionary<string, object>;
//IDictionary<string, object> case_doc = case_row as IDictionary<string, object>;
if (case_doc["_id"].ToString().StartsWith("_design", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
System.Data.DataRow row = path_to_csv_writer[core_file_name].Table.NewRow();
string mmria_case_id = case_doc["_id"].ToString();
row["_id"] = mmria_case_id;
List <string> ordered_column_list = get_core_element_list();
foreach (string path in ordered_column_list)
{
if (
!path_to_node_map.ContainsKey(path) ||
!row.Table.Columns.Contains(convert_path_to_field_name(path)) ||
path_to_node_map[path].type.ToLower() == "app" ||
path_to_node_map[path].type.ToLower() == "form" ||
path_to_node_map[path].type.ToLower() == "group"
)
{
continue;
}
//System.Console.WriteLine("path {0}", path);
dynamic val = get_value(case_doc as IDictionary <string, object>, path);
/*
* if (path_to_int_map[path].ToString("X") == "41")
* {
* System.Console.Write("pause");
* }
*/
try
{
switch (path_to_node_map [path].type.ToLower())
{
case "number":
if (val != null && (!string.IsNullOrWhiteSpace(val.ToString())))
{
row [convert_path_to_field_name(path)] = val;
}
break;
case "list":
if
(
path_to_node_map [path].is_multiselect != null &&
path_to_node_map [path].is_multiselect == true
)
{
IList <object> temp = val as IList <object>;
if (temp != null && temp.Count > 0)
{
row [convert_path_to_field_name(path)] = string.Join("|", temp);
}
}
else
{
if (val != null)
{
row [convert_path_to_field_name(path)] = val;
}
}
break;
//case "date":
case "datetime":
case "time":
if (val != null)
{
row [convert_path_to_field_name(path)] = val;
}
break;
default:
if (val != null)
{
row [convert_path_to_field_name(path)] = val;
}
break;
}
}
catch (Exception ex)
{
}
}
path_to_csv_writer[core_file_name].Table.Rows.Add(row);
}
Dictionary <string, string> int_to_path_map = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair <string, int> ptn in path_to_int_map)
{
//int_to_path_map.Add(ptn.Value.ToString("X"), ptn.Key);
string key = convert_path_to_field_name(ptn.Key);
if (int_to_path_map.ContainsKey(key))
{
int_to_path_map.Add("_" + ptn.Value.ToString("X"), ptn.Key);
}
else
{
int_to_path_map.Add(key, ptn.Key);
}
}
WriteCSV mapping_document = new WriteCSV("core_field_mapping.csv", this.item_directory_name);
System.Data.DataColumn column = null;
column = new System.Data.DataColumn("file_name", typeof(string));
mapping_document.Table.Columns.Add(column);
column = new System.Data.DataColumn("mmria_path", typeof(string));
mapping_document.Table.Columns.Add(column);
column = new System.Data.DataColumn("mmria_prompt", typeof(string));
mapping_document.Table.Columns.Add(column);
column = new System.Data.DataColumn("column_name", typeof(string));
mapping_document.Table.Columns.Add(column);
foreach (KeyValuePair <string, WriteCSV> kvp in path_to_csv_writer)
{
foreach (System.Data.DataColumn table_column in kvp.Value.Table.Columns)
{
System.Data.DataRow mapping_row = mapping_document.Table.NewRow();
mapping_row["file_name"] = kvp.Key;
if (int_to_path_map.ContainsKey(table_column.ColumnName))
{
string path = int_to_path_map [table_column.ColumnName];
mapping_row["mmria_path"] = path;
mapping_row ["mmria_prompt"] = path_to_node_map [path].prompt;
}
else
{
switch (table_column.ColumnName)
{
case "_record_index":
case "_parent_index":
default:
mapping_row["mmria_path"] = table_column.ColumnName;
break;
}
}
mapping_row["column_name"] = table_column.ColumnName;
mapping_document.Table.Rows.Add(mapping_row);
}
kvp.Value.WriteToStream();
}
mapping_document.WriteToStream();
mmria.server.util.cFolderCompressor folder_compressor = new mmria.server.util.cFolderCompressor();
folder_compressor.Compress
(
System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings["export_directory"], this.item_file_name),
null, // string password
System.IO.Path.Combine(System.Configuration.ConfigurationManager.AppSettings["export_directory"], this.item_directory_name)
);
var get_item_curl = new cURL("GET", null, Program.config_couchdb_url + "/export_queue/" + this.item_id, null, this.user_name, this.password);
string responseFromServer = get_item_curl.execute();
export_queue_item export_queue_item = Newtonsoft.Json.JsonConvert.DeserializeObject <export_queue_item> (responseFromServer);
export_queue_item.status = "Download";
Newtonsoft.Json.JsonSerializerSettings settings = new Newtonsoft.Json.JsonSerializerSettings();
settings.NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore;
string object_string = Newtonsoft.Json.JsonConvert.SerializeObject(export_queue_item, settings);
var set_item_curl = new cURL("PUT", null, Program.config_couchdb_url + "/export_queue/" + export_queue_item._id, object_string, this.user_name, this.password);
responseFromServer = set_item_curl.execute();
Console.WriteLine("{0} Export Finished.", System.DateTime.Now);
}
