partial void Deletemid_lookup(mid_lookup instance);
partial void Insertmid_lookup(mid_lookup instance);
partial void Updatemid_lookup(mid_lookup instance);
static void push_whq_data(Boolean stage = false, Boolean dev = false, int months = 5, Boolean reset = false, Boolean new_model_only = false)
{
var d = new WHQtoGR.App_Code.whq2grDataContext();
GR gr = null;
ids = new gr_ids(stage);
string whq_system = "";
if (stage)
{
gr = new GR(ConfigurationManager.AppSettings["whq_stage_api_key"], ConfigurationManager.AppSettings["gr_stage_server"], false);
// gr.MeasApi = "http://*****:*****@"SELECT
CAST( DATE_FORMAT(`gma_directorreport`.`directorreport_endDate`, '%Y-%m-01') as DATE) AS `Date`,
`gma_locationtrans`.`location_name` AS `Area`,
`gma_organizationdata`.`organization_countryCode` AS `Country`,
`gma_measurementlbl`.`measurementlbl_name` AS `LMI`,
`gma_strategylbl`.`strategylbl_name` AS `MCC`,
SUM( `gma_directordata`.`directordata_value` ) AS `Value`
FROM
`gma_directordata`
JOIN `gma_directorreport` ON `gma_directordata`.`directorreport_id` = `gma_directorreport`.`directorreport_id`
JOIN `gma_measurement` ON `gma_directordata`.`measurement_id` = `gma_measurement`.`measurement_id`
JOIN `gma_newmeasurementdetails` ON `gma_measurement`.`measurement_details_id` = `gma_newmeasurementdetails`.`measurement_details_id`
JOIN `gma_measurementlbl` ON `gma_measurement`.`measurementlbl_id` = `gma_measurementlbl`.`measurementlbl_id`
JOIN `gma_organizationdata` ON `gma_measurement`.`organization_id` = `gma_organizationdata`.`organization_id`
JOIN `gma_strategydata` ON `gma_newmeasurementdetails`.`strategy_id` = `gma_strategydata`.`strategy_id`
JOIN `gma_strategylbl` ON `gma_strategydata`.`strategylbl_id` = `gma_strategylbl`.`strategylbl_id`
JOIN `gma_locationtrans` ON `gma_organizationdata`.`location_id` = `gma_locationtrans`.`location_id`
WHERE
`gma_organizationdata`.`version_id` = 2
AND `gma_strategydata`.`version_id` = 2
AND `gma_locationtrans`.`language_id` = 1
AND `gma_organizationdata`.`organization_leftIndex` + 1 = `gma_organizationdata`.`organization_rightIndex`
AND MONTH(`gma_directorreport`.`directorreport_endDate`)= MONTH(CURDATE() - INTERVAL " + i + @" MONTH )
AND YEAR(`gma_directorreport`.`directorreport_endDate`)= YEAR(CURDATE() - INTERVAL " + i + @" MONTH )
and Not `gma_locationtrans`.`location_name` = 'NONE' and Not `gma_locationtrans`.`location_name` = 'TEST'
And `gma_directordata`.`directordata_value` >0
GROUP BY
`Date`,
`Area`,
`Country`,
`LMI`,
`MCC`";
var dt = new DataTable();
var da = new MySqlDataAdapter(cmd);
da.Fill(dt);
//get all measurements
// Console.WriteLine(dt.Rows.Count);
var rows = dt.AsEnumerable().Where(row => (d.wierd_countries.Where(c => c.whq_code == row["Country"].ToString() && c.min_code == null).Count() == 0) && ids.m.ContainsKey(row["LMI"].ToString()) && (d.mid_lookups.Where(c => (c.country == row["Country"].ToString()) && c.etl && (c.stage == stage)).Count() == 0 || (row["LMI"].ToString().Contains("actualreportingnodes") || row["LMI"].ToString().Contains("actualreportingnodes")))).ToArray();
//Console.WriteLine("before: " + rows.Where(row => row["Country"].ToString() == "EGY").Count().ToString());
//filter out ETL Active rows and only
//now some tests
int delta = rows.Count() / 10;
int count = 0;
var flat_list = new Dictionary<string, double>();
var flat_list_mt = new Dictionary<string, string>(); //The flatlist contains a name/value list of all measurements (from whq), where the name contains ministry_id : measurement_type_id : dimension. This allows me to lookup a value and see if it has changed.
if (rows.Count() > 0)
{
var everything = gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_win_%25" + m_filter).ToList();
everything.AddRange(gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_build_%25" + m_filter).ToList());
everything.AddRange(gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_send_%25" + m_filter).ToList());
everything.AddRange(gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_movements_%25" + m_filter).ToList());
everything.AddRange(gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_nbr_%25" + m_filter).ToList());
everything.AddRange(gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_actual_reporting_nodes" + m_filter).ToList());
everything.AddRange(gr.GetMeasurements("", ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), ((DateTime)rows.First()["Date"]).ToString("yyyy-MM"), "", "", false, "&filters[perm_link]=lmi_local_possible_reporting_nodes" + m_filter).ToList());
foreach (MeasurementType row in everything)
{
foreach (Measurement m in row.measurements)
{
flat_list.Add(m.RelatedEntityId + ":" + row.ID + ":" + m.Dimension, m.Value);
flat_list_mt.Add(m.RelatedEntityId + ":" + row.ID + ":" + m.Dimension, m.ID ) ;
}
}
}
// Console.WriteLine("rows: " + rows.Count().ToString());
// Console.WriteLine("flatlist count: " + flat_list.Count);
int c_changed = 0;
int c_not_changed = 0;
Console.Write("processing data:");
foreach (DataRow row in rows)
{
count++;
if (delta > 0)
{
if (count % delta == 1)
Console.Write("\rprocessing data for " + ((DateTime)row["Date"]).ToString("yyyy-MM") + ": " + ((int)(100 * count / rows.Count())).ToString() + "% complete.");
}
string min_id = (from c in d.mid_lookups where c.country == row["Country"].ToString() && c.stage == stage select c.ministry_id).FirstOrDefault();
if (string.IsNullOrEmpty(min_id))
{
var nms_wierd = (from c in d.wierd_countries where c.whq_code == row["Country"].ToString() select c.min_code);
Entity nms_ministry = null;
if (nms_wierd.Count() > 0 )
{
nms_ministry = gr.GetEntities("ministry", "&filters[owned_by]=all&filters[min_code]=" + nms_wierd.First() + "&fields=min_code", 0, 0, ref totalPage).FirstOrDefault();
}
if (nms_ministry == null)
{
nms_ministry = gr.GetEntities("ministry", "&filters[owned_by]=all&filters[iso_country:relationship][iso3_code]=" + row["Country"].ToString() + "&filters[area:relationship][area_code]=" + ids.areas[row["Area"].ToString()] + "&fields=min_code", 0, 0, ref totalPage).FirstOrDefault();
}
min_id = nms_ministry.ID;
App_Code.mid_lookup insert = new App_Code.mid_lookup();
insert.ministry_id = min_id;
insert.stage = stage;
insert.country = row["Country"].ToString();
d.mid_lookups.InsertOnSubmit(insert);
d.SubmitChanges();
}
Boolean changed=true;
string perm_link = row["LMI"].ToString().ToLower().Replace("fa_", "lmi_local_").Replace("fr_", "lmi_local_").Replace("movements", "lmi_local_movements").Replace("actualreportingnodes", "lmi_local_actual_reporting_nodes").Replace("possiblereportingnodes", "lmi_local_possible_reporting_nodes").Replace("nbr_staff_reporting", "lmi_local_nbr_staff_reporting").Replace("nbr_nonstaff_reporting", "lmi_local_nbr_nonstaff_reporting");
if (perm_link.StartsWith("lmi_local"))
{
MeasurementType this_mt = ids.nms.Where(c => c.PermLink == perm_link).FirstOrDefault();
if (this_mt == null)
{
//get the measurement
// var cache = new WHQtoGR.App_Code.meas_cacheDataContext();
this_mt = gr.GetMeasurements("", "3000-01", "3000-01", "", "", false, "&filters[perm_link]=" + perm_link).FirstOrDefault();
if (this_mt != null)
{
// Console.WriteLine(perm_link);
ids.nms.Add(this_mt);
}
}
if (this_mt != null)
{
string key = min_id + ":" + this_mt.ID + ":" + ids.mcc_code[row["MCC"].ToString()] + "_gma" ;
if (flat_list.ContainsKey(key ))
{
changed = (flat_list[key] != System.Convert.ToDouble(row["Value"]));
flat_list.Remove(key);
flat_list_mt.Remove(key); //the remaining items need to be deleted, as they exist in GR but not in WHQ
}
if (changed)
{
this_mt.addMeasurement(min_id, ((DateTime)row["Date"]).ToString("yyyy-MM"), System.Convert.ToDouble(row["Value"]), ids.mcc_code[row["MCC"].ToString()] + "_gma");
c_changed++;
}
else c_not_changed++;
//Console.WriteLine(row["Country"].ToString() + " " + row["MCC"].ToString().ToLower() + " " + perm_link + " : " + System.Convert.ToDouble(row["Value"]));
}
else
{
Console.WriteLine("could not create measurement_type");
}
}
else
Console.WriteLine("could not find " + perm_link);
}
//Console.WriteLine("changed: " + c_changed);
//Console.WriteLine("not_changed: " + c_not_changed);
//Console.WriteLine("updloading " + ids.nms.Sum(c => c.measurements.Count()) + " into New Msasurement Model");
//TODO: DELETE ALL MEAUREMENTS THAT ARE LEFT IN THE FLATLIST
Console.WriteLine("extra (to be delete): " + flat_list_mt.Count());
foreach(var m in flat_list_mt)
{
gr.DeleteMeasuerment(m.Value);
//need to get measurement_id into flatlist key
}
da.Dispose();
dt.Dispose();
}
//dt.Dump();
}
Console.WriteLine("Finished generating measurmenets");
Console.WriteLine("updloading " + ids.nms.Sum(c => c.measurements.Count()) + " into New Msasurement Model");
var tasks = new List<Task>();
foreach (MeasurementType mt in ids.nms)
{
tasks.Add(System.Threading.Tasks.Task.Factory.StartNew(() =>
{
gr.AddUpdateMeasurement(mt, true, 100);
}));
}
System.Threading.Tasks.Task.WaitAll(tasks.ToArray<Task>());
Console.WriteLine("Finished ");
// Console.ReadKey();
}