private void Experiment3() { if (finished == 0) { if (DeleteFiles()) { finished++; Experiment3PushBsm(); return; } } var resourse = "b4.b4-131"; LogWrite("Starting experiment 3 on resourse " + resourse); steps = 1; //max 20 int[] sizes = { 1, 3, 4, 5, 6 }; workflows = sizes.Length; /* * foreach (var pair in bsmDescriptionReader.InputFiles) * { * var val = ConfigurationManager.AppSettings[pair.Key]; * _api.BindFile(pair.Key, val); * } */ IEnumerable <string> lines = new List <string>(); IEnumerable <string> overlines = new List <string>(); if (File.Exists(ConfigurationManager.AppSettings["HistoryFile"])) { lines = File.ReadAllLines(ConfigurationManager.AppSettings["HistoryFile"]).Where(line => line.Contains("bsm") && line.Contains(resourse)); overlines = File.ReadAllLines(ConfigurationManager.AppSettings["HistoryFile"].Replace("model_coef", "over")); } var runs = new List <RunRecord>(); //.Where(l=>l.Contains(@"{""ForecastSize"": ""1""}")) foreach (var line in lines) { var rows = line.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries); var rows2 = overlines.Where(l => l.Contains(rows[0])).FirstOrDefault().Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries); //Select(s => double.Parse(s)).ToArray(); var rx = new Regex(@"""ForecastSize"": ""([^,]+)"""); var match = rx.Match(rows[11]); if (match.Success) { var rec = new RunRecord { ComputationTime = double.Parse(rows[7].Replace(".", ",")) }; rec.OverheadTime = TimeSpan.Parse(rows2[7]).TotalSeconds; // База пакетов rec.OverheadTime += TimeSpan.Parse(rows2[8]).TotalSeconds; // Оценка ресурсов rec.OverheadTime += TimeSpan.Parse(rows2[9]).TotalSeconds; // T_Scheduler rec.OverheadTime += TimeSpan.Parse(rows2[10]).TotalSeconds; // Коммуникация rec.OverheadTime += TimeSpan.Parse(rows2[11]).TotalSeconds; // T_InputFilesCopy rec.OverheadTime += TimeSpan.Parse(rows2[12]).TotalSeconds; // T_OutputFilesCopy rec.RunContext.Add("ForecastSize", double.Parse(match.Groups[1].Value)); runs.Add(rec); } } PerformanceModel model = new BsmModel(); var sp = ParametersOptimizer.UpdateServiceComputationParameters(new Dictionary <string, double>(), runs, model); var spp = ParametersOptimizer.UpdateServiceOverheadParameters(new Dictionary <string, double>(), runs); foreach (var p in sp) { LogWrite(String.Format("{0}: {1}", p.Key, p.Value.ToString("0.0000"))); } /* * foreach (var p in spp) * LogWrite(String.Format("{0}: {1}", p.Key, p.Value.ToString("0.0000"))); */ var bsmDescriptionReader = new DescriptionReader("bsm.wf"); _api.UploadFiles(bsmDescriptionReader.InputFiles); int i = 0; foreach (var size in sizes) { LogWrite(String.Format("ForecastSize: {0}", size)); var runRecord = new RunRecord(); runRecord.RunContext.Add("ForecastSize", (double)size); var compuTime = model.GetComputationTime(sp, runRecord.RunContext, runRecord.ExecutionParams); var compuError = model.GetComputationErrorRelative(sp, runRecord.RunContext, runRecord.ExecutionParams); LogWrite(String.Format("Calculation time: {0}+/-{1}", compuTime.ToString("0.000"), compuError.ToString("0.000"))); var overheadTime = model.GetOverheadTime(spp, runRecord.RunContext, runRecord.ExecutionParams); var overheadError = model.GetOverheadError(spp, runRecord.RunContext, runRecord.ExecutionParams); LogWrite(String.Format("Overhead time: {0}+/-{1}", overheadTime.ToString("0.000"), overheadError.ToString("0.000"))); StringBuilder sb = new StringBuilder(); //int stepDiffer = 0; for (int step = 1; step <= steps; step++) { //if (stepDiffer >= sizes.Length) stepDiffer = 0; sb.Append("[Resource = " + '"' + resourse + '"' + "]\n"); sb.Append("step MaskedFullBSM_" + step + " runs bsm \n (\n inMeasurement = measurementFile,\n inHirlam = hirlam" + size + ",\n swan = swanFile,\n inBSH = BSHFile,\n useAssimilation = true,\n useSWAN = true,\n useBSH = true,\n useOldProject = false,\n useMask = false,\n startCalcDate = \"09/01/2007 12:00:00\",\n inAssFields = assFields,\n inProject = projects,\n controlPoints = inControlPoints,\n deleteDirs = true,\n ForecastSize = " + size + " \n)\n"); //stepDiffer++; } _api.Script = sb.ToString(); jobMonitor = _api.CreateMonitor(); jobMonitor.UpdatePeriod = 1000 * 5; jobMonitor.Active += JobMonitorOnActive; jobMonitor.Finished += (sender, jobDecriptionEventArgs) => { JobMonitorStandardFinished(sender, jobDecriptionEventArgs); CopyRows(new string[] { "ForecastSize" }); }; LogWrite(DateTime.Now.ToLongTimeString() + " " + steps + " bsm pushed: " + i + "/" + workflows + "; id:" + jobMonitor.JobId); //LogWrite(_api.Script.Substring(0, 70)); started = DateTime.Now; jobMonitor.Run(); Statuses.Add(jobMonitor.JobId.ToString(), "Pushed"); CalculationTimes.Add(jobMonitor.JobId.ToString(), compuTime.ToString("0.000")); CalculationTimesErr.Add(jobMonitor.JobId.ToString(), compuError.ToString("0.000")); OverheadTimes.Add(jobMonitor.JobId.ToString(), overheadTime.ToString("0.000")); OverheadTimesErr.Add(jobMonitor.JobId.ToString(), overheadError.ToString("0.000")); LogWrite(DateTime.Now.ToLongTimeString() + " " + jobMonitor.JobId + " pushed"); i++; } }
private void Experiment3() { if (finished == 0) if (DeleteFiles()) { finished++; Experiment3PushBsm(); return; } var resourse = "b4.b4-131"; LogWrite("Starting experiment 3 on resourse "+resourse); steps = 1; //max 20 int[] sizes = { 1,3,4,5,6 }; workflows = sizes.Length; /* foreach (var pair in bsmDescriptionReader.InputFiles) { var val = ConfigurationManager.AppSettings[pair.Key]; _api.BindFile(pair.Key, val); } */ IEnumerable<string> lines = new List<string>(); IEnumerable<string> overlines = new List<string>(); if (File.Exists(ConfigurationManager.AppSettings["HistoryFile"])) { lines = File.ReadAllLines(ConfigurationManager.AppSettings["HistoryFile"]).Where(line => line.Contains("bsm") && line.Contains(resourse)); overlines = File.ReadAllLines(ConfigurationManager.AppSettings["HistoryFile"].Replace("model_coef", "over")); } var runs = new List<RunRecord>(); //.Where(l=>l.Contains(@"{""ForecastSize"": ""1""}")) foreach (var line in lines) { var rows = line.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries); var rows2 = overlines.Where(l => l.Contains(rows[0])).FirstOrDefault().Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries); //Select(s => double.Parse(s)).ToArray(); var rx = new Regex(@"""ForecastSize"": ""([^,]+)"""); var match = rx.Match(rows[11]); if (match.Success) { var rec = new RunRecord { ComputationTime = double.Parse(rows[7].Replace(".", ",")) }; rec.OverheadTime = TimeSpan.Parse(rows2[7]).TotalSeconds; // База пакетов rec.OverheadTime += TimeSpan.Parse(rows2[8]).TotalSeconds; // Оценка ресурсов rec.OverheadTime += TimeSpan.Parse(rows2[9]).TotalSeconds; // T_Scheduler rec.OverheadTime += TimeSpan.Parse(rows2[10]).TotalSeconds; // Коммуникация rec.OverheadTime += TimeSpan.Parse(rows2[11]).TotalSeconds; // T_InputFilesCopy rec.OverheadTime += TimeSpan.Parse(rows2[12]).TotalSeconds; // T_OutputFilesCopy rec.RunContext.Add("ForecastSize", double.Parse(match.Groups[1].Value)); runs.Add(rec); } } PerformanceModel model = new BsmModel(); var sp = ParametersOptimizer.UpdateServiceComputationParameters(new Dictionary<string, double>(), runs, model); var spp = ParametersOptimizer.UpdateServiceOverheadParameters(new Dictionary<string, double>(), runs); foreach (var p in sp) LogWrite(String.Format("{0}: {1}", p.Key, p.Value.ToString("0.0000"))); /* foreach (var p in spp) LogWrite(String.Format("{0}: {1}", p.Key, p.Value.ToString("0.0000"))); */ var bsmDescriptionReader = new DescriptionReader("bsm.wf"); _api.UploadFiles(bsmDescriptionReader.InputFiles); int i = 0; foreach (var size in sizes) { LogWrite(String.Format("ForecastSize: {0}",size)); var runRecord = new RunRecord(); runRecord.RunContext.Add("ForecastSize", (double)size); var compuTime = model.GetComputationTime(sp, runRecord.RunContext, runRecord.ExecutionParams); var compuError = model.GetComputationErrorRelative(sp, runRecord.RunContext, runRecord.ExecutionParams); LogWrite(String.Format("Calculation time: {0}+/-{1}", compuTime.ToString("0.000"), compuError.ToString("0.000"))); var overheadTime = model.GetOverheadTime(spp, runRecord.RunContext, runRecord.ExecutionParams); var overheadError = model.GetOverheadError(spp, runRecord.RunContext, runRecord.ExecutionParams); LogWrite(String.Format("Overhead time: {0}+/-{1}", overheadTime.ToString("0.000"), overheadError.ToString("0.000"))); StringBuilder sb = new StringBuilder(); //int stepDiffer = 0; for (int step = 1; step <= steps; step++){ //if (stepDiffer >= sizes.Length) stepDiffer = 0; sb.Append("[Resource = " + '"' + resourse + '"' + "]\n"); sb.Append("step MaskedFullBSM_"+step+" runs bsm \n (\n inMeasurement = measurementFile,\n inHirlam = hirlam" + size + ",\n swan = swanFile,\n inBSH = BSHFile,\n useAssimilation = true,\n useSWAN = true,\n useBSH = true,\n useOldProject = false,\n useMask = false,\n startCalcDate = \"09/01/2007 12:00:00\",\n inAssFields = assFields,\n inProject = projects,\n controlPoints = inControlPoints,\n deleteDirs = true,\n ForecastSize = " + size + " \n)\n"); //stepDiffer++; } _api.Script = sb.ToString(); jobMonitor = _api.CreateMonitor(); jobMonitor.UpdatePeriod = 1000 * 5; jobMonitor.Active += JobMonitorOnActive; jobMonitor.Finished += (sender, jobDecriptionEventArgs) => { JobMonitorStandardFinished(sender, jobDecriptionEventArgs); CopyRows(new string[]{"ForecastSize"}); }; LogWrite(DateTime.Now.ToLongTimeString() + " " + steps + " bsm pushed: " + i + "/" + workflows + "; id:" + jobMonitor.JobId); //LogWrite(_api.Script.Substring(0, 70)); started = DateTime.Now; jobMonitor.Run(); Statuses.Add(jobMonitor.JobId.ToString(), "Pushed"); CalculationTimes.Add(jobMonitor.JobId.ToString(), compuTime.ToString("0.000")); CalculationTimesErr.Add(jobMonitor.JobId.ToString(), compuError.ToString("0.000")); OverheadTimes.Add(jobMonitor.JobId.ToString(), overheadTime.ToString("0.000")); OverheadTimesErr.Add(jobMonitor.JobId.ToString(), overheadError.ToString("0.000")); LogWrite(DateTime.Now.ToLongTimeString() + " " + jobMonitor.JobId + " pushed"); i++; } }