public void TestCompletenessReportGenerationSmard()
{
MiniProfiler profiler = MiniProfiler.StartNew(nameof(TestCompletenessReportGenerationSmard));
IList <CompletenessReport> crlist = new List <CompletenessReport>();
foreach (string boFile in Directory.GetFiles("Energiemenge/completeness", "50hz_prognose*.json"))
{
using (MiniProfiler.Current.Step($"Processing file {boFile}"))
{
JObject json;
using (StreamReader r = new StreamReader(boFile))
{
string jsonString = r.ReadToEnd();
json = JsonConvert.DeserializeObject <JObject>(jsonString);
}
Energiemenge em = (Energiemenge)BoMapper.MapObject(json, LenientParsing.Strict);
CompletenessReport cr = em.GetCompletenessReport();
crlist.Add(cr);
if (boFile.Contains("onshore.json"))
{
Assert.IsNotNull(cr.UserProperties);
Assert.AreEqual <string>("yippi yippi yeah", cr.UserProperties["meineUp0"].Value <string>());
Assert.AreEqual <string>("krawall und remmidemmi", cr.UserProperties["meineUp1"].Value <string>());
}
}
}
string resultString = JsonConvert.SerializeObject(crlist, new StringEnumConverter());
profiler.Stop();
System.Diagnostics.Debug.WriteLine($"Profiler results: {profiler.RenderPlainText()}");
}
public void TestCompletenessReportMitGapToCsv()
{
CompletenessReport cr = new CompletenessReport()
{
LokationsId = "DE12345",
Coverage = 0.87M, // 87%
wertermittlungsverfahren = BO4E.ENUM.Wertermittlungsverfahren.PROGNOSE,
ReferenceTimeFrame = new BO4E.COM.Zeitraum()
{
Startdatum = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 3, 1, 0, 0, 0, DateTimeKind.Utc)
},
};
cr.Values = new List <CompletenessReport.BasicVerbrauch>
{
new CompletenessReport.BasicVerbrauch()
{
Wert = 17,
Startdatum = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc)
},
new CompletenessReport.BasicVerbrauch()
{
Wert = 21,
Startdatum = new DateTime(2019, 1, 7, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 1, 8, 0, 0, 0, DateTimeKind.Utc)
},
new CompletenessReport.BasicVerbrauch()
{
Wert = 35,
Startdatum = new DateTime(2019, 1, 12, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 1, 13, 0, 0, 0, DateTimeKind.Utc)
}
};
cr.Gaps = new List <CompletenessReport.BasicVerbrauch>
{
new CompletenessReport.BasicVerbrauch()
{
Wert = 0,
Startdatum = new DateTime(2017, 1, 1, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2017, 1, 2, 0, 0, 0, DateTimeKind.Utc)
},
new CompletenessReport.BasicVerbrauch()
{
Wert = 0,
Startdatum = new DateTime(2017, 1, 7, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2017, 1, 8, 0, 0, 0, DateTimeKind.Utc)
},
new CompletenessReport.BasicVerbrauch()
{
Wert = 0,
Startdatum = new DateTime(2017, 1, 12, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2017, 1, 13, 0, 0, 0, DateTimeKind.Utc)
}
};
var multiplicityResult = cr.ToCsv(lineTerminator: Environment.NewLine);
Assert.AreEqual(2 + cr.Values.Count + cr.Gaps.Count, new List <string>(multiplicityResult.Split(Environment.NewLine)).Count);
}
public void TestCompletenessReportGenerationSomeCustomer()
{
var files = Directory.GetFiles("Energiemenge/completeness", "somecustomer*.json");
Assert.AreEqual(5, files.Count()); // this is just to make sure the files haven't moved
foreach (string boFile in files)
{
JObject json;
using (StreamReader r = new StreamReader(boFile))
{
string jsonString = r.ReadToEnd();
json = JsonConvert.DeserializeObject <JObject>(jsonString);
}
Energiemenge em = (Energiemenge)BoMapper.MapObject((JObject)json["input"], LenientParsing.Strict);
CompletenessReport cr;
if (boFile.EndsWith("somecustomer1.json"))
{
cr = em.GetCompletenessReport();
Assert.AreEqual((decimal)0.9601, Math.Round(cr.Coverage.Value, 4));
Assert.AreEqual("4-5-6-7", cr.Obiskennzahl);
Assert.AreEqual(Wertermittlungsverfahren.MESSUNG, cr.wertermittlungsverfahren);
Assert.AreEqual(Mengeneinheit.KWH, cr.Einheit);
Assert.AreEqual("DEXXX", cr.LokationsId);
//Assert.AreEqual(15, cr.values[0].wert);
//Assert.AreEqual(TestEnergiemengeExtension.GERMAN_APRIL_2018.Start, cr.values[0].startdatum);
string resultString = JsonConvert.SerializeObject(cr, new StringEnumConverter());
Assert.IsNotNull(cr.Gaps);
Assert.AreEqual(1, cr.Gaps.Count);
Assert.AreEqual(new DateTime(2018, 4, 1, 1, 45, 0, DateTimeKind.Utc), cr.Gaps.First().Startdatum);
Assert.AreEqual(new DateTime(2018, 4, 2, 6, 30, 0, DateTimeKind.Utc), cr.Gaps.First().Enddatum);
}
else if (boFile.EndsWith("somecustomer2.json"))
{
foreach (var combi in em.GetWevObisMeCombinations())
{
cr = em.GetCompletenessReport(TestEnergiemengeExtension.GERMAN_APRIL_2018, combi.Item1, combi.Item2, combi.Item3);
string resultString = JsonConvert.SerializeObject(cr, new StringEnumConverter());
CompletenessReport cr2 = em.GetCompletenessReport(new CompletenessReport.CompletenessReportConfiguration
{
Einheit = combi.Item3,
Obis = combi.Item2,
Wertermittlungsverfahren = combi.Item1,
ReferenceTimeFrame = new BO4E.COM.Zeitraum
{
Startdatum = TestEnergiemengeExtension.GERMAN_APRIL_2018.Start,
Enddatum = TestEnergiemengeExtension.GERMAN_APRIL_2018.End
}
});
//Assert.AreEqual(cr, cr2, "calling report with configuration instead of loose parameters doesn't work.");
}
}
else if (boFile.EndsWith("somecustomer3.json"))
{
}
else if (boFile.EndsWith("somecustomer4.json"))
{
}
}
}
public void TestNullableCoverage()
{
Energiemenge em1 = new Energiemenge()
{
LokationsId = "DE123456789DieseEmhatkeineVerbräuche",
LokationsTyp = Lokationstyp.MeLo,
Energieverbrauch = new List <Verbrauch>() //empty list
};
CompletenessReport cr1 = em1.GetCompletenessReport();
Assert.IsNotNull(cr1);
Assert.IsNull(cr1.Coverage);
JsonConvert.SerializeObject(cr1); // must _not_ throw exception
Energiemenge em2 = new Energiemenge()
{
LokationsId = "54321012345DieseEmhatkeineVerbräuche",
LokationsTyp = Lokationstyp.MeLo,
Energieverbrauch = new List <Verbrauch>() //empty list
};
CompletenessReport cr2 = em2.GetCompletenessReport(CHRISTMAS_2018, Wertermittlungsverfahren.MESSUNG, "1-2-3-4", Mengeneinheit.KUBIKMETER);
Assert.IsNotNull(cr2);
Assert.IsNotNull(cr2.Coverage); // not null because no values but configuration given
Assert.AreEqual(0.0M, cr2.Coverage);
JsonConvert.SerializeObject(cr2); // must _not_ throw exception
CompletenessReport cr3 = em2.GetCompletenessReport(CHRISTMAS_2018);
Assert.IsNotNull(cr3);
Assert.IsNotNull(cr3.Coverage);
Assert.AreEqual(0.0M, cr3.Coverage);
JsonConvert.SerializeObject(cr3); // must _not_ throw exception
}
public void TestRounding()
{
string boFile = Directory.GetFiles("Energiemenge/completeness", "gas_januar_2018.json").First();
JObject json;
using (StreamReader r = new StreamReader(boFile))
{
string jsonString = r.ReadToEnd();
json = JsonConvert.DeserializeObject <JObject>(jsonString);
}
Energiemenge em = (Energiemenge)BoMapper.MapObject(JObject.FromObject(json["input"]), LenientParsing.Strict);
CompletenessReport cr = em.GetCompletenessReport(new TimeRange()
{
Start = new DateTime(2017, 12, 31, 23, 0, 0, 0, DateTimeKind.Utc),
End = new DateTime(2018, 1, 31, 23, 0, 0, 0, DateTimeKind.Utc)
});
Assert.AreEqual(1.0M, cr.Coverage.Value);
Assert.AreEqual(0, cr.Gaps.Count());
var dailies = em.GetDailyCompletenessReports(new TimeRange()
{
Start = new DateTime(2017, 12, 31, 23, 0, 0, 0, DateTimeKind.Utc),
End = new DateTime(2018, 1, 2, 23, 0, 0, 0, DateTimeKind.Utc)
});
foreach (var crDaily in dailies)
{
Assert.AreEqual(1.0M, crDaily.Value.Coverage.Value, $"error in slice {crDaily.Key}");
}
Assert.AreEqual(1.0M, cr.Coverage.Value);
}
public void TestCompletenessReportHashing()
{
BO4E.StaticLogger.Logger = new Microsoft.Extensions.Logging.Debug.DebugLogger("Testlogger", (log, level) => { return(true); });
CompletenessReport cr = new CompletenessReport()
{
LokationsId = "56789012345",
Coverage = 0.9M,
Einheit = BO4E.ENUM.Mengeneinheit.MWH,
wertermittlungsverfahren = BO4E.ENUM.Wertermittlungsverfahren.MESSUNG,
UserProperties = new Dictionary <string, JToken>()
{
{ "anlage", "5012345678" },
{ "profil", "123456" }
}
};
Assert.IsTrue(cr.IsValid());
var conf = new AnonymizerConfiguration();
conf.SetOption(DataCategory.POD, AnonymizerApproach.HASH);
conf.SetOption(DataCategory.USER_PROPERTIES, AnonymizerApproach.HASH);
CompletenessReport hashedReport;
using (Anonymizer anonymizer = new Anonymizer(conf))
{
hashedReport = anonymizer.ApplyOperations <CompletenessReport>(cr);
}
Assert.IsNotNull(hashedReport);
Assert.AreNotEqual(cr.LokationsId, hashedReport.LokationsId);
Assert.IsTrue(Marktlokation.ValidateId(hashedReport.LokationsId));
Assert.IsNotNull(cr.UserProperties["anlage"]);
Assert.AreNotEqual(cr.UserProperties["anlage"].Value <string>(), hashedReport.UserProperties["anlage"].Value <string>());
Assert.IsNotNull(cr.UserProperties["profil"]);
Assert.AreNotEqual(cr.UserProperties["profil"].Value <string>(), hashedReport.UserProperties["profil"].Value <string>());
conf.hashingSalt = "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=";
CompletenessReport saltedReport;
using (Anonymizer anonymizer = new Anonymizer(conf))
{
saltedReport = anonymizer.ApplyOperations <CompletenessReport>(cr);
}
Assert.IsNotNull(saltedReport.LokationsId);
Assert.AreNotEqual(cr.LokationsId, saltedReport.LokationsId);
Assert.AreNotEqual(hashedReport.LokationsId, saltedReport.LokationsId);
Assert.IsTrue(Anonymizer.IsHashedKey(hashedReport.LokationsId));
Assert.IsTrue(Anonymizer.IsHashedKey(saltedReport.LokationsId));
}
public void TestCompletenessReportToCsvExceptions()
{
CompletenessReport cr = new CompletenessReport()
{
LokationsId = "DE12345",
Coverage = 0.87M, // 87%
wertermittlungsverfahren = BO4E.ENUM.Wertermittlungsverfahren.PROGNOSE,
ReferenceTimeFrame = new BO4E.COM.Zeitraum()
{
Startdatum = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 3, 1, 0, 0, 0, DateTimeKind.Utc)
},
};
// reihenfolge
List <Dictionary <string, string> > reihenfolge = new List <Dictionary <string, string> >
{
new Dictionary <string, string>()
{
["LokationsId"] = "messlokationsId"
},
new Dictionary <string, string>()
{
["Coverage"] = "Newcoverage"
},
new Dictionary <string, string>()
{
["Zeitraum.Startdatum"] = "time.startdatum"
},
new Dictionary <string, string>()
{
["Zeitraum.Enddatum"] = "time.enddatum"
},
new Dictionary <string, string>()
{
["Wert"] = null
},
new Dictionary <string, string>()
{
["Startdatum"] = "V.startdatum"
},
new Dictionary <string, string>()
{
["Enddatum"] = "V.enddatum"
},
null
};
string newResult = string.Empty;
Assert.ThrowsException <ArgumentNullException>(() => cr.ToCsv(';', true, Environment.NewLine, reihenfolge));
Assert.AreEqual(newResult, "");
// reihenfolge
List <Dictionary <string, string> > reihenfolge2 = new List <Dictionary <string, string> >
{
new Dictionary <string, string>()
{
["lokationsId"] = "messlokationsId"
},
new Dictionary <string, string>()
{
["coverage"] = "Newcoverage"
},
new Dictionary <string, string>()
{
["Zeitraum.startdatum"] = "time.startdatum"
},
new Dictionary <string, string>()
{
["Zeitraum.enddatum"] = "time.enddatum"
},
new Dictionary <string, string>()
{
["wert"] = "V.wert"
},
new Dictionary <string, string>()
{
["startdatum"] = "V.startdatum"
},
new Dictionary <string, string>()
{
["enddatum"] = "V.enddatum"
},
new Dictionary <string, string>()
{
["asdasd"] = "000"
}
};
Assert.ThrowsException <ArgumentException>(() => cr.ToCsv(';', true, Environment.NewLine, reihenfolge2));
Assert.AreEqual(newResult, "");
}
public void TestCompletenessReportToCsv()
{
CompletenessReport cr = new CompletenessReport()
{
LokationsId = "DE12345",
Coverage = 0.87M, // 87%
wertermittlungsverfahren = BO4E.ENUM.Wertermittlungsverfahren.PROGNOSE,
ReferenceTimeFrame = new BO4E.COM.Zeitraum()
{
Startdatum = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 3, 1, 0, 0, 0, DateTimeKind.Utc)
},
};
string result = cr.ToCsv(';', true, Environment.NewLine);
var lines = new List <string>(result.Split(Environment.NewLine));
Assert.AreEqual(2, lines.Count);
// reihenfolge
List <Dictionary <string, string> > reihenfolge = new List <Dictionary <string, string> >
{
new Dictionary <string, string>()
{
["LokationsId"] = "messlokationsId"
},
new Dictionary <string, string>()
{
["Coverage"] = "Newcoverage"
},
new Dictionary <string, string>()
{
["Zeitraum.Startdatum"] = "time.startdatum"
},
new Dictionary <string, string>()
{
["Zeitraum.Enddatum"] = "time.enddatum"
}
};
//string JSONdata = "{'completenessZfa':[{'lokationsId':'lokationsId'},{'coverage':'coverage'},{'Zeitraum.einheit':'einheit'},{'Zeitraum.dauer':'dauer'},{'Zeitraum.startdatum':'startdatum'},{'Zeitraum.enddatum':'enddatum'},{'obiskennzahl':'obiskennzahl'},{'einheit':'einheit'},{'wertermittlungsverfahren':'wertermittlungsverfahren'},{'startdatum':'Verbrauch.startdatum'},{'enddatum':'Verbrauch.enddatum'},{'wert':'Verbrauch.wert'},{'headerLine':'1'}]}";
//var alldata = JsonConvert.DeserializeObject<Dictionary<string, List<Dictionary<string, string>>>>(JSONdata);
//List<Dictionary<string, string>> reihenfolge = alldata["completenessZfa"];
var Newresult = cr.ToCsv(';', true, Environment.NewLine, reihenfolge);
lines = new List <string>(Newresult.Split(Environment.NewLine));
Assert.AreEqual(2, lines.Count);
string decimalSeparator = Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator;
Assert.AreEqual("DE12345;0" + decimalSeparator + "87;2019-01-01T00:00:00Z;2019-03-01T00:00:00Z;", lines[1]);
var commaResult = cr.ToCsv(',', lineTerminator: Environment.NewLine, reihenfolge: reihenfolge);
Assert.AreEqual("DE12345,0" + decimalSeparator + "87,2019-01-01T00:00:00Z,2019-03-01T00:00:00Z,", commaResult.Split(Environment.NewLine)[1]);
var dpunktResult = cr.ToCsv(':', lineTerminator: Environment.NewLine, reihenfolge: reihenfolge);
Assert.AreEqual("DE12345:0" + decimalSeparator + "87:\"2019-01-01T00:00:00Z\":\"2019-03-01T00:00:00Z\":", dpunktResult.Split(Environment.NewLine)[1]);
cr.Values = new List <CompletenessReport.BasicVerbrauch>
{
new CompletenessReport.BasicVerbrauch()
{
Wert = 17,
Startdatum = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc)
},
new CompletenessReport.BasicVerbrauch()
{
Wert = 21,
Startdatum = new DateTime(2019, 1, 7, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 1, 8, 0, 0, 0, DateTimeKind.Utc)
},
new CompletenessReport.BasicVerbrauch()
{
Wert = 35,
Startdatum = new DateTime(2019, 1, 12, 0, 0, 0, DateTimeKind.Utc),
Enddatum = new DateTime(2019, 1, 13, 0, 0, 0, DateTimeKind.Utc)
}
};
reihenfolge.Add(new Dictionary <string, string>()
{
["Wert"] = "V.wert"
});
reihenfolge.Add(new Dictionary <string, string>()
{
["Startdatum"] = "V.startdatum"
});
reihenfolge.Add(new Dictionary <string, string>()
{
["Enddatum"] = "V.enddatum"
});
var multiplicityResult = cr.ToCsv(lineTerminator: Environment.NewLine, reihenfolge: reihenfolge);
Assert.AreEqual(2 + cr.Values.Count, new List <string>(multiplicityResult.Split(Environment.NewLine)).Count);
}
/// <summary>
/// Generate a <see cref="CompletenessReport"/> for the given parameters.
/// </summary>
/// <param name="em">Energiemenge</param>
/// <param name="reference">reference time frame</param>
/// <param name="wev">Wertermittlungsverfahren</param>
/// <param name="obiskennzahl">OBIS Kennzahl</param>
/// <param name="einheit">Mengeneinheit</param>
/// <returns>the completeness report</returns>
public static CompletenessReport GetCompletenessReport(this BO4E.BO.Energiemenge em, ITimeRange reference, Wertermittlungsverfahren wev, string obiskennzahl, Mengeneinheit einheit)
{
CompletenessReport result;
using (MiniProfiler.Current.Step("create completeness report skeleton + find the coverage"))
{
result = new CompletenessReport
{
LokationsId = em.LokationsId,
Einheit = einheit,
Coverage = GetCoverage(em, reference, wev, obiskennzahl, einheit),
wertermittlungsverfahren = wev,
Obiskennzahl = obiskennzahl,
ReferenceTimeFrame = new Zeitraum
{
Startdatum = DateTime.SpecifyKind(reference.Start, DateTimeKind.Utc),
Enddatum = DateTime.SpecifyKind(reference.End, DateTimeKind.Utc)
},
};
}
if (em.Energieverbrauch != null && em.Energieverbrauch.Count > 0)
{
/*using (MiniProfiler.Current.Step("populating time slices of/with missing/null values"))
* {
* result.values = em.GetMissingTimeRanges(reference, wev, obis, einheit)
* .Select(mtr => new CompletenessReport.BasicVerbrauch
* {
* startdatum = DateTime.SpecifyKind(mtr.Start, DateTimeKind.Utc),
* enddatum = DateTime.SpecifyKind(mtr.End, DateTimeKind.Utc),
* wert = null
* }).ToList<CompletenessReport.BasicVerbrauch>();
* }
* using (MiniProfiler.Current.Step("populating time slices existing values"))
* {
* result.values.AddRange(
* em.energieverbrauch
* //.AsParallel<Verbrauch>()
* .Where(v => v.obiskennzahl == obis && v.einheit == einheit && v.wertermittlungsverfahren == wev)
* .Select(v => new CompletenessReport.BasicVerbrauch
* {
* startdatum = DateTime.SpecifyKind(v.startdatum, DateTimeKind.Utc),
* enddatum = DateTime.SpecifyKind(v.enddatum, DateTimeKind.Utc),
* wert = v.wert
* })
* .ToList<CompletenessReport.BasicVerbrauch>());
* }*/
using (MiniProfiler.Current.Step("Setting aggregated gaps"))
{
var nonNullValues = new TimePeriodCollection(em.Energieverbrauch.Select(v => new TimeRange(v.Startdatum, v.Enddatum)));
ITimeRange limits;
if (result.ReferenceTimeFrame != null && result.ReferenceTimeFrame.Startdatum.HasValue && result.ReferenceTimeFrame.Enddatum.HasValue)
{
limits = new TimeRange(result.ReferenceTimeFrame.Startdatum.Value, result.ReferenceTimeFrame.Enddatum.Value);
}
else
{
limits = null;
}
var gaps = (new TimeGapCalculator <TimeRange>()).GetGaps(nonNullValues, limits: limits);
result.Gaps = gaps.Select(gap => new CompletenessReport.BasicVerbrauch()
{
Startdatum = gap.Start,
Enddatum = gap.End,
Wert = null
}).ToList();
}
/*using (MiniProfiler.Current.Step("sorting result"))
* {
* result.values.Sort(new BasicVerbrauchDateTimeComparer());
* }*/
if (em.IsPure(checkUserProperties: true))
{
try
{
foreach (var kvp in em.Energieverbrauch.Where(v => v.UserProperties != null).SelectMany(v => v.UserProperties))
{
if (result.UserProperties == null)
{
result.UserProperties = new Dictionary <string, JToken>();
}
if (!result.UserProperties.ContainsKey(kvp.Key))
{
result.UserProperties.Add(kvp.Key, kvp.Value);
}
}
}
catch (InvalidOperationException)
{
// ok, there's no Verbrauch with user properties.
}
}
}
/*else
* {
* result.coverage = null;
* result._errorMessage = "energieverbrauch is empty";
* }*/
return(result);
}
