public async Task <SynteticModel> BuildModel(string sessionId)
{
SessionId = sessionId;
_statistics = PersonStatistics.CreateRoot(SessionId, _ageQuants, _numberOfSamplesToTake, 30);
var totalPeople = await Import();
Console.WriteLine($"Import done (numberOfPersons={totalPeople}). Starting AfterImport");
_statistics.AfterImport(_persons, _persons);
Console.WriteLine("AfterImport done. Settings distributions");
_statistics.SetDistribution(true);
var model = new SynteticModel()
{
AgeQuantLevel = _ageQuants,
Statistics = _statistics
};
var confirmed = _persons.Values.Count(x => x.Confirmed);
Console.WriteLine($"Found {confirmed} people");
return(model);
}
private static void BuildDistributionModels(SynteticModel model)
{
BuildDistributionModel(model.Statistics, -1);
foreach (var statforAgeQuant in model.Statistics.StatisticsByAgeQuants.Values)
{
BuildDistributionModel(statforAgeQuant, statforAgeQuant.AgeQuantLevel);
}
}
public static double GetCorrelation(PersonWithMetadata person, SynteticModel model, string valueString1, string valueString2)
{
if (model.Statistics.GetClosestStatisticByAgeQuant(person)?.Correlations == null)
{
return(model.Statistics.Correlations.GetValue(valueString1, valueString2).Value);
}
return(model.Statistics.GetClosestStatisticByAgeQuant(person).Correlations.GetValue(valueString1, valueString2).Value);
}
static void Main(string[] args)
{
try
{
var dataProvider = GetDataProvider();
var builder = new SynteticModelBuilder(dataProvider);
var sessionId = Guid.NewGuid().ToString();
Console.WriteLine("SessionId: " + sessionId);
SynteticModel model = null;
try
{
model = builder.BuildModel(sessionId).GetAwaiter().GetResult();
}
catch (Exception e)
{
Console.WriteLine("In program:" + e.ToString());
Console.ReadKey();
}
var filename = @"C:\temp\pregmodel_" + sessionId + @"_withoutcorr.json";
File.WriteAllText(filename, JsonConvert.SerializeObject(model));
Console.WriteLine("Temporarily written model withouth correlations to file " + filename);
Console.WriteLine("Now calculating correlations");
builder.SetNewDataProvder(GetDataProvider());
builder.BuildCorrelationMatrix(model, sessionId).GetAwaiter().GetResult();
filename = @"C:\temp\pregmodel_" + sessionId + @".json";
var asJson = JsonConvert.SerializeObject(model);
File.WriteAllText(filename, asJson);
Console.WriteLine("Written model to file " + filename);
Console.WriteLine();
Console.WriteLine("Click any button to exit");
Console.ReadKey();
}
catch (Exception e)
{
Console.WriteLine("In program:" + e.ToString());
Console.ReadKey();
throw;
}
}
private RelationshipModel2 GetRelationshipModel2(SynteticModel model, FamilyModel familyModel)
{
var pParentNinExistsGivenHaveParentNin = new Func <PersonWithMetadata, double>(person => model.Statistics.RelationshipStatistics.ParentninExists_GivenHaveParentNin.TrueRatioPercent);
var pParentSameSexGivenHaveValidParentNin = new Func <PersonWithMetadata, double>(person => model.Statistics.GetClosestStatisticByAgeQuant(person).RelationshipStatistics.ParentSameSex_GivenHaveValidParentNin.TrueRatioPercent);
var hasFatherHasMotherHasSpouse = GetHasFatherHasMotherHasSpouse(familyModel);
var getAgeDifferenceInYearsToParent = new Func <PersonWithMetadata, int>(person => (int)model.Statistics.RelationshipStatistics.AgeDifferentalInYearsToParent.GenerateSamples(1).First());
var numberOfChildrenGivenHaveChildren = new Func <PersonWithMetadata, int>(person => model.Statistics.GetClosestStatisticByAgeQuant(person, statistics => statistics.RelationshipStatistics?.NumberOfChildren_GivenHaveChildren.NumberOfSamples > 0 && statistics.RelationshipStatistics.NumberOfChildren_GivenHaveChildren.CanGenerateSamples).RelationshipStatistics.GetNumberOfChildrenSample());
var getSpouseAgeDifferential = new Func <PersonWithMetadata, int>(person => (int)model.Statistics.GetClosestStatisticByAgeQuant(person, statistics => statistics.RelationshipStatistics?.SpouseAgeDifferentialInYears.TotalCount > 0 && statistics.RelationshipStatistics.SpouseAgeDifferentialInYears.CanGenerateSamples).RelationshipStatistics.SpouseAgeDifferentialInYears.GenerateSamples(1).First() * (person.Randy.Hit(50) ? -1 : 1)); //SpouseAgeDifferentialInYears is defined as always positive ..
var pSpouseExistsGivenHasSpouse = new Func <PersonWithMetadata, double>(person => model.Statistics.GetClosestStatisticByAgeQuant(person).RelationshipStatistics.SpouseExists_GivenHasSpouse.TrueRatioPercent);
var pSpouseSameSexGivenHasValidSpouse = new Func <PersonWithMetadata, double>(person => model.Statistics.GetClosestStatisticByAgeQuant(person).RelationshipStatistics.SpouseSameSex_GivenHasValidSpouse.TrueRatioPercent);
var pSpouseDuplexGivenHasValidSpouse = new Func <PersonWithMetadata, double>(person => model.Statistics.GetClosestStatisticByAgeQuant(person).RelationshipStatistics.SpouseDuplex_GivenHasValidSpouse.TrueRatioPercent);
var pParentsAreMarriedGivenhaveMotherAndFather = new Func <PersonWithMetadata, double>(person => model.Statistics.RelationshipStatistics.ParentsAreMarried.TrueRatioPercent);
return(new RelationshipModel2(this, pParentNinExistsGivenHaveParentNin, pParentSameSexGivenHaveValidParentNin, hasFatherHasMotherHasSpouse, getAgeDifferenceInYearsToParent, numberOfChildrenGivenHaveChildren, getSpouseAgeDifferential, pSpouseExistsGivenHasSpouse,
pSpouseSameSexGivenHasValidSpouse, pSpouseDuplexGivenHasValidSpouse, pParentsAreMarriedGivenhaveMotherAndFather));
}
public async Task <bool> BuildCorrelationMatrix(SynteticModel model, string sessionId)
{
SessionId = sessionId;
model.Statistics.Correlations = new CorrelationMatrix(model.Statistics.CorrelationFactory.CorrelationFactoryElements.Count, model.Statistics.CorrelationFactory.CorrelationFactoryElements.Values.Select(x => x.Key).ToArray());
int c = 0;
Console.WriteLine("Staring import");
while (_dataProvider.HasMore())
{
var p = await _dataProvider.GetNextPerson();
if (p == null)
{
continue;
}
bool birthdayWasFoundFromNin;
var ageQuant = PersonStatistics.CalculateAgeQuant(p, _ageQuants, out birthdayWasFoundFromNin);
if (ageQuant == PersonStatistics.InvalidQuant)
{
continue;
}
model.Statistics.Correlations.Update(p, ageQuant, model.Statistics.Statistics);
if (c++ % 10000 == 0)
{
Console.WriteLine("Done calculating correlations for " + c);
}
}
model.Statistics.Correlations.Closure(model.Statistics.CorrelationFactory);
return(true);
}
public PregEngine(SynteticModel model)
{
_model = model;
IdControl = new IdentifierDublicateControl();
}
public PregEngine(PregEngineConfiguration configuration)
{
_model = JsonConvert.DeserializeObject <SynteticModel>(File.ReadAllText(configuration.ModelFilePath));
IdControl = new IdentifierDublicateControl();
_logFilePath = configuration.LogfilePath;
}
public SigmaBuilder(SynteticModel model)
{
_model = model;
}
public SynteticDataBuilderV1(SynteticModel model, IPushPregData pusherPregData, IdentifierDublicateControl idControl)
{
_pusherPregData = pusherPregData;
_idControl = idControl;
Model = model;
}
