/// <summary>
/// Calculate statistic. Person should be filled with person career
/// </summary>
/// <param name="person">Person. Should be filled with person career</param>
/// <returns></returns>
public PersonStatistics Calculate(Person person)
{
PersonCareer personCareerItem = GetPersonCareerItemOfTheGame(person);
if (personCareerItem == null || game.ProtocolRecord == null)
{
return(null);
}
IEnumerable <ProtocolRecordPersonInfo> protocolPersonInfo =
game.ProtocolRecord.Select(pr => new ProtocolRecordPersonInfo(person, pr));
var personStats = new PersonStatistics()
{
teamId = personCareerItem.teamId,
personId = personCareerItem.personId
};
CalculatePlayerOnThePitchStats(person, personCareerItem, protocolPersonInfo, ref personStats);
if (personStats.Games > 0)
{
CalculateGoalkeeperAdditionalStats(person, personCareerItem, protocolPersonInfo, ref personStats);
}
return(personStats.Games > 0 ? personStats : null);
}
public async Task <SynteticModel> BuildModel(string sessionId)
{
SessionId = sessionId;
_statistics = PersonStatistics.CreateRoot(SessionId, _ageQuants, _numberOfSamplesToTake, 30);
var totalPeople = await Import();
Console.WriteLine($"Import done (numberOfPersons={totalPeople}). Starting AfterImport");
_statistics.AfterImport(_persons, _persons);
Console.WriteLine("AfterImport done. Settings distributions");
_statistics.SetDistribution(true);
var model = new SynteticModel()
{
AgeQuantLevel = _ageQuants,
Statistics = _statistics
};
var confirmed = _persons.Values.Count(x => x.Confirmed);
Console.WriteLine($"Found {confirmed} people");
return(model);
}
public static PersonStatistics LoadFromFile(string filePath)
{
var statistics = new PersonStatistics();
using (var reader = new StreamReader(filePath))
{
var separators = new[] { ',' };
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (string.IsNullOrWhiteSpace(line))
{
continue; //--malformed line
}
var lParts = line.Split(separators, StringSplitOptions.RemoveEmptyEntries);
if (lParts.Length != 4)
{
continue; //--malformed line
}
var person = new Person
{
Age = int.Parse(lParts[0]),
Gender = lParts[1],
MaritalStatus = lParts[2],
District = int.Parse(lParts[3])
};
statistics.Persons.Add(person);
}
}
return(statistics);
}
private static void BuildDistributionModel(PersonStatistics statistics, int?ageQuant)
{
BuildDistributionModels(statistics, ageQuant);
BuildDistributionModels(statistics.RelationshipStatistics, ageQuant);
BuildDistributionModels(statistics.SivilstatusStatistics, ageQuant);
BuildDistributionModels(statistics.RegstatusStatistics, ageQuant);
BuildDistributionModels(statistics.NameStatistics, ageQuant);
BuildDistributionModels(statistics.AdressStatistics, ageQuant);
}
public int SavePersonStatistics(int tourneyId, IEnumerable <PersonStatistics> personStatistics)
{
if (personStatistics == null)
{
return(-1);
}
List <PersonStatistics> saveRecords = personStatistics.Where(r => r.tourneyId == tourneyId).ToList();
List <PersonStatistics> dbRecords = GetPersonsStatistics(tourneyId, saveRecords.Select(r => r.personId)).ToList();
var insertRecords = new List <PersonStatistics>();
IEnumerable <PersonStatistics> removeRecords = new PersonStatistics[] { };
int removeCount = dbRecords.Count - saveRecords.Count;
if (removeCount > 0)
{
removeRecords = dbRecords.Skip(saveRecords.Count);
Context.RemoveRange(removeRecords);
}
for (int i = 0; i < saveRecords.Count; i++)
{
PersonStatistics dbRecord = dbRecords.ElementAtOrDefault(i);
if (dbRecord == null)
{
dbRecord = new PersonStatistics();
insertRecords.Add(dbRecord);
}
dbRecord.Assists = saveRecords[i].Assists;
dbRecord.CustomIntValue = saveRecords[i].CustomIntValue;
dbRecord.Games = saveRecords[i].Games;
dbRecord.Goals = saveRecords[i].Goals;
dbRecord.personId = saveRecords[i].personId;
dbRecord.Reds = saveRecords[i].Reds;
dbRecord.Substitutes = saveRecords[i].Substitutes;
dbRecord.teamId = saveRecords[i].teamId;
dbRecord.tourneyId = saveRecords[i].tourneyId;
dbRecord.Yellows = saveRecords[i].Yellows;
}
Context.PersonStatistics.AddRange(insertRecords);
try
{
return(Context.SaveChanges());
}
catch (Exception ex)
{
//TODO: Add logging
throw;
}
}
private void ComputeStatsItems(PersonStatistics incrementalItem, PersonStatistics addItem)
{
incrementalItem.Assists += addItem.Assists;
incrementalItem.Games += addItem.Games;
incrementalItem.Goals += addItem.Goals;
incrementalItem.Reds += addItem.Reds;
incrementalItem.Substitutes += addItem.Substitutes;
incrementalItem.Yellows += addItem.Yellows;
incrementalItem.CustomIntValue += addItem.CustomIntValue;
incrementalItem.teamId = addItem.teamId;
incrementalItem.tourneyId = tourney.Id;
}
private static void SetSigma(PersonStatistics personStats, CorrelationMatrix correlationMatrix)
{
var allBooleans = BooleanStatistic.GetAll(personStats);
allBooleans = ExcludeNonvariantStatistcs(allBooleans);
var sigma = new MultivariateBinaryGenerator();
sigma.BuildCoverianceMatrix(allBooleans, correlationMatrix, RIntegration.GetSigma);
personStats.BinaryGenerator = sigma;
}
private void AddPersonStats(IList <PersonStatistics> personStatistics, PersonStatistics personStatsItem)
{
PersonStatistics existingPersonStatsItem = personStatistics.SingleOrDefault(ps => ps.personId == personStatsItem.personId);
personStatsItem.tourneyId = tourney.Id;
if (existingPersonStatsItem == null)
{
personStatistics.Add(personStatsItem);
return;
}
ComputeStatsItems(existingPersonStatsItem, personStatsItem);
}
public static List <BooleanStatistic> GetAll(PersonStatistics ps)
{
var result = new List <BooleanStatistic>();
var properties = ps.GetType().GetProperties();
foreach (var property in properties)
{
if (property.PropertyType == typeof(BooleanStatistic))
{
result.Add(((BooleanStatistic)property.GetValue(ps, null)));
}
}
return(result);
}
public static void CalculateAdditionalInfo(PersonStatistics pStatistics)
{
if (pStatistics.AmountOfAttempts > _minNumberForAdvStat - 1)
{
var x = new double[pStatistics.AmountOfAttempts];
for (var i = 0; i < x.Length; i++)
{
x[i] = i + 1;
}
pStatistics.AdditionalInfo =
LinearApproximation.LinearApproximation.GetLinearApproximation(x,
pStatistics.Results.ConvertAll(Convert.ToDouble));
}
else
{
pStatistics.AdditionalInfo = null;
}
}
private void Calculate(ref IList <PersonStatistics> personStats)
{
foreach (Round round in tourney.Round.Where(r => r.Game != null))
{
foreach (Game game in round.Game)
{
var gameStatsCalculator = new PersonGameStatsCalculator(game);
foreach (Person person in persons)
{
PersonStatistics personGameStats = gameStatsCalculator.Calculate(person);
if (personGameStats != null)
{
AddPersonStats(personStats, personGameStats);
}
}
}
}
}
public async Task <bool> BuildCorrelationMatrix(SynteticModel model, string sessionId)
{
SessionId = sessionId;
model.Statistics.Correlations = new CorrelationMatrix(model.Statistics.CorrelationFactory.CorrelationFactoryElements.Count, model.Statistics.CorrelationFactory.CorrelationFactoryElements.Values.Select(x => x.Key).ToArray());
int c = 0;
Console.WriteLine("Staring import");
while (_dataProvider.HasMore())
{
var p = await _dataProvider.GetNextPerson();
if (p == null)
{
continue;
}
bool birthdayWasFoundFromNin;
var ageQuant = PersonStatistics.CalculateAgeQuant(p, _ageQuants, out birthdayWasFoundFromNin);
if (ageQuant == PersonStatistics.InvalidQuant)
{
continue;
}
model.Statistics.Correlations.Update(p, ageQuant, model.Statistics.Statistics);
if (c++ % 10000 == 0)
{
Console.WriteLine("Done calculating correlations for " + c);
}
}
model.Statistics.Correlations.Closure(model.Statistics.CorrelationFactory);
return(true);
}
public static PersonStatisticsViewModel ToViewModel(this PersonStatistics personStatistics)
{
if (personStatistics == null)
{
return(null);
}
return(new PersonStatisticsViewModel()
{
id = personStatistics.Id,
assists = personStatistics.Assists,
customIntValue = personStatistics.CustomIntValue,
games = personStatistics.Games,
goals = personStatistics.Goals,
personId = personStatistics.personId,
reds = personStatistics.Reds,
substitutes = personStatistics.Substitutes,
teamId = personStatistics.teamId,
tourneyId = personStatistics.tourneyId,
yellows = personStatistics.Yellows,
team = personStatistics.team.ToViewModel(),
tourney = personStatistics.tourney.ToViewModel()
});
}
public PersonStatistics GetStatictics()
{
IQueryable <DAL.Person> persons = this.All().Where(p => p.AnswerType.IsAsked);
var countDictionary = persons.GroupBy(p => new { Description = p.AnswerType.Description, Id = p.AnswerTypeId })
.Select(g => new { Key = g.Key.Description, Value = g.Count() })
.OrderBy(g => g.Key)
.ToDictionary(r => r.Key, r => r.Value);
IQueryable <decimal> answers = persons.Where(p => p.AnswerConverted.HasValue)
.Select(p => p.AnswerConverted.Value);
var statistics = new PersonStatistics()
{
CountDictionary = countDictionary,
Average = answers.Average(),
Min = answers.Min(),
Max = answers.Max(),
TotalCount = this.All().Count(),
Disabled = persons.Count(p => p.IsAccountDisabled),
};
return(statistics);
}
internal void ComparePersonStatistics(PersonStatistics expected, PersonStatistics actual)
{
Assert.Equal(expected.AmountOfAttempts, actual.AmountOfAttempts);
Assert.Equal(expected.Results, actual.Results);
}
private void CalculateGoalkeeperAdditionalStats(Person person, PersonCareer personCareerItem, IEnumerable <ProtocolRecordPersonInfo> protocolPersonInfo, ref PersonStatistics personStatistics)
{
if (person.roleId != PersonRoleId.rrGoalkeeper)
{
return;
}
ProtocolRecordPersonInfo recordIn = protocolPersonInfo.FirstOrDefault(pr => pr.IsStartMain || pr.IsSubstitutionIn);
if (recordIn == null)
{
return;
}
// The value will be always present because pr.IsStartMain || pr.IsSubstitutionIn check it
int minuteIn = recordIn.ProtocolRecord.Minute.Value;
ProtocolRecordPersonInfo recordOut = protocolPersonInfo.FirstOrDefault(pr => pr.IsSubstitutionOut);
int minuteOut = recordOut != null ? recordOut.ProtocolRecord.Minute.Value : GameDefinitions.MaxGameDurationInMinutes;
IEnumerable <ProtocolRecordInfo> goalsAgainstProtocol =
protocolPersonInfo.Where(pr => pr.ProtocolRecord.teamId != personCareerItem.teamId)
.Select(pr => new ProtocolRecordInfo(pr.ProtocolRecord))
.Where(pr => pr.IsGoal);
personStatistics.CustomIntValue = (short)goalsAgainstProtocol.Count(gp => gp.ProtocolRecord.Minute >= minuteIn && gp.ProtocolRecord.Minute <= minuteOut);
}
public PersonStatistics GetStatictics()
{
PersonStatistics statistics = this.provider.GetStatictics();
return(statistics);
}
private void CalculatePlayerOnThePitchStats(Person person, PersonCareer personCareerItem, IEnumerable <ProtocolRecordPersonInfo> protocolPersonInfo, ref PersonStatistics personStatistics)
{
IEnumerable <ProtocolRecordPersonInfo> playerOnPitchProtocol =
protocolPersonInfo.Where(pr => pr.ProtocolRecord.teamId == personCareerItem.teamId &&
(pr.ProtocolRecord.personId == personCareerItem.personId ||
pr.ProtocolRecord.CustomIntValue == personCareerItem.personId));
if (Guard.IsEmptyIEnumerable(playerOnPitchProtocol))
{
return;
}
foreach (ProtocolRecordPersonInfo prInfo in playerOnPitchProtocol)
{
if (prInfo.IsGoal && person.roleId.HasValue)
{
//if (PersonRoleGroupId.rgTeamPitchPlayer.Contains(person.roleId.Value))
//{
personStatistics.Goals++;
//}
//else if (person.roleId.Value == PersonRoleId.rrGoalkeeper)
//{
//personStatistics.CustomIntValue = (short)(personStatistics.CustomIntValue.HasValue ? personStatistics.CustomIntValue.Value + 1 : 1);
//}
}
else if (prInfo.IsAssist)
{
personStatistics.Assists++;
}
else if (prInfo.IsStartMain)
{
personStatistics.Games++;
}
else if (prInfo.IsSubstitutionIn)
{
personStatistics.Games++;
personStatistics.Substitutes++;
}
else if (prInfo.IsYellowCard)
{
personStatistics.Yellows++;
}
else if (prInfo.IsRedCard)
{
personStatistics.Reds++;
}
}
}
public PersonStatistics GetStatictics()
{
PersonStatistics statistics = this.repository.GetStatictics();
return(statistics);
}
public void FromImport(Person p, int ageQuant)
{
NumberOfAdressObjects.Update(p.Addresses?.Length ?? 0);
if (p.Addresses == null)
{
return;
}
var adress = GetCurrent(p.Addresses);
if (CommonFunctions.HasDnummer(p.NIN))
{
GivenHasDnummer_KommuneNr.Update(adress.St);
}
var hasAdr = IsNotNullOrEmptyOrWhitespace(adress.AddressLine1) || IsNotNullOrEmptyOrWhitespace(adress.AddressLine2) || IsNotNullOrEmptyOrWhitespace(adress.AddressLine3);
HasAdressLine12or3.Update(hasAdr ? 1 : 0);
if (hasAdr)
{
HasAdressLine1_WhenHasAdressLine12or3.Update(IsNotNullOrEmptyOrWhitespace(adress.AddressLine1) ? 1 : 0);
HasAdressLine2_WhenHasAdressLine12or3.Update(IsNotNullOrEmptyOrWhitespace(adress.AddressLine2) ? 1 : 0);
HasAdressLine3_WhenHasAdressLine12or3.Update(IsNotNullOrEmptyOrWhitespace(adress.AddressLine3) ? 1 : 0);
}
HasAppartmentNumber.Update(IsNotNullOrEmptyOrWhitespace(adress.ApartmentNumber) ? 1 : 0);
HasCoAdress.Update(IsNotNullOrEmptyOrWhitespace(adress.CoAddress) ? 1 : 0);
HasCountry.Update(IsNotNullOrEmptyOrWhitespace(adress.Country) ? 1 : 0);
HasDateAdrFrom.Update(adress.DateAdrFrom.HasValue ? 1 : 0);
HasDatePostalPlace.Update(adress.DatePostalType.HasValue ? 1 : 0);
HasHouseLetter.Update(IsNotNullOrEmptyOrWhitespace(adress.HouseLetter) ? 1 : 0);
HasHouseNumber.Update(adress.HouseNumber.HasValue ? 1 : 0);
HasPostalAddressValidFrom.Update(adress.PostalAddressValidFrom.HasValue ? 1 : 0);
HasPostalAdress.Update(IsNotNullOrEmptyOrWhitespace(adress.PostalAddress) ? 1 : 0);
HasPostalCode.Update(IsNotNullOrEmptyOrWhitespace(adress.PostalCode) ? 1 : 0);
HasPostalPlace.Update(IsNotNullOrEmptyOrWhitespace(adress.PostalPlace) ? 1 : 0);
PostalType.Update(adress.PostalType);
HasSameNinAsPersonObject.Update(adress.NIN == p.NIN ? 1 : 0);
HasSchoolDistrict.Update(IsNotNullOrEmptyOrWhitespace(adress.SchoolDistrict) ? 1 : 0);
HasSt.Update(IsNotNullOrEmptyOrWhitespace(adress.St) ? 1 : 0);
HasStreetname.Update(IsNotNullOrEmptyOrWhitespace(adress.StreetName) ? 1 : 0);
HasStreetnumber.Update(IsNotNullOrEmptyOrWhitespace(adress.StreetNumber) ? 1 : 0);
HasXcoordYcoord.Update(adress.XCoord.HasValue || adress.YCoord.HasValue ? 1 : 0);
HasBasicStatisticalUnit.Update(adress.BasicStatisticalUnit.HasValue ? 1 : 0);
HasCadastralNumber.Update(IsNotNullOrEmptyOrWhitespace(adress.CadastralNumber) ? 1:0);
HasConstituency.Update(IsNotNullOrEmptyOrWhitespace(adress.Constituency) ? 1 : 0);
HasDistrictCodeandDistrictName.Update(IsNotNullOrEmptyOrWhitespace(adress.DistrictCode) || IsNotNullOrEmptyOrWhitespace(adress.DistrictName) ? 1 : 0);
HasPropertyNumber.Update(IsNotNullOrEmptyOrWhitespace(adress.PropertyNumber) ? 1 : 0);
if (PersonStatistics.IsPersonDead(p) == 1)
{
HasPostalCodeGivenIsDead.Update(IsNotNullOrEmptyOrWhitespace(adress.PostalCode) ? 1 : 0);
}
if (p.Addresses.Length > 1)
{
foreach (var obsoleteAdress in p.Addresses.Where(a => !a.CurrentAddress))
{
GivenMultipleAdressObject_HasSameNinAsPersonObject.Update(obsoleteAdress.NIN == p.NIN ? 1 : 0);
}
}
}
private async Task <int> Import()
{
Console.WriteLine("Staring import");
while (_dataProvider.HasMore())
{
Imported++;
var p = await _dataProvider.GetNextPerson();
if (p == null)
{
continue;
}
bool birthdayWasFoundFromNin;
var ageQuant = PersonStatistics.CalculateAgeQuant(p, _ageQuants, out birthdayWasFoundFromNin);
if (ageQuant == PersonStatistics.InvalidQuant)
{
continue;
}
var pregNode = new PregNode
{
Confirmed = true,
AgeQuants = ageQuant,
BirthDay = p.DateOfBirth,
Kjonn = CommonFunctions.GetKjonn(p.NIN).Value,
MomNin = GetParent(p.MothersNIN),
DadNin = GetParent(p.FathersNIN),
DebugId = DebugMode ? p.GivenName : null,
MarriedNin = p.SpouseNIN,
Nin = p.NIN,
Id = _nextId++
};
_statistics.FromImport(p, ageQuant);
if (pregNode.MomNin != null)
{
_persons[pregNode.MomNin].ChildNins.Add(pregNode.Nin);
}
if (pregNode.DadNin != null)
{
_persons[pregNode.DadNin].ChildNins.Add(pregNode.Nin);
}
if (_persons.ContainsKey(p.NIN))
{
_persons[p.NIN].MomNin = pregNode.MomNin;
_persons[p.NIN].Confirmed = pregNode.Confirmed;
_persons[p.NIN].DadNin = pregNode.DadNin;
_persons[p.NIN].DebugId = pregNode.DebugId;
_persons[p.NIN].FamilyId = pregNode.FamilyId;
_persons[p.NIN].Nin = pregNode.Nin;
_persons[p.NIN].Id = pregNode.Id;
_persons[p.NIN].AgeQuants = pregNode.AgeQuants;
_persons[p.NIN].Kjonn = pregNode.Kjonn;
}
else
{
_persons.Add(p.NIN, pregNode);
}
if (_persons.Count % 10000 == 0)
{
Console.WriteLine("Done importing " + _persons.Count);
}
}
return(Imported);
}
