public bool Add(QuestionnaireScope scope)
{
try
{
_log.Information("Assign unique id to answers...");
var answers = scope.Entries.SelectMany(z => z.Answers);
var count = 0;
foreach (var answ in answers)
{
answ.Id = ++count;
}
_log.Information("Store to database...");
var qcollection = _database.GetCollection <QuestionnaireScope>(nameof(QuestionnaireScope));
while (qcollection.Count(z => z.Id == scope.Id) != 0)
{
scope.Id = Environment.TickCount;//to prevent references to removed scopes
}
qcollection.InsertOne(scope);
return(true);
}
catch (Exception ex)
{
_log.Error(ex, ex.Message);
return(false);
}
}
public static float[] PredictionsToScales(QuestionnaireScope scope, float[][] choices)
{
if (choices.Length != 120)
{
throw new InvalidOperationException("Incorrent input");
}
var scales = new float[5];
try
{
for (int q = 0; q < choices.Length; q++)
{
var entry = scope.Entries.FirstOrDefault(z => z.Number == q + 1);
if (entry != null)
{
for (int answ = 0; answ < entry.Answers.Count; answ++)
{
foreach (var map in entry.Answers[answ].Mappings)
{
scales[map.OutcomeIndex - 1] += choices[q][answ] * (entry.Answers[answ].Score + 2);
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(scales);
}
public static float[] GetScales(QuestionnaireScope scope, int[] choices)
{
if (choices.Length != 120)
{
throw new InvalidOperationException("Incorrent input");
}
var scales = new float[5];
try
{
var raw = from a in scope.Entries
from b in a.Answers
from c in b.Mappings
let outcome = scope.Outcomes.Find(z => z.Index == c.OutcomeIndex)
select new
{
Key = outcome.Index - 1,
Flag = choices.Contains(b.Id) ? (b.Score + 3) : 0.0f
};
foreach (var r in raw)
{
if (r.Flag != 0)
{
scales[r.Key] += r.Flag - 1.0f;
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(scales);
}
public bool AddQuestionnaire(string token, QuestionnaireScope scope)
{
if (!_userValidation.ValidateToken(token))
{
throw new AddressAccessDeniedException();
}
var res = _questionnaireRepository.Add(scope);
_log.Information($"Operation finished with result: {res}");
return(res);
}
public static double[] CovertAnswersToVector(QuestionnaireScope scope, int[] choices)
{
//var res = outcomeComputer.ComputeOutcomes(choices, scope);
//var retval = new float[scope.Outcomes.Count];
//foreach (var r in res)
//{
// retval[r.Index - 1] = 1;
//}
var raw = from a in scope.Entries
from b in a.Answers
from c in b.Mappings
let outcome = scope.Outcomes.Find(z => z.Index == c.OutcomeIndex)
select new
{
Key = outcome.CaptionText,
A = b.Id,
Flag = choices.Contains(b.Id) ? (double)(b.Score + 3) : 0.0
};
var retval = from a in raw
orderby a.A
where a.Flag != 0
select a.Flag - 1;
if (retval.Count() != 120)
{
Console.WriteLine("haba");
}
//var raw = from a in scope.Entries
// from b in a.Answers
// from c in b.Mappings
// let outcome = scope.Outcomes.Find(z => z.Index == c.OutcomeIndex)
// select new
// {
// Key = outcome.CaptionText,
// Flag = choices.Contains(b.Id) ? (double)b.Score : 0.0
// };
//var retval = from a in raw
// group a by a.Key into g
// orderby g.Key
// select g.Sum(z => z.Flag) / 3;
return(retval.ToArray());
}
private void PackScope(QuestionnaireScope scope)
{
scope.ThumbnailImage = CompressImage(scope.ThumbnailImage);
scope.BannerImage = CompressImage(scope.BannerImage);
foreach (var entry in scope.Entries)
{
entry.QuestionImage = CompressImage(entry.QuestionImage);
}
foreach (var outcoime in scope.Outcomes)
{
try
{
outcoime.DescriptionImage = CompressImage(outcoime.DescriptionImage);
}
catch (Exception ex)
{
_log.Error(ex, ex.Message);
}
}
}
private static QuestionnaireScope CloneQuestionnaireScope(QuestionnaireScope source)
{
return(new QuestionnaireScope
{
Annotation = source.Annotation,
BannerImage = source.BannerImage,
Capacity = source.Capacity,
Description = source.Description,
Entries = new List <QuestionnaireEntry>(source.Entries),
Generation = source.Generation,
Id = source.Id,
Name = source.Name,
OutcomeHeader = source.OutcomeHeader,
OutcomeLimit = source.OutcomeLimit,
Outcomes = new List <QuestionnaireOutcome>(source.Outcomes),
Passed = source.Passed,
QuestionHeader = source.QuestionHeader,
ThumbnailImage = source.ThumbnailImage
});
}
public void Add_SuccessfullyCreateNewRecord()
{
var questionnaireRepository = CreateQuestionnaireRepository();
var scope = new QuestionnaireScope {
Id = 1, Entries = new List <QuestionnaireEntry> {
new QuestionnaireEntry {
Answers = new List <QuestionnaireEntryAnswer> {
new QuestionnaireEntryAnswer {
Id = -1
}
}
}
}
};
var res = questionnaireRepository.Add(scope);
Assert.IsTrue(res);
Assert.AreEqual(1, scope.Entries[0].Answers[0].Id);
}
public bool Upsert(QuestionnaireScope scope)
{
try
{
var qcollection = _database.GetCollection <QuestionnaireScope>(nameof(QuestionnaireScope));
if (qcollection.Count(z => z.Id == scope.Id) == 0)
{
qcollection.InsertOne(scope);
}
else
{
qcollection.ReplaceOne(z => z.Id == scope.Id, scope);
}
return(true);
}
catch (Exception ex)
{
_log.Error(ex, ex.Message);
return(false);
}
}
public void Find_ReturnExistingScope()
{
var questionnaireRepository = CreateQuestionnaireRepository();
var scope = new QuestionnaireScope {
Id = 1, Entries = new List <QuestionnaireEntry> {
new QuestionnaireEntry {
Answers = new List <QuestionnaireEntryAnswer> {
new QuestionnaireEntryAnswer {
Id = -1
}
}
}
}
};
questionnaireRepository.Add(scope);
var res = questionnaireRepository.FindQuestionnaireById(scope.Id);
Assert.AreEqual(scope.Id, res.Id);
Assert.AreEqual(1, scope.Entries[0].Answers[0].Id);
}
public void RemoveQuestionnaire_ReturnNullAfterDeletion()
{
var questionnaireRepository = CreateQuestionnaireRepository();
var scope = new QuestionnaireScope {
Id = 1, Entries = new List <QuestionnaireEntry> {
new QuestionnaireEntry {
Answers = new List <QuestionnaireEntryAnswer> {
new QuestionnaireEntryAnswer {
Id = -1
}
}
}
}
};
questionnaireRepository.Add(scope);
var res = questionnaireRepository.RemoveQuestionnaire(scope.Id);
var aftermath = questionnaireRepository.FindQuestionnaireById(scope.Id);
Assert.IsTrue(res);
Assert.IsNull(aftermath);
}
private static Fifther Fifth()
{
var groupmapping = JsonConvert.DeserializeObject <Dictionary <long, double[]> >(File.ReadAllText("map_groups.json"));
var bagOfTerms = File.ReadAllLines("res\\expert_topics.csv").Skip(1).Select(z => z.Split(',')).ToDictionary(z => int.Parse(z[0]), z => new HashSet <string>(z[1].Split(' ')));
var xFilename = "224053984_dataset.json";
var scope = new QuestionnaireScope();
XmlSerializer serializer = new XmlSerializer(typeof(QuestionnaireScope));
using (var reader = new StreamReader("test_fifth.xml"))
{
scope = (QuestionnaireScope)serializer.Deserialize(reader);
}
var log = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.LiterateConsole()
.CreateLogger();
var repository = new UserGetRepository("passed_tests_ferrets.s3db", log, new CompressorProto());
Dictionary <int, float[]> vectors = new Dictionary <int, float[]>();
var users = new UserGet[0];
var count = 0;
const int batch = 1000;
do
{
try
{
users = repository.RangeSelect(count, batch).ToArray();
foreach (var user in users)
{
vectors.Add(user.id, user.ToVector(groupmapping, bagOfTerms));
}
count += batch;
log.Information("Done {Count} recs.", count);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
} while (users.Length != 0);
//File.WriteAllText("datax.csv", string.Join(", ", FlatUsertToVectorMapping.GetHeader(groupmapping.FirstOrDefault().Value.Length, bagOfTerms.Count)) + "\r\n");
//File.AppendAllLines("datax.csv", vectors.Select(z => string.Join(", ", z.Value.Select(x => x.ToString("0.000", CultureInfo.InvariantCulture)))));
var X = JsonConvert.DeserializeObject <List <FifthAttendance> >(File.ReadAllText(xFilename)).Where(z => vectors.ContainsKey(z.vkid)).Where(z => QuestionaireDatasetPreparation.CovertAnswersToVector(scope, z.AnswersId).Length == 120).ToArray();
var Y = X.Select(z => QuestionaireDatasetPreparation.CovertAnswersToVector(scope, z.AnswersId)).ToArray();
File.WriteAllLines("answers.csv", Y.Select(z => string.Join(", ", z)));
var rnd = new Random(Environment.TickCount);
var train = Enumerable.Range(0, X.Length).OrderBy(z => rnd.NextDouble()).ToArray();
var x_train = train.Take(X.Length * 80 / 100).Select(z => vectors[X[z].vkid]).ToArray();
var y_train = train.Take(X.Length * 80 / 100).Select(z => Y[z]).ToArray();
var x_test = train.Skip(X.Length * 80 / 100).Select(z => vectors[X[z].vkid]).ToArray();
var y_test = train.Skip(X.Length * 80 / 100).Select(z => Y[z]).ToArray();
var fifther = new Fifther();
Console.WriteLine();
for (int qnum = 0; qnum < 120; qnum++)
{
Console.WriteLine($"Question: {qnum}");
var yds = y_train.Select(z => (float)z[qnum]).ToArray();
var ytds = y_test.Select(z => (float)z[qnum]).ToArray();
var parameters = new Dictionary <string, object>();
parameters["max_depth"] = 10;
parameters["learning_rate"] = 0.1f;
parameters["n_estimators"] = 300;
parameters["silent"] = true;
parameters["objective"] = "multi:softprob";//"binary:logistic";//
parameters["nthread"] = -1;
parameters["gamma"] = 4f;
parameters["min_child_weight"] = 2;
parameters["max_delta_step"] = 1;
parameters["subsample"] = 1f;
parameters["colsample_bytree"] = 1f;
parameters["colsample_bylevel"] = 1f;
parameters["reg_alpha"] = 0f;
parameters["reg_lambda"] = 1f;
parameters["scale_pos_weight"] = 1f;
parameters["base_score"] = 0.8F;
parameters["seed"] = 0;
parameters["missing"] = float.NaN;
parameters["num_class"] = 5;
var xgbc = new XGBClassifier(parameters);
xgbc.Fit(x_train, yds);
fifther.AddLevel(qnum, xgbc);
var discrepancy = 0.0;
var dist = 0.0;
var preds = xgbc.PredictDistr(x_train);
for (int pos = 0; pos < preds.Length; pos++)
{
var tmp = new float[5];
tmp[(int)yds[pos]] = 1f;
dist += Math.Abs(det.GetMaxIndex(preds[pos]) - yds[pos]);
discrepancy += det.EuclidianDistance(tmp, preds[pos]);
}
Console.WriteLine("[Train] Discrepancy {0:0.000} Dist {1:0.000}", 1.0 - discrepancy / (preds.Length * Math.Sqrt(2.0)), dist / preds.Length);
preds = xgbc.PredictDistr(x_test);
discrepancy = 0.0;
dist = 0.0;
for (int pos = 0; pos < preds.Length; pos++)
{
var tmp = new float[5];
tmp[(int)yds[pos]] = 1f;
dist += Math.Abs(det.GetMaxIndex(preds[pos]) - yds[pos]);
discrepancy += det.EuclidianDistance(tmp, preds[pos]);
}
Console.WriteLine("[Test ] Discrepancy {0:0.000} Dist {1:0.000}", 1.0 - discrepancy / (preds.Length * Math.Sqrt(2.0)), dist / preds.Length);
}
Console.WriteLine("Done");
fifther.Save(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "fifth"));
Console.ReadLine();
Console.ReadLine();
Console.ReadLine();
Console.ReadLine();
return(fifther);
}
private static void ComputeFifth()
{
var fifther = new Fifther();
fifther.Load(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "fifth"));
Console.Write("Enter dataset:");
var cString = Console.ReadLine();
var bagOfTerms = File.ReadAllLines("res\\expert_topics.csv").Skip(1).Select(z => z.Split(',')).ToDictionary(z => int.Parse(z[0]), z => new HashSet <string>(z[1].Split(' ')));
var scope = new QuestionnaireScope();
XmlSerializer serializer = new XmlSerializer(typeof(QuestionnaireScope));
using (var reader = new StreamReader("test_fifth.xml"))
{
scope = (QuestionnaireScope)serializer.Deserialize(reader);
}
var log = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.LiterateConsole()
.CreateLogger();
var repository = new UserGetRepository(cString, log, new CompressorProto());
var fifthRepo = new FifthResultRepository(cString);
var groupmapping = JsonConvert.DeserializeObject <Dictionary <long, double[]> >(File.ReadAllText("map_groups.json"));
fifthRepo.CleanAll();
var count = 0;
var gsw = Stopwatch.StartNew();
UserGet[] users = new UserGet[0];
const int batch = 1000;
do
{
try
{
users = repository.RangeSelect(count, batch).ToArray();
users = users.Where(z => z?.Groups?.Count > 0).ToArray();
float[][] input = new float[users.Length][];
for (int user = 0; user < users.Length; user++)
{
input[user] = users[user].ToVector(groupmapping, bagOfTerms);
}
var preds = fifther.PredictDistr(input, 5);
fifthRepo.Insert(users.Select(z => z.id).ToArray(), preds.Select(z => QuestionaireDatasetPreparation.PredictionsToScales(scope, z)).ToArray());
count += batch;
log.Information("Done {Count} recs. {DaysForMillion} days", count, TimeSpan.FromMilliseconds(1000000 * gsw.ElapsedMilliseconds / count).TotalDays);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
} while (users.Length != 0);
Console.WriteLine("Done");
Console.ReadLine();
Console.ReadLine();
}
public List <QuestionnaireOutcome> ComputeOutcomes(QuestionnaireChoices passedQuestionnaire, QuestionnaireScope scope)
{
if (passedQuestionnaire == null)
{
return(new List <QuestionnaireOutcome>());
}
if (passedQuestionnaire.AnswerIds == null)
{
return(new List <QuestionnaireOutcome>());
}
if (passedQuestionnaire.AnswerIds.Count == 0)
{
return(new List <QuestionnaireOutcome>());
}
if (passedQuestionnaire.ScopeId != scope.Id)
{
return(new List <QuestionnaireOutcome>());
}
_log.Information($"Respondent results:{passedQuestionnaire}");
var mappings = scope.Entries
.SelectMany(z => z.Answers)
.Where(z => passedQuestionnaire.AnswerIds.Contains(z.Id))
.SelectMany(z => z.Mappings)
.GroupBy(z => z.OutcomeIndex)
.Select(z => new { O = z.Key, W = z.Sum(x => x.Weight) })
.OrderBy(z => - z.W)
.ToArray();
var res = new List <QuestionnaireOutcome>();
foreach (var map in mappings)
{
if (map.W > 0)
{
var outc = scope.Outcomes.Where(z => z.Index == map.O && Match(z, map.W));
if (outc.Any())
{
res.AddRange(outc);
}
}
}
if (!res.Any())
{
var cand = scope.Outcomes.FirstOrDefault(z => z.WeightMin != z.WeightMax && z.WeightMin == 0);
if (cand != null)
{
_log.Information($"Return default outcome {cand}");
return(new List <QuestionnaireOutcome> {
cand
});
}
}
_log.Information($"Oucomes: {string.Join("; ", res)}");
if (scope.OutcomeLimit == 0)
{
return(res.ToList());
}
else
{
return(res.Take(scope.OutcomeLimit).ToList());
}
}