Exemplo n.º 1
0
        private void RunTest(string sdfile, string actvField, CircularFingerprinterClass classType, int folding, int xval, string modelFN, bool perceiveStereo)
        {
            WriteLine("[" + modelFN + "]");
            WriteLine("    Loading " + sdfile);

            try
            {
                var model = new Bayesian(classType, folding)
                {
                    PerceiveStereo = perceiveStereo
                };

                int row = 0, numActives = 0;
                using (var rdr = new EnumerableSDFReader(ResourceLoader.GetAsStream($"NCDK.Data.CDD.{sdfile}"), ChemObjectBuilder.Instance))
                {
                    foreach (var mol in rdr)
                    {
                        row++;

                        var stractv = (string)mol.GetProperties()[actvField];
                        int active  = stractv.Equals("true", StringComparison.Ordinal) ? 1 : stractv.Equals("false", StringComparison.Ordinal) ? 0 : int.Parse(stractv, NumberFormatInfo.InvariantInfo);
                        if (active != 0 && active != 1)
                        {
                            throw new CDKException("Activity field not found or invalid");
                        }

                        model.AddMolecule(mol, active == 1);
                        numActives += active;
                    }
                }

                WriteLine($"    Training with {row} rows, {numActives} actives, {(row - numActives)} inactives");

                model.Build();
                if (xval == 3)
                {
                    model.ValidateThreeFold();
                }
                else if (xval == 5)
                {
                    model.ValidateFiveFold();
                }
                else
                {
                    model.ValidateLeaveOneOut();
                }

                WriteLine($"    Validation: ROC AUC={model.RocAUC}");
                WriteLine($"    Parsing reference model");

                //FileReader frdr=new FileReader(modelFN);
                Bayesian reference;
                using (var mrdr = new StreamReader(ResourceLoader.GetAsStream($"NCDK.Data.CDD.{modelFN}")))
                {
                    reference = Bayesian.Deserialise(mrdr);
                }

                // start comparing the details...

                bool failed = false;
                if (model.Folding != reference.Folding)
                {
                    WriteLine($"    ** reference folding size={reference.Folding}");
                    failed = true;
                }
                if (model.TrainingSize != reference.TrainingSize)
                {
                    WriteLine($"    ** reference training size={reference.TrainingSize}");
                    failed = true;
                }
                if (model.TrainingActives != reference.TrainingActives)
                {
                    WriteLine($"    ** reference training actives={reference.TrainingActives}");
                    failed = true;
                }
                if (model.RocType != reference.RocType)
                {
                    WriteLine($"    ** reference ROC type={reference.RocType}");
                    failed = true;
                }
                if (!DblEqual(model.RocAUC, reference.RocAUC))
                {
                    WriteLine($"    ** reference ROC AUC={reference.RocAUC}");
                    failed = true;
                }
                if (Math.Abs(model.LowThreshold - reference.LowThreshold) > 0.00000000000001)
                {
                    WriteLine($"    ** reference lowThresh={reference.LowThreshold} different to calculated {model.LowThreshold}");
                    failed = true;
                }
                if (Math.Abs(model.HighThreshold - reference.HighThreshold) > 0.00000000000001)
                {
                    WriteLine($"    ** reference highThresh={reference.HighThreshold} different to calculated {model.HighThreshold}");
                    failed = true;
                }

                // make sure individual hash bit contributions match
                var mbits = model.Contributions;
                var rbits = reference.Contributions;
                if (mbits.Count != rbits.Count)
                {
                    WriteLine($"    ** model has {mbits.Count} contribution bits, reference has {rbits.Count}");
                    failed = true;
                }
                foreach (var h in mbits.Keys)
                {
                    if (!rbits.ContainsKey(h))
                    {
                        WriteLine($"    ** model hash bit {h} not found in reference");
                        failed = true;
                        break; // one is enough
                    }
                }
                foreach (var h in rbits.Keys)
                {
                    if (!mbits.ContainsKey(h))
                    {
                        WriteLine($"    ** reference hash bit {h} not found in model");
                        failed = true;
                        break; // one is enough
                    }
                }
                foreach (var h in mbits.Keys)
                {
                    if (rbits.ContainsKey(h))
                    {
                        double c1 = mbits[h], c2 = rbits[h];
                        if (!DblEqual(c1, c2))
                        {
                            WriteLine($"    ** contribution for bit {h}: model={c1}, reference={c2}");
                            failed = true;
                            break; // one is enough
                        }
                    }
                }

                if (failed)
                {
                    throw new CDKException("Comparison to reference failed");
                }
            }
            catch (CDKException)
            {
                throw;
            }
            catch (Exception ex)
            {
                throw new CDKException("Test failed", ex);
            }
        }
Exemplo n.º 2
0
        // builds a model and uses the scaled predictions to rack up a confusion matrix, for comparison
        private static void ConfirmPredictions(string sdfile, int truePos, int trueNeg, int falsePos, int falseNeg)
        {
            WriteLine($"[{sdfile}] comparing confusion matrix");

            var molecules  = new List <IAtomContainer>();
            var activities = new List <bool>();
            var model      = new Bayesian(CircularFingerprinterClass.ECFP6, 1024);

            try
            {
                using (var ins = ResourceLoader.GetAsStream("NCDK.Data.CDD." + sdfile))
                {
                    var rdr = new EnumerableSDFReader(ins, ChemObjectBuilder.Instance);

                    foreach (var mol in rdr)
                    {
                        bool actv = "true" == (string)mol.GetProperties()["Active"];
                        molecules.Add(mol);
                        activities.Add(actv);
                        model.AddMolecule(mol, actv);
                    }
                }
            }
            catch (CDKException)
            {
                throw;
            }
            catch (Exception ex)
            {
                throw new CDKException("Test failed", ex);
            }

            model.Build();
            model.ValidateLeaveOneOut();

            // build the confusion matrix
            int gotTP = 0, gotTN = 0, gotFP = 0, gotFN = 0;

            for (int n = 0; n < molecules.Count; n++)
            {
                double pred = model.ScalePredictor(model.Predict(molecules[n]));
                bool   actv = activities[n];
                if (pred >= 0.5)
                {
                    if (actv)
                    {
                        gotTP++;
                    }
                    else
                    {
                        gotFP++;
                    }
                }
                else
                {
                    if (actv)
                    {
                        gotFN++;
                    }
                    else
                    {
                        gotTN++;
                    }
                }
            }

            WriteLine("    True Positives:  got=" + gotTP + " require=" + truePos);
            WriteLine("         Negatives:  got=" + gotTN + " require=" + trueNeg);
            WriteLine("    False Positives: got=" + gotFP + " require=" + falsePos);
            WriteLine("          Negatives: got=" + gotFN + " require=" + falseNeg);

            if (gotTP != truePos || gotTN != trueNeg || gotFP != falsePos || gotFN != falseNeg)
            {
                throw new CDKException("Confusion matrix mismatch");
            }
        }