public void TestGetPIDWithEuropeanExpiredProductTargetSpecies()
{
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct).Result;
foreach (var missingProduct in pid.ExpiredProducts
.Where(ep => ep.SPC_Link.ToLowerInvariant()
.Contains("ema.europa.eu"))
.Where(ep => ep.TargetSpecies == null ||
!ep.TargetSpecies.Any()))
{
Debug.WriteLine(missingProduct.Name);
}
Assert.IsFalse(pid.ExpiredProducts
.Where(ep => ep.SPC_Link.ToLowerInvariant()
.Contains("ema.europa.eu"))
.Any(ep => ep.TargetSpecies == null ||
!ep.TargetSpecies.Any()));
foreach (var expiredProduct in pid.ExpiredProducts
.Where(ep => ep.SPC_Link.ToLowerInvariant()
.Contains("ema.europa.eu")))
{
foreach (var sp in expiredProduct.TargetSpecies)
{
Debug.WriteLine($"{expiredProduct.Name}\t{sp}");
}
}
}
public void TestGetPIDWithExpiredProductTargetSpecies()
{
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct).Result;
Assert.IsFalse(pid.ExpiredProducts
.Where(ep => ep.SPC_Link.ToLower().EndsWith(".doc") ||
ep.SPC_Link.ToLower().EndsWith(".docx"))
.Any(ep => !ep.TargetSpecies.Any()));
}
public void TestAllProducts()
{
var pid = VMDPIDFactory.GetVmdPid().Result;
var ap = pid.AllProducts;
//check product count
var totalCount = pid.HomoeopathicProducts.Count
+ pid.CurrentlyAuthorisedProducts.Count
+ pid.ExpiredProducts.Count
+ pid.SuspendedProducts.Count;
Assert.IsTrue(totalCount == ap.Count(), "Mismatched total counts");
//check properties of returned products
foreach (var product in ap)
{
//centralised-authorisation products have lots of properties missing
if (product.AuthorisationRoute.Equals("Centralised"))
{
continue;
}
//check all string properties populated
foreach (var property in product.GetType().GetProperties()
.Where(p => p.PropertyType == typeof(string) &&
!p.Name.Equals("UKPAR_Link") &&
!p.Name.Equals("PAAR_Link")))
{
if (_missingTherapeuticGroup.Contains(product.VMNo) && property.Name.Equals("TherapeuticGroup"))
{
continue;
}
if (_missingDistributionCategory.Contains(product.VMNo) && property.Name.Equals("DistributionCategory"))
{
continue;
}
Assert.IsFalse(string.IsNullOrEmpty(property.GetValue(product).ToString()), $"Property {property.Name} empty for product {product}");
}
//check all string list props populated
foreach (var property in typeof(ReferenceProduct).GetProperties()
.Where(p => p.PropertyType == typeof(IEnumerable <string>)))
{
if ((typeof(ExpiredProduct) == product.GetType() ||
_missingTargetSpecies.Contains(product.VMNo)) &&
property.Name.Equals("TargetSpecies"))
{
continue;
}
Assert.IsTrue(((IEnumerable <string>)property.GetValue(product)).Any(), $"Property {property.Name} empty for product {product}");
}
}
}
public void TestGetPID()
{
var pid = VMDPIDFactory.GetVmdPid().Result;
Assert.IsNotNull(pid, "returned PID object is null");
Assert.IsNotNull(pid.CreatedDateTime, "pid.CreatedDateTime is null");
Assert.IsTrue(pid.CurrentlyAuthorisedProducts.Count > 0, "No currently authorised products");
Assert.IsTrue(pid.ExpiredProducts.Count > 0, "No expired products");
Assert.IsTrue(pid.SuspendedProducts.Count > 0, "No suspended products");
Assert.IsTrue(pid.HomoeopathicProducts.Count > 0, "No homoeopathic products");
}
public void TestProductPropertyAggregators()
{
var pid = VMDPIDFactory.GetVmdPid().Result;
foreach (var prop in typeof(VMDPID).GetProperties().Where(p => p.PropertyType == typeof(IEnumerable <string>)))
{
var p = (IEnumerable <string>)prop.GetValue(pid);
Assert.IsNotNull(p, $"{prop.Name} is null");
Assert.IsTrue(p.Any(), $"{prop.Name} empty");
}
}
public void TestStaticTypingOfTargetSpecies()
{
var pid = VMDPIDFactory.GetVmdPid().Result;
foreach (var p in pid.AllProducts)
{
if (p.GetType() == typeof(ExpiredProduct) || _missingTargetSpecies.Contains(p.VMNo))
{
continue;
}
Assert.IsTrue(p.TargetSpeciesTyped != null && p.TargetSpeciesTyped.Any(),
$"Non-expired with un-filled strongly typed target species for {p.Name}");
}
}
public void TestTargetSpeciesExtractionLocal()
{
var pid = VMDPIDFactory.GetVmdPid().Result;
foreach (var ep in pid.ExpiredProducts.Where(ep => !EPARTools.IsEPAR(ep.SPC_Link)))
{
var spc = VMDPIDFactory.GetSpc(ep).Result;
Debug.WriteLine(spc);
spc = spc.ToLowerInvariant().EndsWith(".doc") ? WordConverter.ConvertDocToDocx(spc) : spc;
var ts = SPCParser.GetTargetSpecies(spc);
Assert.IsNotNull(ts, $"null ts for {ep.Name}, {spc}");
Assert.IsTrue(ts.Any(), $"empty ts for {ep.Name}, {spc}");
Assert.IsFalse(ts.Any(string.IsNullOrWhiteSpace), $"blank species for {ep.Name}, {spc}");
}
}
public void TestStaticTypingOfTargetSpeciesEmaLicensedExpiredProducts()
{
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct).Result;
var errorString = "";
var errors = pid.AllProducts
.Where(p => p.GetType() == typeof(ExpiredProduct) && EPARTools.IsEPAR(((ExpiredProduct)p).SPC_Link))
.Where(p => p.TargetSpeciesTyped == null || !p.TargetSpeciesTyped.Any()).ToList();
if (errors.Any())
{
errorString = string.Join(Environment.NewLine,
errors.Select(p => $"{p.Name} " +
$"from {string.Join(';', p.TargetSpecies??new string[0])}"));
}
Assert.IsFalse(errors.Any(),
$"EMA-licensed expired product with un-filled strongly typed target species for: {errorString}");
}
public void TestProductMatchRunner()
{
var cfg = new DefaultProductMatchConfig();
var pmr = new ProductMatchRunner(cfg);
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.PersistentPid).Result;
var ap = new SoldProduct {
TargetSpecies = new[] { "cattle" },
Product = new Product {
Name = "metacam"
},
ActionDate = DateTime.Now
};
var res = pmr.GetMatch(ap, pid.RealProducts);
Assert.IsNotNull(res);
Assert.IsTrue(res.ReferenceProduct.Name.StartsWith("metacam", StringComparison.InvariantCultureIgnoreCase));
Assert.IsTrue(res.ReferenceProduct.TargetSpecies.Any(ts => ts.Equals("cattle", StringComparison.InvariantCultureIgnoreCase)));
}
public void TestStaticTypingOfTargetSpeciesExpiredProducts()
{
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct).Result;
foreach (var p in pid.AllProducts)
{
if (p.GetType() != typeof(ExpiredProduct) ||
EPARTools.IsEPAR(((ExpiredProduct)p).SPC_Link)
)
{
continue;
}
if (!(p.TargetSpeciesTyped != null && p.TargetSpeciesTyped.Any()))
{
Debug.WriteLine(string.Join(';', p.TargetSpecies));
}
Assert.IsTrue(p.TargetSpeciesTyped != null && p.TargetSpeciesTyped.Any(),
$"Expired product with un-filled strongly typed target species for {p.Name}"
+ $"from {string.Join(';', p.TargetSpecies)}");
}
}
public double Evaluate(IChromosome chromosome)
{
var correctCount = 0;
var pid = VMDPIDFactory.GetVmdPid(
PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.PersistentPid
).Result;
var cfg = (FloatingPointChromosome)chromosome;
var threshold = cfg.ToFloatingPoints()[3];
var pmr = new ProductMatchRunner(cfg.GetMatchConfig());
var toMatch = new ConcurrentDictionary <string, string[]>(TruthFactory.GetTruth());
Parallel.ForEach(toMatch, tm =>
{
var ap = new ActionedProduct {
Product = new Product {
Name = tm.Key
}, TargetSpecies = new[] { "cattle" }
};
ReferenceProduct[] rp;
lock (pid)
{
rp = pid.RealProducts.ToArray();
}
var res = pmr.GetMatch(ap, rp);
if ((tm.Value.Contains(res.ReferenceProduct.VMNo) && res.ProductNameSimilarity > threshold) ||
(tm.Value.All(string.IsNullOrEmpty) && res.ProductNameSimilarity < threshold)
)
{
Interlocked.Increment(ref correctCount);
}
});
return((double)correctCount / toMatch.Count);
}
private static void Main(string[] args)
{
if (args.Length > 0 && File.Exists(args[0]))
{
var sb = new StringBuilder("\"Input Name\",\"Matched Name\",\"VM Number\",\"Similarity Score\"" + Environment.NewLine);
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.PersistentPid).Result;
var cfg = new DefaultProductMatchConfig();
var pmr = new ProductMatchRunner(cfg);
var sw = Stopwatch.StartNew();
var inputStrings = new BlockingCollection <string>();
using (var fs = File.OpenText(args[0]))
{
while (!fs.EndOfStream)
{
inputStrings.Add(fs.ReadLine()?.ToLowerInvariant().Trim());
}
}
sw.Restart();
Parallel.ForEach(inputStrings.Where(s => !string.IsNullOrEmpty(s)), inputString =>
{
var ap = new SoldProduct
{
TargetSpecies = new[] { "cattle" },
Product = new Product {
Name = inputString
},
ActionDate = DateTime.Now
};
var res = pmr.GetMatch(ap, pid.RealProducts);
lock (sb)
{
sb.AppendJoin(',',
$"\"{res.InputProduct.Product.Name}\"",
$"\"{res.ReferenceProduct.Name}\"",
$"\"{res.ReferenceProduct.VMNo}\"",
res.ProductNameSimilarity.ToString(CultureInfo.InvariantCulture),
Environment.NewLine);
}
});
Console.WriteLine(sb.ToString());
}
else
if (args.Length > 0 && args[0].Equals("print", StringComparison.InvariantCultureIgnoreCase))
{
var propName = args[1];
var pidProperties = typeof(VMDPID).GetProperties();
try
{
var prop = pidProperties.Single(
p => p.Name.Equals(propName, StringComparison.InvariantCultureIgnoreCase));
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.PersistentPid).Result;
var values = (IEnumerable <string>)prop.GetValue(pid);
foreach (var value in values.Distinct().OrderBy(s => s))
{
Console.WriteLine(value);
}
}
catch (Exception)
{
Console.WriteLine($"Property {propName} not found in VMDPID");
}
}
else
if (args.Length > 1 && args[0].Equals("explainmatch"))
{
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.PersistentPid).Result;
var cfg = new DefaultProductMatchConfig();
var pmr = new ProductMatchRunner(cfg);
var name = args[1];
var ap = new ActionedProduct
{
Product = new Product {
Name = name
},
};
var refprod = pid.AllProducts.Single(p => p.VMNo.Equals(args[2]));
var foo = ap.GetMatchingResult(refprod, cfg);
Console.WriteLine(foo);
}
else
if (args.Length > 2 && args[0].Equals("explain"))
{
var pid = VMDPIDFactory.GetVmdPid(PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.PersistentPid).Result;
var cfg = new DefaultProductMatchConfig();
var pmr = new ProductMatchRunner(cfg);
var species = args[2].Split(',');
var name = args[1];
var ap = new ActionedProduct
{
Product = new Product {
Name = name
},
TargetSpecies = species
};
var mr = pmr.GetMatchResults(ap, pid.RealProducts).ToArray();
var dc = pmr.GetDisambiguationCandidates(mr).ToArray();
var res = pmr.GetMatch(ap, pid.RealProducts);
Console.WriteLine("Matched product:");
Console.WriteLine(string.Join('\t', res.ReferenceProduct.Name, res.ReferenceProduct.VMNo, res.ProductNameSimilarity.ToString(CultureInfo.InvariantCulture)));
Console.WriteLine();
Console.WriteLine("All products:");
foreach (var matchResult in mr)
{
Console.WriteLine(string.Join('\t', matchResult.ReferenceProduct.Name, matchResult.ReferenceProduct.VMNo, matchResult.ProductNameSimilarity.ToString(CultureInfo.InvariantCulture)));
}
Console.WriteLine();
Console.WriteLine("Disambiguation Candidates:");
foreach (var productSimilarityResult in dc)
{
Console.WriteLine(string.Join('\t', productSimilarityResult.ReferenceProduct.Name, productSimilarityResult.ReferenceProduct.VMNo, productSimilarityResult.ProductNameSimilarity.ToString(CultureInfo.InvariantCulture)));
}
var disambiguationConfig = ((HierarchicalFilterWithRandomFinalSelect)cfg.Disambiguator)._cfg;
foreach (var disambiguationFilter in disambiguationConfig.Filters)
{
Console.WriteLine($"Filter: {disambiguationFilter.GetType().Name}");
foreach (var filterResult in disambiguationFilter.FilterResults(dc))
{
Console.WriteLine(string.Join('\t', filterResult.ReferenceProduct.Name, filterResult.ReferenceProduct.VMNo, filterResult.ProductNameSimilarity.ToString(CultureInfo.InvariantCulture)));
}
}
}
else if (args.Length > 1 && args[0].Equals("optimise", StringComparison.InvariantCultureIgnoreCase))
{
TruthFactory.SetPath(args[1]);
var pid = VMDPIDFactory.GetVmdPid(
PidFactoryOptions.GetTargetSpeciesForExpiredEmaProduct |
PidFactoryOptions.GetTargetSpeciesForExpiredVmdProduct |
PidFactoryOptions.PersistentPid
).Result;
var ga = GaRunner.GetGeneticAlgorithm();
Console.WriteLine("Generation, ABWeightRatio, AWeight, BWeight, Threshold, SuccessRate");
var latestFitness = 0.0;
ga.GenerationRan += (sender, e) =>
{
var bestChromosome = ga.BestChromosome as FloatingPointChromosome;
var bestFitness = bestChromosome.Fitness.Value;
if (bestFitness != latestFitness)
{
latestFitness = bestFitness;
var phenotype = bestChromosome.ToFloatingPoints();
Console.WriteLine(
"{0,2},{1},{2},{3},{4},{5}",
ga.GenerationsNumber,
phenotype[0],
phenotype[1],
phenotype[2],
phenotype[3],
bestFitness
);
}
};
ga.Start();
}
else
{
Console.WriteLine("Requires path to file to process as first argument.");
Console.ReadLine();
}
}