private static bool TrainModel(KeywordPredictorFileMapping mapping)
{
IKeywordPredictor predictor = mapping.KeywordPredictorType.GetMethod("GetGlobalModel").Invoke(null, null) as IKeywordPredictor;
var keywordData = new FileSystemDataSource().LoadKeywordTrainingExamples();
var X = KeywordPredictorTrainingUtils.GenerateKeywordTrainingData(keywordData);
var Y = KeywordPredictorTrainingUtils.GenerateKeywordTargetData(keywordData);
predictor.Train(X, Y);
AnsEncoderStream stream;
try
{
stream = new AnsEncoderStream(
new FileStream(mapping.DefaultFileLocation, FileMode.Create, FileAccess.Write),
1048576,
4096);
} catch (IOException)
{
return(false);
}
using (stream)
{
bool res = predictor.Save(stream);
stream.Flush();
return(res);
}
}
public DatabaseQueryProcessor(DatabaseQueryProcessorSettings settingsIn = null)
{
if (settingsIn == null)
{
settingsIn = DatabaseQueryProcessorSettings.GenerateDefaultSettings();
}
KeywordTagger = AveragedPerceptronTagger.GetTagger();
var keywordPredictorType = Type.GetType("OldManInTheShopServer.Models.KeywordPrediction." + settingsIn.KeywordPredictorIdString);
if (keywordPredictorType == null)
{
throw new NullReferenceException("Type string " + settingsIn.KeywordPredictorIdString + " was not found matching a known class in KeywordPrediction");
}
try
{
KeywordPredictor = keywordPredictorType.GetMethod("GetGlobalModel").Invoke(null, null) as IKeywordPredictor;
} catch (NullReferenceException e)
{
Console.WriteLine("Type " + settingsIn.KeywordPredictorIdString + " did not contain a GetGlobalModel method. It needs to have one.");
throw e;
}
if (KeywordPredictor == null)
{
throw new InvalidCastException("Loaded class could not be cast to IKeywordPredictor");
}
var keywordClustererType = Type.GetType("OldManInTheShopServer.Models.KeywordClustering." + settingsIn.KeywordClustererIdString);
if (keywordClustererType == null)
{
throw new NullReferenceException("Type string " + settingsIn.KeywordClustererIdString + " was not found matching a known class in KeywordClustering");
}
KeywordClusterer = keywordClustererType.GetConstructor(new Type[0]).Invoke(null) as IDatabaseKeywordClusterer;
if (KeywordClusterer == null)
{
throw new InvalidCastException("Loaded class could not be cast to IDatabaseKeywordClusterer");
}
var problemPredictorType = Type.GetType("OldManInTheShopServer.Models.QueryProblemPrediction." + settingsIn.ProblemPredictorIdString);
if (keywordPredictorType == null)
{
throw new NullReferenceException("Type string " + settingsIn.ProblemPredictorIdString + " was not found matching a known class in QueryProblemPrediction");
}
ProblemPredictor = problemPredictorType.GetConstructor(new Type[0]).Invoke(null) as IDatabaseQueryProblemPredictor;
if (ProblemPredictor == null)
{
throw new InvalidCastException("Loaded class could not be cast to IDatabaseQueryProblemPredictor");
}
}
private static bool LoadModel(KeywordPredictorFileMapping mapping)
{
IKeywordPredictor predictor = mapping.KeywordPredictorType.GetMethod("GetGlobalModel").Invoke(null, null) as IKeywordPredictor;
Stream streamIn = null;
try
{
streamIn = new AnsDecoderStream(new FileStream(mapping.DefaultFileLocation, FileMode.Open, FileAccess.Read));
} catch (FileNotFoundException)
{
return(false);
}
using (streamIn)
{
return(predictor.Load(streamIn));
}
}
private static float CalcSimilarity(RepairJobEntry query, RepairJobEntry other)
{
IKeywordPredictor keyPred = NaiveBayesKeywordPredictor.GetGlobalModel();
AveragedPerceptronTagger tagger = AveragedPerceptronTagger.GetTagger();
List <String> tokened = SentenceTokenizer.TokenizeSentence(query.Complaint);
List <List <String> > tagged = tagger.Tag(tokened);
List <String> InputComplaintKeywords = keyPred.PredictKeywords(tagged);
tokened = SentenceTokenizer.TokenizeSentence(query.Problem);
tagged = tagger.Tag(tokened);
List <String> InputProblemKeywords = keyPred.PredictKeywords(tagged);
float score = 0;
tokened = SentenceTokenizer.TokenizeSentence(other.Complaint);
tagged = tagger.Tag(tokened);
List <String> JobComplaintKeywords = keyPred.PredictKeywords(tagged);
tokened = SentenceTokenizer.TokenizeSentence(other.Problem);
tagged = tagger.Tag(tokened);
List <String> JobProblemKeywords = keyPred.PredictKeywords(tagged);
foreach (String keyword in JobComplaintKeywords)
{
if (InputComplaintKeywords.Contains(keyword))
{
score++;
}
}
foreach (String keyword in JobProblemKeywords)
{
if (InputProblemKeywords.Contains(keyword))
{
score++;
}
}
return(score / (JobComplaintKeywords.Count + JobProblemKeywords.Count));
}
/**
* Summary:
* Loads the the currently selected KeywordPredictor from the file specified
* Note:
* This method does not do the above currently. It is hard coded to use the NaiveBayesKeywordPredictor.
* This is a hold out until the proper method is in place to switch between implementations
* through a configuration file.
*/
private bool LoadKeywordPredictor(string filePath)
{
KeywordPredictor = new NaiveBayesKeywordPredictor();
AnsDecoderStream streamIn;
try
{
streamIn = new AnsDecoderStream(
new FileStream(filePath, FileMode.Open, FileAccess.Read)
);
} catch (IOException)
{
return(false);
}
if (!KeywordPredictor.Load(streamIn))
{
streamIn.Close();
KeywordPredictorValid = false;
return(false);
}
streamIn.Close();
KeywordPredictorValid = true;
return(true);
}