private long processLine(String line, long averageCount)
{
if (string.IsNullOrEmpty(line) || line[0] == COMMENT_CHAR)
{
return(averageCount);
}
String[] tokens = SEPARATOR.Split(line);
//Preconditions.checkArgument(tokens.length >= 3 && tokens.length != 5, "Bad line: %s", line);
long itemID1 = long.Parse(tokens[0]);
long itemID2 = long.Parse(tokens[1]);
double diff = double.Parse(tokens[2]);
int count = tokens.Length >= 4 ? int.Parse(tokens[3]) : 1;
bool hasMkSk = tokens.Length >= 5;
if (itemID1 > itemID2)
{
long temp = itemID1;
itemID1 = itemID2;
itemID2 = temp;
}
FastByIDMap <RunningAverage> level1Map = averageDiffs.get(itemID1);
if (level1Map == null)
{
level1Map = new FastByIDMap <RunningAverage>();
averageDiffs.put(itemID1, level1Map);
}
RunningAverage average = level1Map.get(itemID2);
if (average != null)
{
throw new Exception("Duplicated line for item-item pair " + itemID1 + " / " + itemID2);
}
if (averageCount < maxEntries)
{
if (hasMkSk)
{
double mk = Double.Parse(tokens[4]);
double sk = Double.Parse(tokens[5]);
average = new FullRunningAverageAndStdDev(count, diff, mk, sk);
}
else
{
average = new FullRunningAverage(count, diff);
}
level1Map.put(itemID2, average);
averageCount++;
}
allRecommendableItemIDs.add(itemID1);
allRecommendableItemIDs.add(itemID2);
return(averageCount);
}
public RunningAverageAndStdDev GetValue(User user)
{
RunningAverageAndStdDev running = new FullRunningAverageAndStdDev();
Preference[] prefs = user.GetPreferencesAsArray();
foreach (Preference preference in prefs)
{
running.AddDatum(preference.Value);
}
return(running);
}
private double getEvaluation(FastByIDMap <PreferenceArray> testPrefs, Recommender recommender)
{
this.reset();
List <Action> callables = new List <Action>();
AtomicInteger noEstimateCounter = new AtomicInteger();
using (IEnumerator <KeyValuePair <long, PreferenceArray> > enumerator = testPrefs.entrySet().GetEnumerator())
{
while (enumerator.MoveNext())
{
Action item = null;
KeyValuePair <long, PreferenceArray> entry = enumerator.Current;
if (item == null)
{
item = delegate
{
long userID = entry.Key;
PreferenceArray array = entry.Value;
foreach (Preference preference in array)
{
float f = float.NaN;
try
{
f = recommender.estimatePreference(userID, preference.getItemID());
}
catch (NoSuchUserException)
{
log.info("User exists in test data but not training data: {}", new object[] { userID });
}
catch (NoSuchItemException)
{
log.info("Item exists in test data but not training data: {}", new object[] { preference.getItemID() });
}
if (float.IsNaN(f))
{
noEstimateCounter.incrementAndGet();
}
else
{
f = this.capEstimatedPreference(f);
this.processOneEstimate(f, preference);
}
}
};
}
callables.Add(item);
}
}
log.info("Beginning evaluation of {} users", new object[] { callables.Count });
RunningAverageAndStdDev timing = new FullRunningAverageAndStdDev();
execute(callables, noEstimateCounter, timing);
return(this.computeFinalEvaluation());
}
private static double computeThreshold(PreferenceArray prefs)
{
if (prefs.length() < 2)
{
return(double.NegativeInfinity);
}
RunningAverageAndStdDev dev = new FullRunningAverageAndStdDev();
int num = prefs.length();
for (int i = 0; i < num; i++)
{
dev.addDatum((double)prefs.getValue(i));
}
return(dev.getAverage() + dev.getStandardDeviation());
}
private static double computeThreshold(IPreferenceArray prefs)
{
if (prefs.Length() < 2)
{
// Not enough data points -- return a threshold that allows everything
return(Double.NegativeInfinity);
}
IRunningAverageAndStdDev stdDev = new FullRunningAverageAndStdDev();
int size = prefs.Length();
for (int i = 0; i < size; i++)
{
stdDev.AddDatum(prefs.GetValue(i));
}
return(stdDev.GetAverage() + stdDev.GetStandardDeviation());
}
private double getEvaluation(FastByIDMap <IPreferenceArray> testPrefs, IRecommender recommender)
{
reset();
var estimateCallables = new List <Action>();
AtomicInteger noEstimateCounter = new AtomicInteger();
foreach (var entry in testPrefs.EntrySet())
{
estimateCallables.Add(() => {
var testUserID = entry.Key;
var prefs = entry.Value;
foreach (IPreference realPref in prefs)
{
float estimatedPreference = float.NaN;
try {
estimatedPreference = recommender.EstimatePreference(testUserID, realPref.GetItemID());
} catch (NoSuchUserException nsue) {
// It's possible that an item exists in the test data but not training data in which case
// NSEE will be thrown. Just ignore it and move on.
log.Info("User exists in test data but not training data: {}", testUserID);
} catch (NoSuchItemException nsie) {
log.Info("Item exists in test data but not training data: {}", realPref.GetItemID());
}
if (float.IsNaN(estimatedPreference))
{
noEstimateCounter.incrementAndGet();
}
else
{
estimatedPreference = capEstimatedPreference(estimatedPreference);
processOneEstimate(estimatedPreference, realPref);
}
}
});
// new PreferenceEstimateCallable(recommender, entry.Key, entry.Value, noEstimateCounter));
}
log.Info("Beginning evaluation of {} users", estimateCallables.Count);
IRunningAverageAndStdDev timing = new FullRunningAverageAndStdDev();
execute(estimateCallables, noEstimateCounter, timing);
return(computeFinalEvaluation());
}
public static LoadStatistics runLoad(Recommender recommender, int howMany)
{
DataModel model = recommender.getDataModel();
int num = model.getNumUsers();
double samplingRate = 1000.0 / ((double)num);
IEnumerator <long> enumerator = SamplingLongPrimitiveIterator.maybeWrapIterator(model.getUserIDs(), samplingRate);
if (enumerator.MoveNext())
{
recommender.recommend(enumerator.Current, howMany);
}
List <Action> callables = new List <Action>();
while (enumerator.MoveNext())
{
callables.Add(new Action(new LoadCallable(recommender, enumerator.Current).call));
}
AtomicInteger noEstimateCounter = new AtomicInteger();
RunningAverageAndStdDev timing = new FullRunningAverageAndStdDev();
AbstractDifferenceRecommenderEvaluator.execute(callables, noEstimateCounter, timing);
return(new LoadStatistics(timing));
}
public static LoadStatistics runLoad(IRecommender recommender, int howMany)
{
IDataModel dataModel = recommender.GetDataModel();
int numUsers = dataModel.GetNumUsers();
double sampleRate = 1000.0 / numUsers;
var userSampler =
SamplinglongPrimitiveIterator.MaybeWrapIterator(dataModel.GetUserIDs(), sampleRate);
if (userSampler.MoveNext())
{
recommender.Recommend(userSampler.Current, howMany); // Warm up
}
var callables = new List <Action>();
while (userSampler.MoveNext())
{
callables.Add(new LoadCallable(recommender, userSampler.Current).call);
}
AtomicInteger noEstimateCounter = new AtomicInteger();
IRunningAverageAndStdDev timing = new FullRunningAverageAndStdDev();
AbstractDifferenceRecommenderEvaluator.execute(callables, noEstimateCounter, timing);
return(new LoadStatistics(timing));
}
