public void testGrow()
{
FastByIDMap<String> map = new FastByIDMap<String>(1,1);
map.Put(500000L, "alpha");
map.Put(47L, "bang");
Assert.IsNull(map.Get(500000L));
Assert.AreEqual("bang", map.Get(47L));
}
public virtual double[] getItemFeatures(long itemID)
{
int?index = itemIDMapping.Get(itemID);
if (index == null)
{
throw new NoSuchItemException(itemID);
}
return(itemFeatures[index.Value]);
}
public override IPreferenceArray GetPreferencesForItem(long itemID)
{
IPreferenceArray prefs = preferenceForItems.Get(itemID);
if (prefs == null)
{
throw new NoSuchItemException(itemID);
}
return(prefs);
}
public virtual double[] getUserFeatures(long userID)
{
int?index = userIDMapping.Get(userID);
if (index == null)
{
throw new NoSuchUserException(userID);
}
return(userFeatures[index.Value]);
}
public void testMaxSize()
{
FastByIDMap<String> map = new FastByIDMap<String>();
map.Put(4, "bang");
Assert.AreEqual(1, map.Count());
map.Put(47L, "bang");
Assert.AreEqual(2, map.Count());
Assert.IsNull(map.Get(500000L));
map.Put(47L, "buzz");
Assert.AreEqual(2, map.Count());
Assert.AreEqual("buzz", map.Get(47L));
}
protected void processLineWithoutID(String line,
FastByIDMap <FastIDSet> data,
FastByIDMap <FastByIDMap <DateTime?> > timestamps)
{
if (String.IsNullOrWhiteSpace(line) || line[0] == COMMENT_CHAR)
{
return;
}
var tokens = SplitLine(line);
string userIDString = tokens[0];
string itemIDString = tokens[1];
bool hasPreference = tokens.Length > 2;
string preferenceValueString = hasPreference ? tokens[2] : "";
bool hasTimestamp = tokens.Length > 3;
string timestampString = hasTimestamp ? tokens[3] : null;
long userID = readUserIDFromString(userIDString);
long itemID = readItemIDFromString(itemIDString);
if (transpose)
{
long tmp = userID;
userID = itemID;
itemID = tmp;
}
if (hasPreference && !hasTimestamp && String.IsNullOrEmpty(preferenceValueString))
{
// Then line is of form "userID,itemID,", meaning remove
FastIDSet itemIDs = data.Get(userID);
if (itemIDs != null)
{
itemIDs.Remove(itemID);
}
removeTimestamp(userID, itemID, timestamps);
}
else
{
FastIDSet itemIDs = data.Get(userID);
if (itemIDs == null)
{
itemIDs = new FastIDSet(2);
data.Put(userID, itemIDs);
}
itemIDs.Add(itemID);
addTimestamp(userID, itemID, timestampString, timestamps);
}
}
/// <p>
/// Returns the similarity between two items. Note that similarity is assumed to be symmetric, that
/// {@code itemSimilarity(item1, item2) == itemSimilarity(item2, item1)}, and that
/// {@code itemSimilarity(item1,item1) == 1.0} for all items.
/// </p>
///
/// @param itemID1
/// first item
/// @param itemID2
/// second item
/// @return similarity between the two
public double ItemSimilarity(long itemID1, long itemID2)
{
if (itemID1 == itemID2)
{
return(1.0);
}
long firstID;
long secondID;
if (itemID1 < itemID2)
{
firstID = itemID1;
secondID = itemID2;
}
else
{
firstID = itemID2;
secondID = itemID1;
}
FastByIDMap <double?> nextMap = similarityMaps.Get(firstID);
if (nextMap == null)
{
return(Double.NaN);
}
double?similarity = nextMap.Get(secondID);
return(!similarity.HasValue ? Double.NaN : similarity.Value);
}
public double UserSimilarity(long userID1, long userID2)
{
if (userID1 == userID2)
{
return(1.0);
}
long first;
long second;
if (userID1 < userID2)
{
first = userID1;
second = userID2;
}
else
{
first = userID2;
second = userID1;
}
FastByIDMap <Double> nextMap = similarityMaps.Get(first);
if (nextMap == null)
{
return(Double.NaN);
}
Double similarity = nextMap.Get(second);
return(similarity == null ? Double.NaN : similarity);
}
internal static IDataModel BuildModel(IList <UserItem> userItems, bool isReviewBased)
{
FastByIDMap <IList <IPreference> > userPreferencesMap = new FastByIDMap <IList <IPreference> >();
foreach (var userItem in userItems)
{
var userPreferences = userPreferencesMap.Get(userItem.UserId);
if (userPreferences == null)
{
userPreferences = new List <IPreference>(3);
userPreferencesMap.Put(userItem.UserId, userPreferences);
}
if (isReviewBased)
{
userPreferences.Add(new GenericPreference(userItem.UserId, userItem.ItemId, userItem.Rating));
}
else
{
userPreferences.Add(new BooleanPreference(userItem.UserId, userItem.ItemId));
}
}
var resultUserPreferences = new FastByIDMap <IPreferenceArray>(userPreferencesMap.Count());
foreach (var entry in userPreferencesMap.EntrySet())
{
var prefList = (List <IPreference>)entry.Value;
resultUserPreferences.Put(entry.Key, isReviewBased ?
new GenericUserPreferenceArray(prefList) :
(IPreferenceArray) new BooleanUserPreferenceArray(prefList));
}
return(new GenericDataModel(resultUserPreferences));
}
/// <summary>
/// Creates a new <see cref="GenericDataModel"/> from the given users (and their preferences). This
/// <see cref="IDataModel"/> retains all this information in memory and is effectively immutable.
/// </summary>
/// <param name="userData">users to include; (see also <see cref="GenericDataModel.ToDataMap(FastByIDMap, bool)"/>)</param>
/// <param name="timestamps">timestamps optionally, provided timestamps of preferences as milliseconds since the epoch. User IDs are mapped to maps of item IDs to long timestamps.</param>
public GenericDataModel(FastByIDMap<IPreferenceArray> userData, FastByIDMap<FastByIDMap<DateTime?>> timestamps) {
//Preconditions.checkArgument(userData != null, "userData is null");
this.preferenceFromUsers = userData;
FastByIDMap<IList<IPreference>> prefsForItems = new FastByIDMap<IList<IPreference>>();
FastIDSet itemIDSet = new FastIDSet();
int currentCount = 0;
float maxPrefValue = float.NegativeInfinity;
float minPrefValue = float.PositiveInfinity;
foreach (var entry in preferenceFromUsers.EntrySet()) {
IPreferenceArray prefs = entry.Value;
prefs.SortByItem();
foreach (IPreference preference in prefs) {
long itemID = preference.GetItemID();
itemIDSet.Add(itemID);
var prefsForItem = prefsForItems.Get(itemID);
if (prefsForItem == null) {
prefsForItem = new List<IPreference>(2);
prefsForItems.Put(itemID, prefsForItem);
}
prefsForItem.Add(preference);
float value = preference.GetValue();
if (value > maxPrefValue) {
maxPrefValue = value;
}
if (value < minPrefValue) {
minPrefValue = value;
}
}
if (++currentCount % 10000 == 0) {
log.Info("Processed {0} users", currentCount);
}
}
log.Info("Processed {0} users", currentCount);
setMinPreference(minPrefValue);
setMaxPreference(maxPrefValue);
this.itemIDs = itemIDSet.ToArray();
itemIDSet = null; // Might help GC -- this is big
Array.Sort(itemIDs);
this.preferenceForItems = ToDataMap(prefsForItems, false);
foreach (var entry in preferenceForItems.EntrySet()) {
entry.Value.SortByUser();
}
this.userIDs = new long[userData.Count()];
int i = 0;
foreach (var v in userData.Keys) {
userIDs[i++] = v;
}
Array.Sort(userIDs);
this.timestamps = timestamps;
}
public void testClear()
{
FastByIDMap<long?> map = new FastByIDMap<long?>();
map.Put(500000L, 2L);
map.Clear();
Assert.AreEqual(0, map.Count());
Assert.True(map.IsEmpty());
Assert.IsNull(map.Get(500000L));
}
/// @throws NoSuchUserException
/// if there is no such user
public override IPreferenceArray GetPreferencesFromUser(long userID)
{
IPreferenceArray prefs = preferenceFromUsers.Get(userID);
if (prefs == null)
{
throw new NoSuchUserException(userID);
}
return(prefs);
}
public override IPreferenceArray GetPreferencesForItem(long itemID)
{
FastIDSet userIDs = preferenceForItems.Get(itemID);
if (userIDs == null)
{
throw new NoSuchItemException(itemID);
}
IPreferenceArray prefArray = new BooleanItemPreferenceArray(userIDs.Count());
int i = 0;
var it = userIDs.GetEnumerator();
while (it.MoveNext())
{
prefArray.SetUserID(i, it.Current);
prefArray.SetItemID(i, itemID);
i++;
}
return(prefArray);
}
private static void removeTimestamp(long userID,
long itemID,
FastByIDMap <FastByIDMap <DateTime?> > timestamps)
{
FastByIDMap <DateTime?> itemTimestamps = timestamps.Get(userID);
if (itemTimestamps != null)
{
itemTimestamps.Remove(itemID);
}
}
protected int itemIndex(long itemID)
{
int?itemIndex = itemIDMapping.Get(itemID);
if (itemIndex == null)
{
itemIndex = itemIDMapping.Count();
itemIDMapping.Put(itemID, itemIndex);
}
return(itemIndex.Value);
}
/// @throws NoSuchUserException
/// if there is no such user
public override IPreferenceArray GetPreferencesFromUser(long userID)
{
FastIDSet itemIDs = preferenceFromUsers.Get(userID);
if (itemIDs == null)
{
throw new NoSuchUserException(userID);
}
IPreferenceArray prefArray = new BooleanUserPreferenceArray(itemIDs.Count());
int i = 0;
var it = itemIDs.GetEnumerator();
while (it.MoveNext())
{
prefArray.SetUserID(i, userID);
prefArray.SetItemID(i, it.Current);
i++;
}
return(prefArray);
}
protected int userIndex(long userID)
{
int?userIndex = userIDMapping.Get(userID);
if (userIndex == null)
{
userIndex = userIDMapping.Count();
userIDMapping.Put(userID, userIndex);
}
return(userIndex.Value);
}
private static void addDatumAndCreateIfNeeded(long itemID, float value, FastByIDMap <IRunningAverage> averages)
{
IRunningAverage itemAverage = averages.Get(itemID);
if (itemAverage == null)
{
itemAverage = new FullRunningAverage();
averages.Put(itemID, itemAverage);
}
itemAverage.AddDatum(value);
}
public void testPreferenceShufflerWithSyntheticData()
{
setUpSyntheticData();
ParallelSGDFactorizer.PreferenceShuffler shuffler = new ParallelSGDFactorizer.PreferenceShuffler(dataModel);
shuffler.shuffle();
shuffler.stage();
FastByIDMap <FastByIDMap <bool?> > checkedLst = new FastByIDMap <FastByIDMap <bool?> >();
for (int i = 0; i < shuffler.size(); i++)
{
IPreference pref = shuffler.get(i);
float?value = dataModel.GetPreferenceValue(pref.GetUserID(), pref.GetItemID());
Assert.AreEqual(pref.GetValue(), value.Value, 0.0);
if (!checkedLst.ContainsKey(pref.GetUserID()))
{
checkedLst.Put(pref.GetUserID(), new FastByIDMap <bool?>());
}
Assert.IsNull(checkedLst.Get(pref.GetUserID()).Get(pref.GetItemID()));
checkedLst.Get(pref.GetUserID()).Put(pref.GetItemID(), true);
}
var userIDs = dataModel.GetUserIDs();
int index = 0;
while (userIDs.MoveNext())
{
long userID = userIDs.Current;
IPreferenceArray preferencesFromUser = dataModel.GetPreferencesFromUser(userID);
foreach (IPreference preference in preferencesFromUser)
{
Assert.True(checkedLst.Get(preference.GetUserID()).Get(preference.GetItemID()).Value);
index++;
}
}
Assert.AreEqual(index, shuffler.size());
}
public override DateTime?GetPreferenceTime(long userID, long itemID)
{
if (timestamps == null)
{
return(null);
}
var itemTimestamps = timestamps.Get(userID);
if (itemTimestamps == null)
{
throw new NoSuchUserException(userID);
}
return(itemTimestamps.Get(itemID));
}
private void addTimestamp(long userID,
long itemID,
string timestampString,
FastByIDMap <FastByIDMap <DateTime?> > timestamps)
{
if (timestampString != null)
{
FastByIDMap <DateTime?> itemTimestamps = timestamps.Get(userID);
if (itemTimestamps == null)
{
itemTimestamps = new FastByIDMap <DateTime?>();
timestamps.Put(userID, itemTimestamps);
}
var timestamp = readTimestampFromString(timestampString);
itemTimestamps.Put(itemID, timestamp);
}
}
/// <p>
/// Creates a new {@link GenericDataModel} from the given users (and their preferences). This
/// {@link DataModel} retains all this information in memory and is effectively immutable.
/// </p>
///
/// @param userData users to include
/// @param timestamps optionally, provided timestamps of preferences as milliseconds since the epoch.
/// User IDs are mapped to maps of item IDs to long timestamps.
public GenericBooleanPrefDataModel(FastByIDMap <FastIDSet> userData, FastByIDMap <FastByIDMap <DateTime?> > timestamps)
{
//Preconditions.checkArgument(userData != null, "userData is null");
this.preferenceFromUsers = userData;
this.preferenceForItems = new FastByIDMap <FastIDSet>();
FastIDSet itemIDSet = new FastIDSet();
foreach (var entry in preferenceFromUsers.EntrySet())
{
long userID = entry.Key;
FastIDSet itemIDs1 = entry.Value;
itemIDSet.AddAll(itemIDs1);
var it = itemIDs1.GetEnumerator();
while (it.MoveNext())
{
long itemID = it.Current;
FastIDSet userIDs1 = preferenceForItems.Get(itemID);
if (userIDs1 == null)
{
userIDs1 = new FastIDSet(2);
preferenceForItems.Put(itemID, userIDs1);
}
userIDs1.Add(userID);
}
}
this.itemIDs = itemIDSet.ToArray();
itemIDSet = null; // Might help GC -- this is big
Array.Sort(itemIDs);
this.userIDs = new long[userData.Count()];
int i = 0;
var it1 = userData.Keys.GetEnumerator();
while (it1.MoveNext())
{
userIDs[i++] = it1.Current;
}
Array.Sort(userIDs);
this.timestamps = timestamps;
}
public IDataModel Load()
{
var hasPrefVal = !String.IsNullOrEmpty(PrefValFld);
FastByIDMap <IList <IPreference> > data = new FastByIDMap <IList <IPreference> >();
using (var dbRdr = SelectCmd.ExecuteReader()) {
while (dbRdr.Read())
{
long userID = Convert.ToInt64(dbRdr[UserIdFld]);
long itemID = Convert.ToInt64(dbRdr[ItemIdFld]);
var userPrefs = data.Get(userID);
if (userPrefs == null)
{
userPrefs = new List <IPreference>(3);
data.Put(userID, userPrefs);
}
if (hasPrefVal)
{
var prefVal = Convert.ToSingle(dbRdr[PrefValFld]);
userPrefs.Add(new GenericPreference(userID, itemID, prefVal));
}
else
{
userPrefs.Add(new BooleanPreference(userID, itemID));
}
}
}
var newData = new FastByIDMap <IPreferenceArray>(data.Count());
foreach (var entry in data.EntrySet())
{
var prefList = (List <IPreference>)entry.Value;
newData.Put(entry.Key, hasPrefVal ?
(IPreferenceArray) new GenericUserPreferenceArray(prefList) :
(IPreferenceArray) new BooleanUserPreferenceArray(prefList));
}
return(new GenericDataModel(newData));
}
private IDataModel GetDataModel()
{
var cacheKey = "RecommenderDataModel";
IDataModel dataModel = _memoryCache.Get <IDataModel>(cacheKey);
if (dataModel != null)
{
return(dataModel);
}
var movieRatings = _unitOfWork.MovieRatingRepository.GetAll();
FastByIDMap <IList <IPreference> > data = new FastByIDMap <IList <IPreference> >();
foreach (var movieRating in movieRatings)
{
var userPreferences = data.Get(movieRating.UserId);
if (userPreferences == null)
{
userPreferences = new List <IPreference>(3);
data.Put(movieRating.UserId, userPreferences);
}
userPreferences.Add(new BooleanPreference(movieRating.UserId, movieRating.MovieId));
}
var newData = new FastByIDMap <IPreferenceArray>(data.Count());
foreach (var entry in data.EntrySet())
{
var prefList = (List <IPreference>)entry.Value;
newData.Put(entry.Key, (IPreferenceArray) new BooleanUserPreferenceArray(prefList));
}
dataModel = new GenericDataModel(newData);
_memoryCache.Set(cacheKey, dataModel);
return(new GenericDataModel(newData));
}
/// <p>
/// Creates a new {@link GenericDataModel} from the given users (and their preferences). This
/// {@link DataModel} retains all this information in memory and is effectively immutable.
/// </p>
///
/// @param userData users to include
/// @param timestamps optionally, provided timestamps of preferences as milliseconds since the epoch.
/// User IDs are mapped to maps of item IDs to long timestamps.
public GenericBooleanPrefDataModel(FastByIDMap<FastIDSet> userData, FastByIDMap<FastByIDMap<DateTime?>> timestamps) {
//Preconditions.checkArgument(userData != null, "userData is null");
this.preferenceFromUsers = userData;
this.preferenceForItems = new FastByIDMap<FastIDSet>();
FastIDSet itemIDSet = new FastIDSet();
foreach (var entry in preferenceFromUsers.EntrySet()) {
long userID = entry.Key;
FastIDSet itemIDs1 = entry.Value;
itemIDSet.AddAll(itemIDs1);
var it = itemIDs1.GetEnumerator();
while (it.MoveNext()) {
long itemID = it.Current;
FastIDSet userIDs1 = preferenceForItems.Get(itemID);
if (userIDs1 == null) {
userIDs1 = new FastIDSet(2);
preferenceForItems.Put(itemID, userIDs1);
}
userIDs1.Add(userID);
}
}
this.itemIDs = itemIDSet.ToArray();
itemIDSet = null; // Might help GC -- this is big
Array.Sort(itemIDs);
this.userIDs = new long[userData.Count()];
int i = 0;
var it1 = userData.Keys.GetEnumerator();
while (it1.MoveNext()) {
userIDs[i++] = it1.Current;
}
Array.Sort(userIDs);
this.timestamps = timestamps;
}
public void testVersusHashMap()
{
FastByIDMap<String> actual = new FastByIDMap<String>();
IDictionary<long, string> expected = new Dictionary<long,string>(1000000);
var r = RandomUtils.getRandom();
for (int i = 0; i < 1000000; i++) {
double d = r.nextDouble();
long key = (long) r.nextInt(100);
if (d < 0.4) {
Assert.AreEqual( expected.ContainsKey(key)?expected[key]:null, actual.Get(key));
} else {
if (d < 0.7) {
var expectedOldVal = expected.ContainsKey(key) ? expected[key] : null;
expected[key] = "bang";
Assert.AreEqual(expectedOldVal, actual.Put(key, "bang"));
} else {
var expectedOldVal = expected.ContainsKey(key) ? expected[key] : null;
expected.Remove(key);
Assert.AreEqual(expectedOldVal, actual.Remove(key));
}
Assert.AreEqual(expected.Count, actual.Count());
Assert.AreEqual(expected.Count==0, actual.IsEmpty());
}
}
}
public void testPutAndGet()
{
FastByIDMap<long?> map = new FastByIDMap<long?>();
Assert.IsNull(map.Get(500000L));
map.Put(500000L, 2L);
Assert.AreEqual(2L, (long) map.Get(500000L));
}
/// <summary>
/// Creates a new <see cref="GenericDataModel"/> from the given users (and their preferences). This
/// <see cref="IDataModel"/> retains all this information in memory and is effectively immutable.
/// </summary>
/// <param name="userData">users to include; (see also <see cref="GenericDataModel.ToDataMap(FastByIDMap, bool)"/>)</param>
/// <param name="timestamps">timestamps optionally, provided timestamps of preferences as milliseconds since the epoch. User IDs are mapped to maps of item IDs to long timestamps.</param>
public GenericDataModel(FastByIDMap <IPreferenceArray> userData, FastByIDMap <FastByIDMap <DateTime?> > timestamps)
{
//Preconditions.checkArgument(userData != null, "userData is null");
this.preferenceFromUsers = userData;
FastByIDMap <IList <IPreference> > prefsForItems = new FastByIDMap <IList <IPreference> >();
FastIDSet itemIDSet = new FastIDSet();
int currentCount = 0;
float maxPrefValue = float.NegativeInfinity;
float minPrefValue = float.PositiveInfinity;
foreach (var entry in preferenceFromUsers.EntrySet())
{
IPreferenceArray prefs = entry.Value;
prefs.SortByItem();
foreach (IPreference preference in prefs)
{
long itemID = preference.GetItemID();
itemIDSet.Add(itemID);
var prefsForItem = prefsForItems.Get(itemID);
if (prefsForItem == null)
{
prefsForItem = new List <IPreference>(2);
prefsForItems.Put(itemID, prefsForItem);
}
prefsForItem.Add(preference);
float value = preference.GetValue();
if (value > maxPrefValue)
{
maxPrefValue = value;
}
if (value < minPrefValue)
{
minPrefValue = value;
}
}
if (++currentCount % 10000 == 0)
{
log.Info("Processed {0} users", currentCount);
}
}
log.Info("Processed {0} users", currentCount);
setMinPreference(minPrefValue);
setMaxPreference(maxPrefValue);
this.itemIDs = itemIDSet.ToArray();
itemIDSet = null; // Might help GC -- this is big
Array.Sort(itemIDs);
this.preferenceForItems = ToDataMap(prefsForItems, false);
foreach (var entry in preferenceForItems.EntrySet())
{
entry.Value.SortByUser();
}
this.userIDs = new long[userData.Count()];
int i = 0;
foreach (var v in userData.Keys)
{
userIDs[i++] = v;
}
Array.Sort(userIDs);
this.timestamps = timestamps;
}
public void testPreferenceShufflerWithSyntheticData()
{
setUpSyntheticData();
ParallelSGDFactorizer.PreferenceShuffler shuffler = new ParallelSGDFactorizer.PreferenceShuffler(dataModel);
shuffler.shuffle();
shuffler.stage();
FastByIDMap<FastByIDMap<bool?>> checkedLst = new FastByIDMap<FastByIDMap<bool?>>();
for (int i = 0; i < shuffler.size(); i++) {
IPreference pref=shuffler.get(i);
float? value = dataModel.GetPreferenceValue(pref.GetUserID(), pref.GetItemID());
Assert.AreEqual(pref.GetValue(), value.Value, 0.0);
if (!checkedLst.ContainsKey(pref.GetUserID())) {
checkedLst.Put(pref.GetUserID(), new FastByIDMap<bool?>());
}
Assert.IsNull(checkedLst.Get(pref.GetUserID()).Get(pref.GetItemID()));
checkedLst.Get(pref.GetUserID()).Put(pref.GetItemID(), true);
}
var userIDs = dataModel.GetUserIDs();
int index=0;
while (userIDs.MoveNext()) {
long userID = userIDs.Current;
IPreferenceArray preferencesFromUser = dataModel.GetPreferencesFromUser(userID);
foreach (IPreference preference in preferencesFromUser) {
Assert.True(checkedLst.Get(preference.GetUserID()).Get(preference.GetItemID()).Value);
index++;
}
}
Assert.AreEqual(index, shuffler.size());
}
/// <p>
/// Reads one line from the input file and adds the data to a {@link FastByIDMap} data structure which maps user IDs
/// to preferences. This assumes that each line of the input file corresponds to one preference. After
/// reading a line and determining which user and item the preference pertains to, the method should look to
/// see if the data contains a mapping for the user ID already, and if not, add an empty data structure of preferences
/// as appropriate to the data.
/// </p>
///
/// <p>
/// Note that if the line is empty or begins with '#' it will be ignored as a comment.
/// </p>
///
/// @param line
/// line from input data file
/// @param data
/// all data read so far, as a mapping from user IDs to preferences
/// @param fromPriorData an implementation detail -- if true, data will map IDs to
/// {@link PreferenceArray} since the framework is attempting to read and update raw
/// data that is already in memory. Otherwise it maps to {@link Collection}s of
/// {@link Preference}s, since it's reading fresh data. Subclasses must be prepared
/// to handle this wrinkle.
protected void processLine <T>(string line,
FastByIDMap <T> data,
FastByIDMap <FastByIDMap <DateTime?> > timestamps,
bool fromPriorData)
{
// Ignore empty lines and comments
if (line.Length == 0 || line[0] == COMMENT_CHAR)
{
return;
}
var tokens = SplitLine(line);
string userIDString = tokens[0];
string itemIDString = tokens[1];
string preferenceValueString = tokens[2];
bool hasTimestamp = tokens.Length > 3;
string timestampString = hasTimestamp ? tokens[3] : null;
long userID = readUserIDFromString(userIDString);
long itemID = readItemIDFromString(itemIDString);
if (transpose)
{
long tmp = userID;
userID = itemID;
itemID = tmp;
}
// This is kind of gross but need to handle two types of storage
var maybePrefs = data.Get(userID);
if (fromPriorData)
{
// Data are PreferenceArray
IPreferenceArray prefs = (IPreferenceArray)maybePrefs;
if (!hasTimestamp && String.IsNullOrWhiteSpace(preferenceValueString))
{
// Then line is of form "userID,itemID,", meaning remove
if (prefs != null)
{
bool exists = false;
int length = prefs.Length();
for (int i = 0; i < length; i++)
{
if (prefs.GetItemID(i) == itemID)
{
exists = true;
break;
}
}
if (exists)
{
if (length == 1)
{
data.Remove(userID);
}
else
{
IPreferenceArray newPrefs = new GenericUserPreferenceArray(length - 1);
for (int i = 0, j = 0; i < length; i++, j++)
{
if (prefs.GetItemID(i) == itemID)
{
j--;
}
else
{
newPrefs.Set(j, prefs.Get(i));
}
}
data.Put(userID, (T)newPrefs);
}
}
}
removeTimestamp(userID, itemID, timestamps);
}
else
{
float preferenceValue = float.Parse(preferenceValueString, CultureInfo.InvariantCulture);
bool exists = false;
if (uniqueUserItemCheck && prefs != null)
{
for (int i = 0; i < prefs.Length(); i++)
{
if (prefs.GetItemID(i) == itemID)
{
exists = true;
prefs.SetValue(i, preferenceValue);
break;
}
}
}
if (!exists)
{
if (prefs == null)
{
prefs = new GenericUserPreferenceArray(1);
}
else
{
IPreferenceArray newPrefs = new GenericUserPreferenceArray(prefs.Length() + 1);
for (int i = 0, j = 1; i < prefs.Length(); i++, j++)
{
newPrefs.Set(j, prefs.Get(i));
}
prefs = newPrefs;
}
prefs.SetUserID(0, userID);
prefs.SetItemID(0, itemID);
prefs.SetValue(0, preferenceValue);
data.Put(userID, (T)prefs);
}
}
addTimestamp(userID, itemID, timestampString, timestamps);
}
else
{
// Data are IEnumerable<Preference>
IEnumerable <IPreference> prefs = ((IEnumerable <IPreference>)maybePrefs);
if (!hasTimestamp && String.IsNullOrWhiteSpace(preferenceValueString))
{
// Then line is of form "userID,itemID,", meaning remove
if (prefs != null)
{
// remove pref
var prefsIterator = ((IEnumerable <IPreference>)prefs.ToArray()).GetEnumerator();
while (prefsIterator.MoveNext())
{
IPreference pref = prefsIterator.Current;
if (pref.GetItemID() == itemID)
{
if (prefs is IList <IPreference> )
{
((IList <IPreference>)maybePrefs).Remove(pref);// prefsIterator.remove()
}
break;
}
}
}
removeTimestamp(userID, itemID, timestamps);
}
else
{
float preferenceValue = float.Parse(preferenceValueString, CultureInfo.InvariantCulture);
bool exists = false;
if (uniqueUserItemCheck && prefs != null)
{
foreach (IPreference pref in prefs)
{
if (pref.GetItemID() == itemID)
{
exists = true;
pref.SetValue(preferenceValue);
break;
}
}
}
if (!exists)
{
if (prefs == null)
{
prefs = new List <IPreference>(5);
data.Put(userID, (T)prefs);
}
if (prefs is IList <IPreference> )
{
((IList <IPreference>)prefs).Add(new GenericPreference(userID, itemID, preferenceValue));
}
}
addTimestamp(userID, itemID, timestampString, timestamps);
}
}
}