/// <summary>
/// Creates a new instance of the <see cref="Recommender"/> class.
/// </summary>
/// <param name="trainedModel">A trained model</param>
/// <param name="documentStore">A document store for storing user history is user-to-item is enabled</param>
/// <param name="tracer">A message tracer to use for logging</param>
public Recommender(ITrainedModel trainedModel, IDocumentStore documentStore = null, ITracer tracer = null)
{
if (trainedModel == null)
{
throw new ArgumentNullException(nameof(trainedModel));
}
if (trainedModel.Properties == null)
{
throw new ArgumentException("Trained model properties can not be null", nameof(trainedModel.Properties));
}
if (trainedModel.ItemIdIndex == null)
{
throw new ArgumentException("Trained model item index can not be null", nameof(trainedModel.ItemIdIndex));
}
// create a tracer if not provided
_tracer = tracer ?? new DefaultTracer();
// create a SAR scorer
_scorer = new SarScorer(trainedModel.RecommenderData?.Recommender, _tracer);
_properties = trainedModel.Properties;
_itemIdIndex = trainedModel.ItemIdIndex;
if (documentStore != null)
{
_userHistoryStore = new UserHistoryStore(documentStore, _properties.UniqueUsersCount, _tracer);
}
// create a reverse item id lookup
_itemIdReverseLookup =
_itemIdIndex.Select((id, index) => new { id, index })
.ToDictionary(x => x.id, x => (uint)x.index + 1);
}
/// <summary>
/// Gets or creates a <see cref="Recommender"/> for the input model
/// </summary>
private async Task <Recommender> GetOrCreateRecommenderAsync(Guid modelId, CancellationToken cancellationToken)
{
var key = modelId.ToString();
var recommender = _recommendersCache.Get(key) as Recommender;
if (recommender == null)
{
using (var stream = new MemoryStream())
{
Trace.TraceInformation($"Downloading the serialized trained model '{modelId}' from blob storage");
await DownloadTrainedModelAsync(modelId, stream, cancellationToken);
// rewind the stream
stream.Seek(0, SeekOrigin.Begin);
Trace.TraceInformation($"Deserializing the trained model '{modelId}'");
ITrainedModel trainedModel = DeserializeTrainedModel(stream, modelId);
// get the model's user history store
IDocumentStore modelDocumentStore = _documentStoreProvider.GetDocumentStore(modelId);
// create a recommender from the trained model
recommender = new Recommender(trainedModel, modelDocumentStore, new Tracer(nameof(Recommender)));
}
bool result = _recommendersCache.Add(key, recommender,
new CacheItemPolicy {
AbsoluteExpiration = DateTimeOffset.UtcNow.AddDays(1)
});
Trace.TraceVerbose($"Addition of model {modelId} recommender to the cache resulted with '{result}'");
}
return(recommender);
}
/// <summary>
/// Deserialize the input stream as a trained model
/// </summary>
private ITrainedModel DeserializeTrainedModel(Stream trainedModelStream, Guid modelId)
{
try
{
// create a zip stream, leaving the target stream open after disposing
using (var gzipStream = new GZipStream(trainedModelStream, CompressionMode.Decompress, leaveOpen: true))
{
// deserialize the stream
var binaryFormatter = new BinaryFormatter();
object deserializedObject = binaryFormatter.Deserialize(gzipStream);
ITrainedModel trainedModel = deserializedObject as ITrainedModel;
if (trainedModel == null)
{
throw new Exception(
$"Unexpected object type found in stream. Found type {deserializedObject.GetType().Name}");
}
return(trainedModel);
}
}
catch (Exception ex)
{
var exception = new Exception($"Failed deserializing trained model {modelId} from stream", ex);
Trace.TraceError(exception.ToString());
throw exception;
}
}
/// <summary>
/// Serializes the input trained model into a zip stream
/// </summary>
private void SerializeTrainedModel(ITrainedModel trainedModel, Stream targetStream, Guid modelId)
{
try
{
// create a zip stream, leaving the target stream open after disposing
using (var gzipStream = new GZipStream(targetStream, CompressionMode.Compress, leaveOpen: true))
{
// binary serialize the trained model to the stream
new BinaryFormatter().Serialize(gzipStream, trainedModel);
}
}
catch (Exception ex)
{
var exception = new Exception($"Failed serializing trained model {modelId} to stream", ex);
Trace.TraceError(exception.ToString());
throw exception;
}
}
/// <summary>
/// Scores the users in <see cref="usageEvents"/> who are also present in <see cref="evaluationUsageEvents"/>
/// </summary>
/// <param name="model">Model to be used for scoring</param>
/// <param name="usageEvents">List of usage events</param>
/// <param name="evaluationUsageEvents">List of evaluation events</param>
/// <returns></returns>
private IList <SarScoreResult> ScoreTestUsers(ITrainedModel model, IList <SarUsageEvent> usageEvents,
IList <SarUsageEvent> evaluationUsageEvents)
{
// Score the test users
var evaluationRecommender = new Recommender(model, null, _tracer);
// extract the user ids from the evaluation usage events
HashSet <uint> evaluationUsers = new HashSet <uint>(evaluationUsageEvents.Select(usageEvent => usageEvent.UserId));
// filter out usage events of users not found in the evaluation set
IEnumerable <SarUsageEvent> usageEventsFiltered =
usageEvents.Where(usageEvent => evaluationUsers.Contains(usageEvent.UserId));
// group usage event by user, and calculate score
List <SarScoreResult> scores =
usageEventsFiltered.GroupBy(usageEvent => usageEvent.UserId)
.SelectMany(group => evaluationRecommender.ScoreUsageEvents(group.ToList(), RecommendationCount))
.ToList();
return(scores);
}
/// <summary>
/// Computes Precision and Diversity metrics for given model, usage events it was trained on, and evaluation events it
/// would be evaluated on.
/// </summary>
/// <param name="model">Model to be used for scoring</param>
/// <param name="usageEvents">List of usage events</param>
/// <param name="evaluationUsageEvents">List of evaluation events</param>
/// <param name="cancellationToken">A cancellation token used to abort the evaluation</param>
/// <returns></returns>
public ModelMetrics Evaluate(ITrainedModel model, IList <SarUsageEvent> usageEvents,
IList <SarUsageEvent> evaluationUsageEvents, CancellationToken cancellationToken)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
if (usageEvents == null)
{
throw new ArgumentNullException(nameof(usageEvents));
}
if (evaluationUsageEvents == null)
{
throw new ArgumentNullException(nameof(evaluationUsageEvents));
}
// score the test usage events
IList <SarScoreResult> scores = ScoreTestUsers(model, usageEvents, evaluationUsageEvents);
// use the scoring result to compute precision and diversity
return(ComputeMetrics(usageEvents, evaluationUsageEvents, scores, cancellationToken));
}
