public void ParseAValidCatalogFileTest()
        {
            const int    catalogItemsCount = 20;
            const string baseFolder        = nameof(ParseAValidCatalogFileTest);

            Directory.CreateDirectory(baseFolder);

            var    generator   = new ModelTrainingFilesGenerator(itemsCount: catalogItemsCount);
            string catalogFile = Path.Combine(baseFolder, "catalog.csv");

            generator.CreateCatalogFile(catalogFile);

            IList <SarCatalogItem> catalogItems;

            string[]          featureNames;
            var               parser = new CatalogFileParser();
            FileParsingReport report =
                parser.ParseCatalogFile(catalogFile, CancellationToken.None, out catalogItems, out featureNames);

            Assert.IsNotNull(report);
            Assert.IsTrue(report.IsCompletedSuccessfuly);
            Assert.AreEqual(catalogItemsCount, report.SuccessfulLinesCount);
            Assert.AreEqual(catalogItemsCount, report.TotalLinesCount);
            Assert.IsTrue(report.Errors == null || !report.Errors.Any());
        }
Ejemplo n.º 2
0
        public void TrainModelWithRangeOfPossibleParametersTest()
        {
            const string baseFolder = nameof(TrainModelWithRangeOfPossibleParametersTest);

            Directory.CreateDirectory(baseFolder);

            var generator = new ModelTrainingFilesGenerator();

            // create catalog items
            IList <SarCatalogItem> catalogItems;

            string[] featureNames;
            string   catalogFilePath = Path.Combine(baseFolder, "catalog.csv");

            generator.CreateCatalogFile(catalogFilePath);
            var itemIdsIndex  = new ConcurrentDictionary <string, uint>();
            var catalogParser = new CatalogFileParser(0, itemIdsIndex);
            var parsingReport = catalogParser.ParseCatalogFile(catalogFilePath, CancellationToken.None, out catalogItems, out featureNames);

            Assert.IsTrue(parsingReport.IsCompletedSuccessfuly);

            // create usage items
            IList <SarUsageEvent> usageEvents;
            string usageFileFolderPath = Path.Combine(baseFolder, "usage");

            Directory.CreateDirectory(usageFileFolderPath);
            generator.CreateUsageFile(Path.Combine(usageFileFolderPath, "usage.csv"), 10000);
            var userIdsIndex     = new ConcurrentDictionary <string, uint>();
            var usageFilesParser = new UsageEventsFilesParser(itemIdsIndex, userIdsIndex);

            parsingReport = usageFilesParser.ParseUsageEventFiles(usageFileFolderPath, CancellationToken.None, out usageEvents);
            Assert.IsTrue(parsingReport.IsCompletedSuccessfuly);

            int count      = 0;
            var sarTrainer = new SarTrainer();
            IDictionary <string, double> catalogFeatureWeights;

            foreach (IModelTrainerSettings settings in GetAllModelTrainingParameters())
            {
                IPredictorModel model = sarTrainer.Train(settings, usageEvents, catalogItems, featureNames, userIdsIndex.Count, itemIdsIndex.Count, out catalogFeatureWeights);
                Assert.IsNotNull(model, $"Expected training to complete successfully when using settings#{count}: {settings}");
                count++;
            }
        }
        /// <summary>
        /// Trains a model using the input files
        /// </summary>
        /// <param name="settings">The trainer settings</param>
        /// <param name="workFolderPath">A temp work folder for storing intermediate files</param>
        /// <param name="usageFolderPath">The path to the folder of usage files</param>
        /// <param name="catalogFilePath">The path to the catalog file</param>
        /// <param name="evaluationFolderPath">The path to the evaluation file (optional) </param>
        /// <param name="cancellationToken">A cancellation token used to abort the training</param>
        private ModelTrainResult TrainModelInternal(IModelTrainerSettings settings, string workFolderPath, string usageFolderPath,
                                                    string catalogFilePath, string evaluationFolderPath, CancellationToken cancellationToken)
        {
            var duration = ModelTraininigDuration.Start();
            var result   = new ModelTrainResult {
                Duration = duration
            };

            var userIdsIndexMap = new ConcurrentDictionary <string, uint>();
            var itemIdsIndexMap = new ConcurrentDictionary <string, uint>();

            // parse the catalog file
            IList <SarCatalogItem> catalogItems = null;

            string[] catalogFeatureNames = null;
            if (!string.IsNullOrWhiteSpace(catalogFilePath) && File.Exists(catalogFilePath))
            {
                // report progress
                _progressMessageReportDelegate("Parsing Catalog File");

                // create a catalog file parser
                var catalogParser = new CatalogFileParser(MaximumParsingErrorsCount, itemIdsIndexMap, _tracer);

                // parse the catalog file
                result.CatalogFilesParsingReport = catalogParser.ParseCatalogFile(catalogFilePath, cancellationToken,
                                                                                  out catalogItems, out catalogFeatureNames);

                // record the catalog parsing duration
                duration.SetCatalogParsingDuration();
                _tracer.TraceInformation($"Catalog parsing completed in {duration.CatalogParsingDuration.TotalMinutes} minutes");

                // get the catalog items count
                result.CatalogItemsCount = catalogItems.Count;

                // fail the training if parsing had failed or yielded no items
                if (!result.CatalogFilesParsingReport.IsCompletedSuccessfuly || !catalogItems.Any())
                {
                    result.CompletionMessage = "Failed to parse catalog file or parsing found no valid items";
                    _tracer.TraceInformation(result.CompletionMessage);
                    return(result);
                }

                // clear the catalog items list if it's not used anymore
                if (!settings.EnableColdItemPlacement)
                {
                    catalogItems.Clear();
                }
            }

            // report progress
            _progressMessageReportDelegate("Parsing Usage Events Files");

            // create a usage events files parser that skips events of unknown item ids (if catalog was provided))
            var usageEventsParser = new UsageEventsFilesParser(itemIdsIndexMap, userIdsIndexMap,
                                                               MaximumParsingErrorsCount, catalogItems != null, _tracer);

            _tracer.TraceInformation("Parsing the usage event files");
            IList <SarUsageEvent> usageEvents;

            result.UsageFilesParsingReport =
                usageEventsParser.ParseUsageEventFiles(usageFolderPath, cancellationToken, out usageEvents);

            // record the usage files parsing duration
            duration.SetUsageFilesParsingDuration();
            _tracer.TraceInformation($"Usage file(s) parsing completed in {duration.UsageFilesParsingDuration.TotalMinutes} minutes");

            // fail the training if parsing had failed or yielded no events
            if (!result.UsageFilesParsingReport.IsCompletedSuccessfuly || !usageEvents.Any())
            {
                result.CompletionMessage = "Failed to parse usage file(s) or parsing found no valid items";
                _tracer.TraceInformation(result.CompletionMessage);
                return(result);
            }

            _tracer.TraceInformation($"Found {userIdsIndexMap.Count} unique users");
            result.UniqueUsersCount = userIdsIndexMap.Count;

            _tracer.TraceInformation($"Found {itemIdsIndexMap.Count} unique items");
            result.UniqueItemsCount = usageEvents.Select(x => x.ItemId).Distinct().Count();

            _tracer.TraceInformation("Extracting the indexed item ids from the item index map");
            string[] itemIdsIndex = itemIdsIndexMap.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray();

            _tracer.TraceInformation($"Sorting the usage events based on the cooccurrenceUnit unit ({settings.CooccurrenceUnit})");
            switch (settings.CooccurrenceUnit)
            {
            case CooccurrenceUnit.User:
                usageEvents = usageEvents.OrderBy(x => x.UserId).ToArray();
                break;

            case CooccurrenceUnit.Timestamp:
                usageEvents = usageEvents.OrderBy(x => x.Timestamp).ThenBy(x => x.UserId).ToArray();
                break;
            }

            _tracer.TraceInformation("Finished sorting usage events.");

            Stopwatch storeUserHistoryDuration = null;
            Task      storeUserHistoryTask     = null;

            if (settings.EnableUserToItemRecommendations && _userHistoryStore != null)
            {
                storeUserHistoryDuration = Stopwatch.StartNew();
                _tracer.TraceInformation($"Extracting the indexed user ids from the user index map ({userIdsIndexMap.Count:N} users)");
                string[] userIdsIndex = userIdsIndexMap.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray();

                _tracer.TraceInformation($"Asynchronously starting to store usage events per user (total of {usageEvents.Count:N} items)");
                storeUserHistoryTask = Task.Run(() =>
                                                _userHistoryStore.StoreUserHistoryEventsAsync(usageEvents, userIdsIndex, cancellationToken), cancellationToken);
            }

            // if provided, parse the evaluation usage event files
            int    evaluationUsageEventsCount          = 0;
            string parsedEvaluationUsageEventsFilePath = null;

            if (!string.IsNullOrWhiteSpace(evaluationFolderPath) && Directory.Exists(evaluationFolderPath))
            {
                // report progress
                _progressMessageReportDelegate("Parsing Evaluation Usage Events Files");

                _tracer.TraceInformation("Parsing the evaluation usage event files");
                IList <SarUsageEvent> evaluationUsageEvents;
                result.EvaluationFilesParsingReport = usageEventsParser.ParseUsageEventFiles(evaluationFolderPath,
                                                                                             cancellationToken, out evaluationUsageEvents);

                if (result.EvaluationFilesParsingReport.IsCompletedSuccessfuly)
                {
                    // set the evaluation usage events count
                    evaluationUsageEventsCount = evaluationUsageEvents.Count;

                    _tracer.TraceInformation("Storing the parsed usage events for evaluation to reduce memory print");
                    parsedEvaluationUsageEventsFilePath = Path.Combine(workFolderPath, Path.GetTempFileName());
                    File.WriteAllLines(parsedEvaluationUsageEventsFilePath,
                                       evaluationUsageEvents.Select(JsonConvert.SerializeObject));
                }
                else
                {
                    _tracer.TraceWarning("Skipping model evaluation as it failed to parse evaluation usage files.");
                }

                // record the evaluation usage files parsing duration
                duration.SetEvaluationUsageFilesParsingDuration();
                _tracer.TraceInformation($"Evaluation usage file(s) parsing completed in {duration.EvaluationUsageFilesParsingDuration.TotalMinutes} minutes");
            }

            // clear the indices maps as they are no longer needed
            userIdsIndexMap.Clear();
            itemIdsIndexMap.Clear();

            cancellationToken.ThrowIfCancellationRequested();

            // report progress
            _progressMessageReportDelegate("Core Training");

            _tracer.TraceInformation("Training a new model using SAR trainer");
            IDictionary <string, double> catalogFeatureWeights;
            var             sarTrainer = new SarTrainer(_tracer);
            IPredictorModel sarModel   = sarTrainer.Train(settings, usageEvents, catalogItems, catalogFeatureNames, result.UniqueUsersCount,
                                                          result.CatalogItemsCount ?? result.UniqueItemsCount, out catalogFeatureWeights, cancellationToken);

            _tracer.TraceInformation("SAR training was completed.");

            // create the trained model properties
            var modelProperties = new ModelProperties
            {
                IncludeHistory     = settings.AllowSeedItemsInRecommendations,
                EnableUserAffinity = settings.EnableUserAffinity,
                IsUserToItemRecommendationsSupported = settings.EnableUserToItemRecommendations,
                Decay            = TimeSpan.FromDays(settings.DecayPeriodInDays),
                ReferenceDate    = usageEventsParser.MostRecentEventTimestamp,
                UniqueUsersCount = result.UniqueUsersCount,
            };

            // create the trained model
            result.Model = new TrainedModel(sarModel, modelProperties, itemIdsIndex);

            // set the catalog features weights
            result.CatalogFeatureWeights = catalogFeatureWeights;

            // record the core training duration
            duration.SetTrainingDuration();

            // run model evaluation if evaluation usage event are available
            if (evaluationUsageEventsCount > 0 && parsedEvaluationUsageEventsFilePath != null)
            {
                // report progress
                _progressMessageReportDelegate("Evaluating Trained Model");

                var evaluationUsageEvents = new List <SarUsageEvent>(evaluationUsageEventsCount);

                // load the evaluation usage events
                using (var reader = new StreamReader(parsedEvaluationUsageEventsFilePath))
                {
                    while (!reader.EndOfStream)
                    {
                        evaluationUsageEvents.Add(JsonConvert.DeserializeObject <SarUsageEvent>(reader.ReadLine()));
                    }
                }

                _tracer.TraceInformation("Starting model evaluation");
                var evaluator = new ModelEvaluator(_tracer);
                result.ModelMetrics = evaluator.Evaluate(result.Model, usageEvents, evaluationUsageEvents, cancellationToken);

                // record the evaluation duration
                duration.SetEvaluationDuration();
            }

            if (storeUserHistoryTask != null)
            {
                _tracer.TraceInformation("Waiting for storing of usage events per user (user history) to complete");

                if (!storeUserHistoryTask.IsCompleted)
                {
                    _progressMessageReportDelegate("Storing User History");

                    // set the reporting flag to true so usage history upload progress will get reported to model status
                    _reportUserHistoryProgress = true;
                }

                try
                {
                    storeUserHistoryTask.Wait(cancellationToken);
                    storeUserHistoryDuration?.Stop();
                    duration.StoringUserHistoryDuration = storeUserHistoryDuration?.Elapsed;
                    _tracer.TraceInformation(
                        $"Storing usage events per user (user history) to complete after {duration.StoringUserHistoryDuration.Value.TotalMinutes} minutes");
                }
                catch (AggregateException ex)
                {
                    var exception = new Exception("Exception while trying to store user history", ex);
                    _tracer.TraceError(exception.ToString());
                    throw exception;
                }
            }

            // stop measuring the duration and record the total duration
            duration.Stop();

            // return the train result
            result.CompletionMessage = "Model Training Completed Successfully";
            return(result);
        }