/// <summary> /// Parsed a TSV for RT data then adds the new data to existing IMDB OmdbEntrys in the db /// </summary> /// <param name="tom_filepath"></param> public static void OptimizedRtTsvParse(string tom_filepath) { //i don't think this is necessary, because you always want to update the RT data ////build a list of hashes //MovieDbContext tempDb = new MovieDbContext(); //List<int> omdbHashes = // tempDb.Omdb.Select(item => item.GetHashCode()) // .Distinct() // .ToList(); int num_of_RT_movies_per_loop = 5000; using ( CsvReader tom_csvReader = new CsvReader(new StreamReader(tom_filepath), true, '\t', '~', '`', '~', ValueTrimmingOptions.None)) { while (tom_csvReader.ReadNextRecord()) { //loop through the imdbtsv, creating a omdbentry for the first 5000 items List<OmdbEntry> new_tom_omdb_entries = new List<OmdbEntry>(); for (int i = 0; i < num_of_RT_movies_per_loop; i++) { //read the row and create an omdb from it parse the current TSV row var entry = Omdb.CreateOmdbEntryFromTsvRecord(tomReader: tom_csvReader); // add entry to a list new_tom_omdb_entries.Add(entry); //if nothing left to read, break out of the loop if (tom_csvReader.ReadNextRecord() == false) { Tools.TraceLine( "ReadNextRecord was false, breaking out of loop to save it"); break; } } //find all existing IMDB entries in db MovieDbContext db = new MovieDbContext(); db.Configuration.AutoDetectChangesEnabled = true; //find all existing OE that match the omdb_ids of the listed ones Tools.TraceLine("items in db.Omdb {0}", db.Omdb.Count()); //get the omdb_ids of the new RT omdbentrys List<int> tom_omdb_ids_to_match = new_tom_omdb_entries.Select(omdb => omdb.ombd_ID) .ToList(); //match the ids to the exist IMDB omdbentries var res = (from imdb in db.Omdb where tom_omdb_ids_to_match.Contains(imdb.ombd_ID) select imdb); //Tools.TraceLine("items in res {0}", res.Count()); List<OmdbEntry> matched_existing_imdb_omdbentrys = res.ToList(); //alter the existing IMDB entries and save the changes... foreach ( OmdbEntry matchedExistingImdbOmdbentry in matched_existing_imdb_omdbentrys) { //the RT omdb tha matches the imdb entry for the omdb_id var matching_RT_data = new_tom_omdb_entries.First( item => item.ombd_ID == matchedExistingImdbOmdbentry.ombd_ID); //updates the IMDB OmdbEntry with the RT OmdbEntry's information UpdateImdbEntryWithRtEntry(matchedExistingImdbOmdbentry, matching_RT_data); } //save the updated OmdbEntry information, dispose of the context db.SaveChanges(); db.Dispose(); Tools.TraceLine("Updated existing IMDB OmdbEntrys, done saving"); } } }
protected override void Dispose(bool disposing) { MovieDbContext db = new MovieDbContext(); db.Dispose(); base.Dispose(disposing); }
/// <summary> /// Adds Title, Art, Year, Runtime, Genres /// </summary> /// <param name="createdTitle"></param> public static void AddPrimaryData(Title createdTitle, XmlNode catalog_title) { //find the title node, use the short title var title_node = catalog_title.SelectNodes("title")[0]; var short_title = title_node.Attributes["short"].Value; var regular_title = title_node.Attributes["regular"].Value; createdTitle.TitleString = short_title; //TODO: Box Art //find the year released var release_node = catalog_title.SelectSingleNode("release_year"); var year = release_node.InnerText; if (year != "") { createdTitle.ReleaseYear = Convert.ToInt32(year); } else if (year == "") { createdTitle.ReleaseYear = 0; } //find the runtime var runtime_node = catalog_title.SelectSingleNode("link/delivery_formats/availability/runtime"); if (runtime_node != null) { var runtime = runtime_node.InnerText; try { createdTitle.RuntimeInSeconds = Convert.ToInt32(runtime); } catch (Exception ex) { createdTitle.RuntimeInSeconds = 0; Tools.TraceLine("error, could not convert runtime from string {0}", ex); } } //TODO Genres, need to figure out best way to sort multiple vals //find the genres //since it's so similar I've set the maturity level here instead of in the rating method string path = @"/catalog_title/category[@scheme]"; var nodes = catalog_title.SelectNodes(path); List<String> genre_list = new List<string>(); foreach (XmlNode node in nodes) { string label = node.Attributes["label"].Value; genre_list.Add(label); } //since the last one is always maturity level, use that as mat_level // but make sure it's a number first, otherwise it's a genre and there's no mat level string maturity_level = genre_list[genre_list.Count-1]; int mat_level; if (int.TryParse(maturity_level, out mat_level)) { //if maturity_level is a number, assign it to Title and remove it //waste mat_level genre_list.Remove(maturity_level); try { createdTitle.MaturityLevel = Convert.ToInt32(maturity_level); } catch (Exception ex) { createdTitle.MaturityLevel = 200; Tools.TraceLine("Could not convert maturity string to int {0}", ex); } } //now we join the genre_list with commas and assign it to Title string genres = string.Join(", ", genre_list); createdTitle.Genres = genres; //create all the Genres found too //var db = NextFlicksMVC4.Controllers.MoviesController.db; var db = new MovieDbContext(); foreach (string genre_string in genre_list) { //look in Genres Table for genre_String that matches and pull that out and add it to ListGenres //string qry = "select * from Genres where genre_string = {0}"; //var res =db.Genres.SqlQuery(qry, genre_string); //var selected_genre = res.ToList()[0]; var selected_genre = db.Genres.First(gen => gen.genre_string == genre_string); createdTitle.ListGenres.Add(selected_genre); } db.Dispose(); //Trace.WriteLine("added Primary Data to title"); }