static void SaveMoviesToDB() { MovieCreditsContext ctx = new MovieCreditsContext(); int moviesAdded = 0; while (continueAddingMoviesToDb || !movieCreditsQueue.IsEmpty) { while (!movieCreditsQueue.IsEmpty && moviesAdded < 100) { MovieCredits movieCredits; if (movieCreditsQueue.TryDequeue(out movieCredits)) { ctx.MovieCreditsSet.Add(movieCredits); if (movieCredits.id != 0) { if (movieCredits.cast != null) { /*foreach (var castMember in movieCredits.cast) * { * //castMember.movie_credits_id = movieCredits.id; * castMember.movie_credits = movieCredits; * }*/ ctx.CastSet.AddRange(movieCredits.cast); /*if(movieCredits.cast.Count > 0) * Console.WriteLine($"ID BEFORE SAVE: {movieCredits.id}, with character name: {movieCredits.cast[0].character}");*/ } if (movieCredits.crew != null) { /*foreach (var crewMember in movieCredits.crew) * { * //crewMember.movie_credits_id = movieCredits.id; * crewMember.movie_credits = movieCredits; * }*/ ctx.CrewSet.AddRange(movieCredits.crew); } moviesAdded += 1; ctx.SaveChanges(); } } } if (moviesAdded > 0) { Console.WriteLine("Processed Another: {0}, {1} ", moviesAdded, DateTime.Now.ToLocalTime()); } moviesAdded = 0; ctx = new MovieCreditsContext(); } Console.WriteLine("THIS SHOULD HAPPEN AT END"); Console.WriteLine("THIS SHOULD HAPPEN AT END"); }
static void Main(string[] args) { Console.WriteLine("Working..."); string apiKey = args[0]; string movieSplitFileName = "MovieListSplit"; var mode = EXECUTIONMODE.DOWNLOAD_DELTAS_FROM_BASELINE; if (args.Length > 1) { int modeNum = -1; int.TryParse(args[1], out modeNum); if (modeNum == 1) { mode = EXECUTIONMODE.GENERATE_SPLIT_DIFF_FILES; } else if (modeNum == 2) { mode = EXECUTIONMODE.UPDATE_PRE_EXISTING_IDS_FILE; } else if (modeNum == 3) { mode = EXECUTIONMODE.DOWNLOAD_FROM_SPLIT_DIFF_FILES; } } string preExistingFileIdsFileName = "preExistingIds"; var filePreExistingIds = File.Exists(preExistingFileIdsFileName) ? File.ReadAllLines(preExistingFileIdsFileName).Select(idEntry => int.Parse(idEntry)) : new List <int>(); MovieCreditsContext ctx = new MovieCreditsContext(); var existingMovieIds = ctx.MovieCreditsSet.Select(movie => movie.id).ToList(); var allPreExistingMovieIds = existingMovieIds.Union(filePreExistingIds).Distinct().ToDictionary(idEntry => idEntry); if (mode == EXECUTIONMODE.UPDATE_PRE_EXISTING_IDS_FILE) { File.WriteAllLines(preExistingFileIdsFileName, allPreExistingMovieIds.Select(entry => entry.Key.ToString())); } if (mode == EXECUTIONMODE.GENERATE_SPLIT_DIFF_FILES) { var allMovieDeltas = GetUpdatedMovieIds(allPreExistingMovieIds); int numSplits = -1; if (int.TryParse(args[2], out numSplits)) { for (int i = 0; i < numSplits; i++) { TextWriter tw = new StreamWriter($"{movieSplitFileName + i}.txt"); var finalIndexNum = -1; if (i == numSplits - 1) { finalIndexNum = allMovieDeltas.Count - 1; } else { finalIndexNum = (allMovieDeltas.Count / numSplits) * (i + 1); } for (int j = i * allMovieDeltas.Count / numSplits; j < finalIndexNum; j++) { tw.WriteLine(allMovieDeltas[j]); } tw.Close(); } } } List <int> movieIdDeltas = new List <int>(); if (mode == EXECUTIONMODE.DOWNLOAD_DELTAS_FROM_BASELINE) { movieIdDeltas = GetUpdatedMovieIds(allPreExistingMovieIds); Console.WriteLine($"Num Movies Missing: { movieIdDeltas.Count }"); } if (mode == EXECUTIONMODE.DOWNLOAD_FROM_SPLIT_DIFF_FILES) { int splitDiffFileNum = -1; if (int.TryParse(args[2], out splitDiffFileNum)) { var splitFileIds = File.ReadAllLines(movieSplitFileName + splitDiffFileNum + ".txt").Select(id => int.Parse(id)); Parallel.ForEach(splitFileIds, splitFileId => { if (!allPreExistingMovieIds.ContainsKey(splitFileId)) { movieIdDeltas.Add(splitFileId); } }); Console.WriteLine($"Num Movies Missing: { movieIdDeltas.Count }"); } else { mode = EXECUTIONMODE.ERROR; Console.WriteLine("Please provide a command argument indicating which Movie Split File Number to parse"); } } if (mode == EXECUTIONMODE.DOWNLOAD_DELTAS_FROM_BASELINE || mode == EXECUTIONMODE.DOWNLOAD_FROM_SPLIT_DIFF_FILES) { Console.WriteLine("Beginning to download"); movieClient.BaseAddress = new Uri("https://api.themoviedb.org/3/"); movieClient.DefaultRequestHeaders.Accept.Clear(); movieClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json")); var saveMoviesTask = Task.Run(() => SaveMoviesToDB()); int rateLimitPeriodSeconds = 10; TimeSpan rateLimitSecondSpan = new TimeSpan(0, 0, rateLimitPeriodSeconds); int rateLimitPerPeriod = 30; var queries = new Task[rateLimitPerPeriod]; DateTime firstQueryTime; for (int i = 0; i < movieIdDeltas.Count; i += rateLimitPerPeriod) { for (int requestIdx = 0; requestIdx < rateLimitPerPeriod; requestIdx++) { queries[requestIdx] = RetrieveWriteMovie(movieIdDeltas[i + requestIdx], apiKey); } firstQueryTime = DateTime.Now; Task.WaitAll(queries); queries = new Task[rateLimitPerPeriod]; var elapsedTimeSinceFirstQuery = DateTime.Now - firstQueryTime; //Console.WriteLine($"Elapsed Time Since Query: {elapsedTimeSinceFirstQuery}"); if (elapsedTimeSinceFirstQuery.TotalSeconds < rateLimitPerPeriod) { var sleepTime = (int)(rateLimitSecondSpan - elapsedTimeSinceFirstQuery).TotalMilliseconds; if (sleepTime > 0) { //Console.WriteLine($"Sleeping for {sleepTime} Milliseconds"); Thread.Sleep(sleepTime); } } /*if (i % 100 == 0) * { * Console.WriteLine($"Another 100 queries sent off: {DateTime.Now.ToString()}"); * }*/ while (movieCreditsQueue.Count > 300) { Thread.Sleep(4000); } } while (movieCreditsQueue.Count > 0) { Thread.Sleep(4000); } continueAddingMoviesToDb = false; saveMoviesTask.Wait(); } Console.WriteLine("Finished"); Console.ReadKey(); }