static void SaveMoviesToDB()
{
MovieCreditsContext ctx = new MovieCreditsContext();
int moviesAdded = 0;
while (continueAddingMoviesToDb || !movieCreditsQueue.IsEmpty)
{
while (!movieCreditsQueue.IsEmpty && moviesAdded < 100)
{
MovieCredits movieCredits;
if (movieCreditsQueue.TryDequeue(out movieCredits))
{
ctx.MovieCreditsSet.Add(movieCredits);
if (movieCredits.id != 0)
{
if (movieCredits.cast != null)
{
/*foreach (var castMember in movieCredits.cast)
* {
* //castMember.movie_credits_id = movieCredits.id;
* castMember.movie_credits = movieCredits;
* }*/
ctx.CastSet.AddRange(movieCredits.cast);
/*if(movieCredits.cast.Count > 0)
* Console.WriteLine($"ID BEFORE SAVE: {movieCredits.id}, with character name: {movieCredits.cast[0].character}");*/
}
if (movieCredits.crew != null)
{
/*foreach (var crewMember in movieCredits.crew)
* {
* //crewMember.movie_credits_id = movieCredits.id;
* crewMember.movie_credits = movieCredits;
* }*/
ctx.CrewSet.AddRange(movieCredits.crew);
}
moviesAdded += 1;
ctx.SaveChanges();
}
}
}
if (moviesAdded > 0)
{
Console.WriteLine("Processed Another: {0}, {1} ", moviesAdded, DateTime.Now.ToLocalTime());
}
moviesAdded = 0;
ctx = new MovieCreditsContext();
}
Console.WriteLine("THIS SHOULD HAPPEN AT END");
Console.WriteLine("THIS SHOULD HAPPEN AT END");
}
static void Main(string[] args)
{
Console.WriteLine("Working...");
string apiKey = args[0];
string movieSplitFileName = "MovieListSplit";
var mode = EXECUTIONMODE.DOWNLOAD_DELTAS_FROM_BASELINE;
if (args.Length > 1)
{
int modeNum = -1;
int.TryParse(args[1], out modeNum);
if (modeNum == 1)
{
mode = EXECUTIONMODE.GENERATE_SPLIT_DIFF_FILES;
}
else if (modeNum == 2)
{
mode = EXECUTIONMODE.UPDATE_PRE_EXISTING_IDS_FILE;
}
else if (modeNum == 3)
{
mode = EXECUTIONMODE.DOWNLOAD_FROM_SPLIT_DIFF_FILES;
}
}
string preExistingFileIdsFileName = "preExistingIds";
var filePreExistingIds = File.Exists(preExistingFileIdsFileName) ? File.ReadAllLines(preExistingFileIdsFileName).Select(idEntry => int.Parse(idEntry)) : new List <int>();
MovieCreditsContext ctx = new MovieCreditsContext();
var existingMovieIds = ctx.MovieCreditsSet.Select(movie => movie.id).ToList();
var allPreExistingMovieIds = existingMovieIds.Union(filePreExistingIds).Distinct().ToDictionary(idEntry => idEntry);
if (mode == EXECUTIONMODE.UPDATE_PRE_EXISTING_IDS_FILE)
{
File.WriteAllLines(preExistingFileIdsFileName, allPreExistingMovieIds.Select(entry => entry.Key.ToString()));
}
if (mode == EXECUTIONMODE.GENERATE_SPLIT_DIFF_FILES)
{
var allMovieDeltas = GetUpdatedMovieIds(allPreExistingMovieIds);
int numSplits = -1;
if (int.TryParse(args[2], out numSplits))
{
for (int i = 0; i < numSplits; i++)
{
TextWriter tw = new StreamWriter($"{movieSplitFileName + i}.txt");
var finalIndexNum = -1;
if (i == numSplits - 1)
{
finalIndexNum = allMovieDeltas.Count - 1;
}
else
{
finalIndexNum = (allMovieDeltas.Count / numSplits) * (i + 1);
}
for (int j = i * allMovieDeltas.Count / numSplits; j < finalIndexNum; j++)
{
tw.WriteLine(allMovieDeltas[j]);
}
tw.Close();
}
}
}
List <int> movieIdDeltas = new List <int>();
if (mode == EXECUTIONMODE.DOWNLOAD_DELTAS_FROM_BASELINE)
{
movieIdDeltas = GetUpdatedMovieIds(allPreExistingMovieIds);
Console.WriteLine($"Num Movies Missing: { movieIdDeltas.Count }");
}
if (mode == EXECUTIONMODE.DOWNLOAD_FROM_SPLIT_DIFF_FILES)
{
int splitDiffFileNum = -1;
if (int.TryParse(args[2], out splitDiffFileNum))
{
var splitFileIds = File.ReadAllLines(movieSplitFileName + splitDiffFileNum + ".txt").Select(id => int.Parse(id));
Parallel.ForEach(splitFileIds, splitFileId =>
{
if (!allPreExistingMovieIds.ContainsKey(splitFileId))
{
movieIdDeltas.Add(splitFileId);
}
});
Console.WriteLine($"Num Movies Missing: { movieIdDeltas.Count }");
}
else
{
mode = EXECUTIONMODE.ERROR;
Console.WriteLine("Please provide a command argument indicating which Movie Split File Number to parse");
}
}
if (mode == EXECUTIONMODE.DOWNLOAD_DELTAS_FROM_BASELINE || mode == EXECUTIONMODE.DOWNLOAD_FROM_SPLIT_DIFF_FILES)
{
Console.WriteLine("Beginning to download");
movieClient.BaseAddress = new Uri("https://api.themoviedb.org/3/");
movieClient.DefaultRequestHeaders.Accept.Clear();
movieClient.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
var saveMoviesTask = Task.Run(() => SaveMoviesToDB());
int rateLimitPeriodSeconds = 10;
TimeSpan rateLimitSecondSpan = new TimeSpan(0, 0, rateLimitPeriodSeconds);
int rateLimitPerPeriod = 30;
var queries = new Task[rateLimitPerPeriod];
DateTime firstQueryTime;
for (int i = 0; i < movieIdDeltas.Count; i += rateLimitPerPeriod)
{
for (int requestIdx = 0; requestIdx < rateLimitPerPeriod; requestIdx++)
{
queries[requestIdx] = RetrieveWriteMovie(movieIdDeltas[i + requestIdx], apiKey);
}
firstQueryTime = DateTime.Now;
Task.WaitAll(queries);
queries = new Task[rateLimitPerPeriod];
var elapsedTimeSinceFirstQuery = DateTime.Now - firstQueryTime;
//Console.WriteLine($"Elapsed Time Since Query: {elapsedTimeSinceFirstQuery}");
if (elapsedTimeSinceFirstQuery.TotalSeconds < rateLimitPerPeriod)
{
var sleepTime = (int)(rateLimitSecondSpan - elapsedTimeSinceFirstQuery).TotalMilliseconds;
if (sleepTime > 0)
{
//Console.WriteLine($"Sleeping for {sleepTime} Milliseconds");
Thread.Sleep(sleepTime);
}
}
/*if (i % 100 == 0)
* {
* Console.WriteLine($"Another 100 queries sent off: {DateTime.Now.ToString()}");
* }*/
while (movieCreditsQueue.Count > 300)
{
Thread.Sleep(4000);
}
}
while (movieCreditsQueue.Count > 0)
{
Thread.Sleep(4000);
}
continueAddingMoviesToDb = false;
saveMoviesTask.Wait();
}
Console.WriteLine("Finished");
Console.ReadKey();
}