protected async override Task OnConsume(string example, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, "ExampleTask"); // may throw AlreadyInProgress exception _logger.LogInformation("Example Task Starting"); int captionCount = 0; int transcriptionCount = 0; using (var _context = CTDbContext.CreateDbContext()) { CaptionQueries captionQueries = new CaptionQueries(_context); var transcriptions = await _context.Transcriptions.Take(30).ToListAsync(); foreach (var transcription in transcriptions) { var transcriptionId = transcription.Id; var videoID = transcription.VideoId; var captions = await captionQueries.GetCaptionsAsync(transcriptionId); _logger.LogInformation($"{transcription.Id}: Caption count= {captions.Count}"); transcriptionCount++; } } _logger.LogInformation($"Example Task Done. transcriptionCount={transcriptionCount} captionCount={captionCount}"); }
public async Task <Video> DownloadYoutubeVideo(Media media) { var mediaResponse = await _rpcClient.PythonServerClient.DownloadYoutubeVideoRPCAsync(new CTGrpc.MediaRequest { VideoUrl = media.JsonMetadata["videoUrl"].ToString() }); if (FileRecord.IsValidFile(mediaResponse.FilePath)) { Video video = new Video { Video1 = await FileRecord.GetNewFileRecordAsync(mediaResponse.FilePath, mediaResponse.Ext) }; return(video); } else { // Deleting media is fine if download failed as we can get it back from the youtube playlist. GetLogger().LogError("DownloadYoutubeVideo failed. mediaId {0}, removing Media record", media.Id); using (var context = CTDbContext.CreateDbContext()) { context.Medias.Remove(media); context.SaveChanges(); } return(null); } }
/// <summary>Extracts scene information for a video. /// Beware: It is possible to start another scene task while the first one is still running</summary> protected async override Task OnConsume(string videoId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, videoId); // may throw AlreadyInProgress exception using (var _context = CTDbContext.CreateDbContext()) { Video video = await _context.Videos.FindAsync(videoId); if (video.SceneData == null || taskParameters.Force) { var jsonString = await _rpcClient.PythonServerClient.GetScenesRPCAsync(new CTGrpc.File { FilePath = video.Video1.VMPath }); JArray scenes = JArray.Parse(jsonString.Json); video.SceneData = new JObject { { "Scenes", scenes } }; await _context.SaveChangesAsync(); } } }
/// <summary> /// Updates the accessToken and refreshToken. These keys must already exist in the Dictionary table. /// </summary> public async Task RefreshAccessTokenAsync() { try { using (var _context = CTDbContext.CreateDbContext()) { var accessToken = await _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_ACCESS_TOKEN).FirstAsync(); var refreshToken = await _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_REFRESH_TOKEN).FirstAsync(); var config = new BoxConfig(Globals.appSettings.BOX_CLIENT_ID, Globals.appSettings.BOX_CLIENT_SECRET, new Uri("http://locahost")); var auth = new OAuthSession(accessToken.Value, refreshToken.Value, 3600, "bearer"); var client = new BoxClient(config, auth); /// Try to refresh the access token auth = await client.Auth.RefreshAccessTokenAsync(auth.AccessToken); /// Create the client again client = new BoxClient(config, auth); _logger.LogInformation("Refreshed Tokens"); accessToken.Value = auth.AccessToken; refreshToken.Value = auth.RefreshToken; await _context.SaveChangesAsync(); } } catch (Box.V2.Exceptions.BoxSessionInvalidatedException e) { _logger.LogError(e, "Box Token Failure."); await _slack.PostErrorAsync(e, "Box Token Failure."); throw; } }
// To generate authCode on a browser open, // https://account.box.com/api/oauth2/authorize?client_id=[CLIENT_ID]&response_type=code /// <summary>Updates Box accessToken and refreshToken values in the Dictionary table. /// Optionally creates these keys if they do not exist. /// </summary> public async Task CreateAccessTokenAsync(string authCode) { // This implementation is overly chatty with the database, but we rarely create access tokens so it is not a problem using (var _context = CTDbContext.CreateDbContext()) { if (!await _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_ACCESS_TOKEN).AnyAsync()) { _context.Dictionaries.Add(new Dictionary { Key = CommonUtils.BOX_ACCESS_TOKEN }); await _context.SaveChangesAsync(); } if (!await _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_REFRESH_TOKEN).AnyAsync()) { _context.Dictionaries.Add(new Dictionary { Key = CommonUtils.BOX_REFRESH_TOKEN }); await _context.SaveChangesAsync(); } var accessToken = _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_ACCESS_TOKEN).First(); var refreshToken = _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_REFRESH_TOKEN).First(); var config = new BoxConfig(Globals.appSettings.BOX_CLIENT_ID, Globals.appSettings.BOX_CLIENT_SECRET, new Uri("http://locahost")); var client = new Box.V2.BoxClient(config); var auth = await client.Auth.AuthenticateAsync(authCode); _logger.LogInformation("Created Box Tokens"); accessToken.Value = auth.AccessToken; refreshToken.Value = auth.RefreshToken; await _context.SaveChangesAsync(); } }
public async Task <Video> DownloadLocalPlaylist(Media media) { try { Video video = new Video(); if (media.JsonMetadata.ContainsKey("video1Path")) { var video1Path = media.JsonMetadata["video1Path"].ToString(); video.Video1 = await FileRecord.GetNewFileRecordAsync(video1Path, Path.GetExtension(video1Path)); } if (media.JsonMetadata.ContainsKey("video2Path")) { var video2Path = media.JsonMetadata["video2Path"].ToString(); video.Video2 = await FileRecord.GetNewFileRecordAsync(video2Path, Path.GetExtension(video2Path)); } return(video); } catch (Exception e) { GetLogger().LogError(e, "DownloadLocalPlaylist failed. mediaId {0}", media.Id); using (var context = CTDbContext.CreateDbContext()) { context.Medias.Remove(media); context.SaveChanges(); } return(null); } }
protected async override Task OnConsume(string example, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, "BuildElasticIndexTask"); // may throw AlreadyInProgress exception GetLogger().LogInformation("BuildElasticIndexTask Starting"); using (var _context = CTDbContext.CreateDbContext()) { CaptionQueries captionQueries = new CaptionQueries(_context); var all_transcriptions = await _context.Transcriptions.Where(t => t.Language == Languages.ENGLISH_AMERICAN).ToListAsync(); foreach (var transcription in all_transcriptions) { var all_captions = transcription.Captions; // each index has the unique name "index_string_unique", the current in use one has the alias "index_string_alias" var index_string_base = transcription.Id + "_" + Languages.ENGLISH_AMERICAN.ToLower(System.Globalization.CultureInfo.CurrentCulture); var index_string_unique = index_string_base + "_" + $"{DateTime.Now:yyyyMMddHHmmss}"; var index_string_alias = index_string_base + "_" + "primary"; var asyncBulkIndexResponse = await _client.BulkAsync(b => b .Index(index_string_unique) .IndexMany(all_captions) ); var alias_exist = await _client.Indices.ExistsAsync(index_string_alias); if (alias_exist.Exists) { var oldIndices = await _client.GetIndicesPointingToAliasAsync(index_string_alias); var oldIndexName = oldIndices.First().ToString(); var indexResponse = await _client.Indices.BulkAliasAsync(new BulkAliasRequest { Actions = new List <IAliasAction> { new AliasRemoveAction { Remove = new AliasRemoveOperation { Index = oldIndexName, Alias = index_string_alias } }, new AliasAddAction { Add = new AliasAddOperation { Index = index_string_unique, Alias = index_string_alias } } } }); } else { var putAliasResponse = await _client.Indices.PutAliasAsync(new PutAliasRequest(index_string_unique, index_string_alias)); } } } GetLogger().LogInformation("BuildElasticIndexTask Done"); }
public async Task <Video> DownloadEchoVideo(Media media) { Video video = new Video(); bool video1Success = false, video2Success = false; var mediaResponse = await _rpcClient.PythonServerClient.DownloadEchoVideoRPCAsync(new CTGrpc.MediaRequest { VideoUrl = media.JsonMetadata["videoUrl"].ToString(), AdditionalInfo = media.Playlist.JsonMetadata["downloadHeader"].ToString() }); video1Success = FileRecord.IsValidFile(mediaResponse.FilePath); if (video1Success) { video.Video1 = await FileRecord.GetNewFileRecordAsync(mediaResponse.FilePath, mediaResponse.Ext); } if (!string.IsNullOrEmpty(media.JsonMetadata["altVideoUrl"].ToString())) { var mediaResponse2 = await _rpcClient.PythonServerClient.DownloadEchoVideoRPCAsync(new CTGrpc.MediaRequest { VideoUrl = media.JsonMetadata["altVideoUrl"].ToString(), AdditionalInfo = media.Playlist.JsonMetadata["downloadHeader"].ToString() }); video2Success = FileRecord.IsValidFile(mediaResponse2.FilePath); if (video2Success) { video.Video2 = await FileRecord.GetNewFileRecordAsync(mediaResponse2.FilePath, mediaResponse.Ext); } } else { // As there is no file to download, it's "successfull" video2Success = true; } if (video1Success && video2Success) { return(video); } else { // Deleting media is fine if download failed as we can get it back from the echo playlist. GetLogger().LogError("DownloadEchoVideo failed. mediaId {0}, removing Media record", media.Id); using (var context = CTDbContext.CreateDbContext()) { context.Medias.Remove(media); context.SaveChanges(); } return(null); } }
protected async override Task OnConsume(string transcriptionId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, transcriptionId); // may throw AlreadyInProgress exception GetLogger().LogInformation($"Creating VTT & SRT files for ({transcriptionId})"); using (var _context = CTDbContext.CreateDbContext()) { var transcription = await _context.Transcriptions.FindAsync(transcriptionId); CaptionQueries captionQueries = new CaptionQueries(_context); var captions = await captionQueries.GetCaptionsAsync(transcription.Id); var vttfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateWebVTTFile(captions, transcription.Language), ".vtt"); FileRecord?existingVtt = await _context.FileRecords.FindAsync(transcription.FileId); if (existingVtt is null) { GetLogger().LogInformation($"{transcriptionId}: Creating new vtt file {vttfile.FileName}"); await _context.FileRecords.AddAsync(vttfile); transcription.File = vttfile; _context.Entry(transcription).State = EntityState.Modified; } else { GetLogger().LogInformation($"{transcriptionId}: replacing existing vtt file contents {existingVtt.FileName}"); existingVtt.ReplaceWith(vttfile); _context.Entry(existingVtt).State = EntityState.Modified; } var srtfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateSrtFile(captions), ".srt"); FileRecord?existingSrt = await _context.FileRecords.FindAsync(transcription.SrtFileId); if (existingSrt is null) { GetLogger().LogInformation($"{transcriptionId}: Creating new srt file {srtfile.FileName}"); await _context.FileRecords.AddAsync(srtfile); transcription.SrtFile = srtfile; _context.Entry(transcription).State = EntityState.Modified; } else { GetLogger().LogInformation($"{transcriptionId}: replacing existing srt file contents {existingSrt.FileName}"); existingSrt.ReplaceWith(srtfile); _context.Entry(existingSrt).State = EntityState.Modified; } await _context.SaveChangesAsync(); GetLogger().LogInformation($"{transcriptionId}: Database updated"); } }
protected async override Task OnConsume(string transcriptionId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, transcriptionId); // may throw AlreadyInProgress exception using (var _context = CTDbContext.CreateDbContext()) { var transcription = await _context.Transcriptions.FindAsync(transcriptionId); FileRecord existingVtt = await _context.FileRecords.FindAsync(transcription.FileId); FileRecord existingSrt = await _context.FileRecords.FindAsync(transcription.SrtFileId); CaptionQueries captionQueries = new CaptionQueries(_context); var captions = await captionQueries.GetCaptionsAsync(transcription.Id); var vttfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateWebVTTFile(captions, transcription.Language), ".vtt"); if (string.IsNullOrEmpty(transcription.FileId)) { await _context.FileRecords.AddAsync(vttfile); transcription.File = vttfile; _context.Entry(transcription).State = EntityState.Modified; } else { existingVtt.ReplaceWith(vttfile); _context.Entry(existingVtt).State = EntityState.Modified; } var srtfile = await FileRecord.GetNewFileRecordAsync(Caption.GenerateSrtFile(captions), ".srt"); if (string.IsNullOrEmpty(transcription.SrtFileId)) { await _context.FileRecords.AddAsync(srtfile); transcription.SrtFile = srtfile; _context.Entry(transcription).State = EntityState.Modified; } else { existingSrt.ReplaceWith(srtfile); _context.Entry(existingSrt).State = EntityState.Modified; } await _context.SaveChangesAsync(); } }
/// <summary> /// Original implementation of OnConsume. This code may be deleted if it is no longer useful. It is left as available for now as a template /// </summary> /// <param name="videoId"></param> /// <param name="taskParameters"></param> /// <returns></returns> private async Task OldOnConsumeNotUsed(string videoId, TaskParameters taskParameters) { using (var _context = CTDbContext.CreateDbContext()) { // Get the video object var video = await _context.Videos.FindAsync(videoId); _logger.LogInformation("Consuming" + video); // Make RPC call to produce audio file. var file = await _rpcClient.PythonServerClient.ConvertVideoToWavRPCWithOffsetAsync(new CTGrpc.FileForConversion { File = new CTGrpc.File { FilePath = video.Video1.VMPath } }); // Check if a valid file was returned. if (FileRecord.IsValidFile(file.FilePath)) { var fileRecord = await FileRecord.GetNewFileRecordAsync(file.FilePath, file.Ext); // Get the latest video object, in case it has changed var videoLatest = await _context.Videos.FindAsync(video.Id); // If there is no Audio file present, then update. if (videoLatest.Audio == null) { await _context.FileRecords.AddAsync(fileRecord); videoLatest.Audio = fileRecord; await _context.SaveChangesAsync(); // If no transcriptions present, produce transcriptions. if (!videoLatest.Transcriptions.Any()) { _transcriptionTask.Publish(videoLatest.Id); } } } else { throw new Exception("ConvertVideoToWavTask Failed + " + video.Id); } } }
/// Requests _downloadPlaylistInfoTask for all recent playlists private async Task DownloadAllPlaylists() { List <String> playlists; using (var _context = CTDbContext.CreateDbContext()) { _downloadPlaylistInfoTask.PurgeQueue(); var period = DateTime.Now.AddMonths(-6); //TODO/TOREVIEW: Suggest Term.EndDate < Today plus 2 weeks (but let's check the semester dates in the DB and document this in the frontend) playlists = await _context.Offerings.Where(o => o.Term.StartDate >= period).SelectMany(o => o.Playlists).Select(p => p.Id).ToListAsync(); } GetLogger().LogInformation($"DownloadAllPlaylists(); _downloadPlaylistInfoTask publishing {playlists.Count} tasks"); playlists.ForEach(p => _downloadPlaylistInfoTask.Publish(p)); GetLogger().LogInformation("DownloadAllPlaylists() - Complete"); }
private async void buildMockCaptions(string videoId) { GetLogger().LogInformation($"Building Mock Captions for video {videoId}"); using (var _context = CTDbContext.CreateDbContext()) { Video video = await _context.Videos.Include(v => v.Transcriptions).SingleAsync(v => v.Id == videoId); string[] languages = new string[] { Languages.ENGLISH_AMERICAN, Languages.SPANISH }; foreach (var language in languages) { var transcription = video.Transcriptions.SingleOrDefault(t => t.Language == language); // Did we get the default or an existing Transcription entity? if (transcription == null) { transcription = new Transcription() { Language = language, VideoId = video.Id }; _context.Add(transcription); } ; TimeSpan time = new TimeSpan(); TimeSpan duration = new TimeSpan(0, 0, 3); // seconds for (int index = 1; index <= 3; index++) { TimeSpan end = time.Add(duration); Caption c = new Caption { Index = index, Text = $"The Caption in {language} is {index + 100} on {DateTime.Now}", Begin = time, End = end, TranscriptionId = transcription.Id }; _context.Add(c); time = end; } // for } // for language await _context.SaveChangesAsync(); } }
/// <summary> /// Creates a new box client, after first refreshing the access and refresh token. /// </summary> public async Task <BoxClient> GetBoxClientAsync() { // Todo RefreshAccessTokenAsync could return this information for us; and avoid another trip to the database await RefreshAccessTokenAsync(); BoxClient boxClient; using (var _context = CTDbContext.CreateDbContext()) { var accessToken = await _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_ACCESS_TOKEN).FirstAsync(); var refreshToken = await _context.Dictionaries.Where(d => d.Key == CommonUtils.BOX_REFRESH_TOKEN).FirstAsync(); var config = new BoxConfig(Globals.appSettings.BOX_CLIENT_ID, Globals.appSettings.BOX_CLIENT_SECRET, new Uri("http://locahost")); var auth = new OAuthSession(accessToken.Value, refreshToken.Value, 3600, "bearer"); boxClient = new Box.V2.BoxClient(config, auth); } return(boxClient); }
public async Task <Video> DownloadBoxVideo(Media media) { try { var guid = Guid.NewGuid().ToString(); var newPath = Path.Combine(Globals.appSettings.DATA_DIRECTORY, guid + ".mp4"); var client = await _box.GetBoxClientAsync(); var stream = await client.FilesManager.DownloadAsync(media.UniqueMediaIdentifier); using (var fileStream = File.Create(newPath)) { stream.CopyTo(fileStream); } if (FileRecord.IsValidFile(newPath)) { Video video = new Video { Video1 = await FileRecord.GetNewFileRecordAsync(newPath, Path.GetExtension(newPath)) }; return(video); } else { // Deleting media is fine if download failed as we can get it back from the youtube playlist. GetLogger().LogError("DownloadBoxVideo failed. mediaId {0}, removing Media record", media.Id); using (var context = CTDbContext.CreateDbContext()) { context.Medias.Remove(media); context.SaveChanges(); } return(null); } } catch (Box.V2.Exceptions.BoxSessionInvalidatedException e) { GetLogger().LogError(e, "Box Token Failure."); await _slack.PostErrorAsync(e, "Box Token Failure."); throw; } }
public static void SeedCourses() { // Dry run code before using it on Production. string file = Path.Combine(Globals.appSettings.DATA_DIRECTORY, "seed", "Fall2019InstructorList.csv"); TextReader reader = new StreamReader(file); var csvReader = new CsvReader(reader, System.Globalization.CultureInfo.CurrentCulture); var records = csvReader.GetRecords <CSVCourse>(); List <CSVCourse> csvCourses = new List <CSVCourse>(records); using (var _context = CTDbContext.CreateDbContext()) { Department eceDept = _context.Departments.Where(d => d.Acronym == "ECE" && d.UniversityId == "1001").FirstOrDefault(); List <Course> courses = csvCourses.Where(c => c.SUBJ == "ECE").GroupBy(c => new { c.CRS_TITLE, c.NBR, c.SUBJ }).Select(c => new Course { CourseNumber = c.First().NBR, Department = eceDept }).ToList(); _context.Courses.AddRange(courses); _context.SaveChanges(); } }
protected override async Task OnConsume(string playlistId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, playlistId); // may throw AlreadyInProgress exception using (var _context = CTDbContext.CreateDbContext()) { var playlist = await _context.Playlists.FindAsync(playlistId); List <Media> medias = new List <Media>(); switch (playlist.SourceType) { case SourceType.Echo360: medias = await GetEchoPlaylist(playlist, _context); break; case SourceType.Youtube: medias = await GetYoutubePlaylist(playlist, _context); break; case SourceType.Local: medias = await GetLocalPlaylist(playlist, _context); break; case SourceType.Kaltura: medias = await GetKalturaPlaylist(playlist, _context); break; case SourceType.Box: medias = await GetBoxPlaylist(playlist, _context); break; } // TASK DEPENDENCY (REFACTOR) medias.ForEach(m => _downloadMediaTask.Publish(m.Id)); } }
public CoursesControllerTest() { _context = CTDbContext.CreateDbContext(); _context.Database.EnsureCreated(); _controller = new CoursesController(_context); }
/// <summary>Finds incomplete tasks and adds them all a TaskItem table. /// This appears to be defunct and not yet used code - grep FindPendingJobs, found no callers of this function /// </summary> // private async Task FindPendingJobs() // { // using (var context = CTDbContext.CreateDbContext()) // { // // Medias for which no videos have downloaded // var toDownloadMediaIds = await context.Medias.Where(m => m.Video == null).Select(m => // new TaskItem // { // UniqueId = m.Id, // ResultData = new JObject(), // TaskParameters = new JObject(), // TaskType = TaskType.DownloadMedia, // Attempts = 0 // }).ToListAsync(); // // Videos which haven't been converted to wav // var toConvertVideoIds = await context.Videos.Where(v => v.Medias.Any() && v.Audio == null).Select(v => // new TaskItem // { // UniqueId = v.Id, // ResultData = new JObject(), // TaskParameters = new JObject(), // TaskType = TaskType.ConvertMedia, // Attempts = 0 // }).ToListAsync(); // // Transcribe pending videos. // var toTranscribeVideoIds = await context.Videos.Where(v => v.TranscribingAttempts < 3 && // v.TranscriptionStatus != "NoError" && // v.Medias.Any() && v.Audio != null).Select(v => // new TaskItem // { // UniqueId = v.Id, // ResultData = new JObject(), // TaskParameters = new JObject(), // TaskType = TaskType.Transcribe, // Attempts = 0 // }).ToListAsync(); // // Completed Transcriptions which haven't generated vtt files // var toGenerateVTTsTranscriptionIds = await context.Transcriptions.Where(t => t.Captions.Count > 0 && t.File == null) // .Select(t => // new TaskItem // { // UniqueId = t.Id, // ResultData = new JObject(), // TaskParameters = new JObject(), // TaskType = TaskType.GenerateVTTFile, // Attempts = 0 // }).ToListAsync(); // var allTaskItems = new List<TaskItem>(); // allTaskItems.AddRange(toDownloadMediaIds); // allTaskItems.AddRange(toConvertVideoIds); // allTaskItems.AddRange(toTranscribeVideoIds); // allTaskItems.AddRange(toGenerateVTTsTranscriptionIds); // foreach(var taskItem in allTaskItems) // { // if(!await context.TaskItems.AnyAsync(t => t.TaskType == taskItem.TaskType && t.UniqueId == taskItem.UniqueId)) // { // await context.TaskItems.AddAsync(taskItem); // } // } // await context.SaveChangesAsync(); // } // } /// <summary> Used by the PeriodicCheck to identify and enqueue missing tasks. /// This Task is started after all playlists are updated. /// </summary> private async Task PendingJobs() { // Update Box Token every few hours _updateBoxTokenTask.Publish(""); //We will use these outside of the DB scope List <String> todoVTTs; List <String> todoProcessVideos; List <String> todoTranscriptions; List <String> todoDownloads; using (var context = CTDbContext.CreateDbContext()) { // Most tasks are created directly from within a task when it normally completed. // This code exists to detect missing items and to publish tasks to complete them // A redesigned taskengine should not have the direct coupling inside each task // Since downloading a video could also create a Video, it is better to do these with little time delay in-between and then publish all the tasks // I believe there is still a race condition: Prior to this, we've just polled all active playlists and at least one of these may have already completed // So let's only consider items that are older than 10 minutes // Okay this is bandaid on the current design until we redesign the taskengine // Ideas For the future: // * Consider setting TTL on these messages to be 5 minutes short of thethe Periodic Refresh? // * If/when we drop the direct appoach consider: Random ordering. Most recent first (or randomly choosing either) // If an object was created during the middle of a periodic cycle, give it a full cycle to queue, and another cycle to complete its tasks int minutesCutOff = Math.Max(1, Convert.ToInt32(Globals.appSettings.PERIODIC_CHECK_OLDER_THAN_MINUTES)); var tooRecentCutoff = DateTime.Now.AddMinutes(-minutesCutOff); // This is the first use of 'AsNoTracking' in this project; let's check it works in Production as expected // TODO/TOREVIEW: Does EF create the complete entity and then project out the ID column in dot Net, or does it request only the ID from the database? // TODO/TOREVIEW: Since this code just pulls the IDs from the database, I expect this will be harmless no-op, however all DB reads should use AsNoTracking as a best practice // See https://code-maze.com/queries-in-entity-framework-core/ // See https://docs.microsoft.com/en-us/ef/core/querying/tracking // Completed Transcriptions which haven't generated vtt files // TODO: Should also check dates too GetLogger().LogInformation($"Finding incomplete VTTs, Transcriptions and Downloads from before {tooRecentCutoff}, minutesCutOff=({minutesCutOff})"); // Todo Could also check for secondary video too todoProcessVideos = await context.Videos.AsNoTracking().Where( v => (v.Duration == null && !String.IsNullOrEmpty(v.Video1Id)) ).OrderByDescending(t => t.CreatedAt).Select(e => e.Id).ToListAsync(); todoVTTs = await context.Transcriptions.AsNoTracking().Where( t => t.Captions.Count > 0 && t.File == null && t.CreatedAt < tooRecentCutoff ).OrderByDescending(t => t.CreatedAt).Select(e => e.Id).ToListAsync(); todoTranscriptions = await context.Videos.AsNoTracking().Where( v => v.TranscribingAttempts < 1 && v.TranscriptionStatus != "NoError" && v.Medias.Any() && v.CreatedAt < tooRecentCutoff ).OrderByDescending(t => t.CreatedAt).Select(e => e.Id).ToListAsync(); // Medias for which no videos have downloaded todoDownloads = await context.Medias.AsNoTracking().Where( m => m.Video == null && m.CreatedAt < tooRecentCutoff ).OrderByDescending(t => t.CreatedAt).Select(e => e.Id).ToListAsync(); } // We have a list of outstanding tasks // However some of these may already be in progress // So don't queue theses GetLogger().LogInformation($"Found {todoProcessVideos.Count},{todoVTTs.Count},{todoTranscriptions.Count},{todoDownloads.Count} counts before filtering"); ClientActiveTasks currentProcessVideos = _processVideoTask.GetCurrentTasks(); todoProcessVideos.RemoveAll(e => currentProcessVideos.Contains(e)); ClientActiveTasks currentVTTs = _generateVTTFileTask.GetCurrentTasks(); todoVTTs.RemoveAll(e => currentVTTs.Contains(e)); ClientActiveTasks currentTranscription = _transcriptionTask.GetCurrentTasks(); todoTranscriptions.RemoveAll(e => currentTranscription.Contains(e)); ClientActiveTasks currentDownloads = _transcriptionTask.GetCurrentTasks(); todoDownloads.RemoveAll(e => currentDownloads.Contains(e)); GetLogger().LogInformation($"Current In progress {currentProcessVideos.Count},{currentVTTs.Count},{currentTranscription.Count},{currentDownloads.Count} counts after filtering"); GetLogger().LogInformation($"Found {todoProcessVideos.Count},{todoVTTs.Count},{todoTranscriptions.Count},{todoDownloads.Count} counts after filtering"); // Now we have a list of new things we want to do GetLogger().LogInformation($"Publishing processingVideos ({String.Join(",", todoProcessVideos)})"); todoProcessVideos.ForEach(t => _processVideoTask.Publish(t)); GetLogger().LogInformation($"Publishing todoVTTs ({String.Join(",", todoVTTs)})"); todoVTTs.ForEach(t => _generateVTTFileTask.Publish(t)); GetLogger().LogInformation($"Publishing todoTranscriptions ({String.Join(",", todoTranscriptions)})"); todoTranscriptions.ForEach(v => _transcriptionTask.Publish(v)); GetLogger().LogInformation($"Publishing todoDownloads ({String.Join(",", todoDownloads)})"); todoDownloads.ForEach(m => _downloadMediaTask.Publish(m)); //// Not used Videos which haven't been converted to wav /// Code Not deleted because one day we will just reuse the one wav file and use an offset into that file //(await context.Videos.Where(v => v.Medias.Any() && v.Audio == null).ToListAsync()).ForEach(v => _convertVideoToWavTask.Publish(v.Id)); // Videos which have failed in transcribing GetLogger().LogInformation("Pending Jobs - completed"); }
public DepartmentsControllerTest() { _context = CTDbContext.CreateDbContext(); _context.Database.EnsureCreated(); _controller = new DepartmentsController(_context); }
/// <summary> /// [2020.7.7] For the purpose of resuming failed transcriptions, all the captions of the failed transcription would now be stored in the database. /// Before resuming, all the old captions would be queried out from the database and stored in a dictionary, which will be passed to the /// transcription function along with the resume time point. /// </summary> /// <param name="videoId"></param> /// <param name="taskParameters"></param> /// <returns></returns> protected async override Task OnConsume(string videoId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, videoId); // may throw AlreadyInProgress exception if (Globals.appSettings.MOCK_RECOGNITION == "MOCK") { buildMockCaptions(videoId); } using (var _context = CTDbContext.CreateDbContext()) { // TODO: taskParameters.Force should wipe all captions and reset the Transcription Status Video video = await _context.Videos.Include(v => v.Video1).Where(v => v.Id == videoId).FirstAsync(); // ! Note the 'Include' ; we don't build the whole tree of related Entities if (video.TranscriptionStatus == Video.TranscriptionStatusMessages.NOERROR) { GetLogger().LogInformation($"{videoId}:Skipping Transcribing of- already complete"); return; } // GetKey can throw if the video.Id is currently being transcribed // However registerTask should have already detected that Key key = TaskEngineGlobals.KeyProvider.GetKey(video.Id); video.TranscribingAttempts += 10; await _context.SaveChangesAsync(); try { // create Dictionary and pass it to the recognition function var captionsMap = new Dictionary <string, List <Caption> >(); // Set Source Language and Target (translation) Languages var sourceLanguage = String.IsNullOrWhiteSpace(Globals.appSettings.SPEECH_RECOGNITION_DIALECT) ? Languages.ENGLISH_AMERICAN : Globals.appSettings.SPEECH_RECOGNITION_DIALECT.Trim(); var translations = new List <string> { Languages.ENGLISH_AMERICAN, Languages.SIMPLIFIED_CHINESE, Languages.KOREAN, Languages.SPANISH, Languages.FRENCH }; if (!String.IsNullOrWhiteSpace(Globals.appSettings.LANGUAGE_TRANSLATIONS)) { translations = Globals.appSettings.LANGUAGE_TRANSLATIONS.Split(',').ToList(); } GetLogger().LogInformation($"{videoId}: ({sourceLanguage}). Translation(s) = ({String.Join(',', translations)})"); // Different languages may not be as complete // So find the minimum timespan of the max observed ending for each language TimeSpan shortestTime = TimeSpan.MaxValue; // Cant use TimeSpan.Zero to mean unset var startAfterMap = new Dictionary <string, TimeSpan>(); var allLanguages = new List <string>(translations); allLanguages.Add(sourceLanguage); foreach (string language in allLanguages) { var existing = await _captionQueries.GetCaptionsAsync(video.Id, language); captionsMap[language] = existing; startAfterMap[language] = TimeSpan.Zero; if (existing.Any()) { TimeSpan lastCaptionTime = existing.Select(c => c.End).Max(); startAfterMap[language] = lastCaptionTime; GetLogger().LogInformation($"{ videoId}:{language}. Last Caption at {lastCaptionTime}"); } } //var lastSuccessTime = shortestTime < TimeSpan.MaxValue ? shortestTime : TimeSpan.Zero; //if (video.JsonMetadata != null && video.JsonMetadata["LastSuccessfulTime"] != null) //{ // lastSuccessTime = TimeSpan.Parse(video.JsonMetadata["LastSuccessfulTime"].ToString()); //} var result = await _msTranscriptionService.RecognitionWithVideoStreamAsync(videoId, video.Video1, key, captionsMap, sourceLanguage, startAfterMap); if (video.JsonMetadata == null) { video.JsonMetadata = new JObject(); } TaskEngineGlobals.KeyProvider.ReleaseKey(key, video.Id); foreach (var captionsInLanguage in result.Captions) { var theLanguage = captionsInLanguage.Key; var theCaptions = captionsInLanguage.Value; if (theCaptions.Any()) { var t = _context.Transcriptions.SingleOrDefault(t => t.VideoId == video.Id && t.Language == theLanguage); GetLogger().LogInformation($"Find Existing Transcriptions null={t == null}"); // Did we get the default or an existing Transcription entity? if (t == null) { t = new Transcription() { Captions = theCaptions, Language = theLanguage, VideoId = video.Id }; _context.Add(t); } else { // Transcriptions already existed, we are just completing them, so add only the new ones // TODO/TOREVIEW: Does this filter actually help, if existing caption entries were edited by hand? var newCaptions = theCaptions.Where(c => c.Id == null); t.Captions.AddRange(newCaptions); } } } video.TranscriptionStatus = result.ErrorCode; video.JsonMetadata["LastSuccessfulTime"] = result.LastSuccessTime.ToString(); await _context.SaveChangesAsync(); _sceneDetectionTask.Publish(video.Id); video.Transcriptions.ForEach(t => _generateVTTFileTask.Publish(t.Id)); } catch (Exception ex) { GetLogger().LogError(ex, $"{videoId}: Transcription Exception:${ex.StackTrace}"); video.TranscribingAttempts += 1000; await _context.SaveChangesAsync(); throw; } } }
protected async override Task OnConsume(JObject jObject, TaskParameters taskParameters, ClientActiveTasks cleanup) { using (var _context = CTDbContext.CreateDbContext()) { var type = jObject["Type"].ToString(); if (type == TaskType.PeriodicCheck.ToString()) { await _slackLogger.PostMessageAsync("Periodic Check."); registerTask(cleanup, "PeriodicCheck"); _updateBoxTokenTask.Publish(""); _buildElasticIndexTask.Publish(""); _cleanUpElasticIndexTask.Publish(""); _exampleTask.Publish(""); await DownloadAllPlaylists(); await PendingJobs(); } else if (type == TaskType.DownloadAllPlaylists.ToString()) { await DownloadAllPlaylists(); } else if (type == TaskType.DownloadPlaylistInfo.ToString()) { var playlistId = jObject["PlaylistId"].ToString(); var playlist = await _context.Playlists.FindAsync(playlistId); _downloadPlaylistInfoTask.Publish(playlist.Id); } else if (type == TaskType.GenerateVTTFile.ToString()) { var transcriptionId = jObject["TranscriptionId"].ToString(); var transcription = await _context.Transcriptions.FindAsync(transcriptionId); _generateVTTFileTask.Publish(transcription.Id); } else if (type == TaskType.SceneDetection.ToString()) { var mediaId = jObject["mediaId"].ToString(); var media = _context.Medias.Find(mediaId); _scenedDetectionTask.Publish(media.Video.Id); } else if (type == TaskType.CreateBoxToken.ToString()) { var authCode = jObject["authCode"].ToString(); _createBoxTokenTask.Publish(authCode); } else if (type == TaskType.DownloadMedia.ToString()) { var mediaId = jObject["mediaId"].ToString(); var media = await _context.Medias.FindAsync(mediaId); _downloadMediaTask.Publish(media.Id); } //else if (type == TaskType.ConvertMedia.ToString()) //{ // var videoId = jObject["videoId"].ToString(); // var video = await _context.Videos.FindAsync(videoId); // _convertVideoToWavTask.Publish(video.Id); //} else if (type == TaskType.TranscribeVideo.ToString()) { var id = jObject["videoOrMediaId"].ToString(); GetLogger().LogInformation($"{type}:{id}"); var video = await _context.Videos.FindAsync(id); if (video == null) { var media = await _context.Medias.FindAsync(id); if (media != null) { GetLogger().LogInformation($"{id}: media Found. videoID=({media.VideoId})"); video = media.Video; } } if (video == null) { GetLogger().LogInformation($"No video found for video/mediaId ({id})"); return; } //TODO: These properties should not be literal strings bool deleteExisting = false; try { deleteExisting = jObject["DeleteExisting"].Value <bool>(); } catch (Exception) { } if (deleteExisting) { GetLogger().LogInformation($"{id}:Removing Transcriptions for video ({video.Id})"); var transcriptions = video.Transcriptions; _context.Transcriptions.RemoveRange(transcriptions); video.TranscriptionStatus = ""; // Could also remove LastSuccessTime and reset attempts await _context.SaveChangesAsync(); } _transcriptionTask.Publish(video.Id); } else if (type == TaskType.UpdateOffering.ToString()) { var offeringId = jObject["offeringId"].ToString(); (await _context.Playlists.Where(o => o.OfferingId == offeringId).ToListAsync()) .ForEach(p => _downloadPlaylistInfoTask.Publish(p.Id)); } else if (type == TaskType.ReTranscribePlaylist.ToString()) { var playlistId = jObject["PlaylistId"].ToString(); // Get all videos var videos = await _context.Playlists.Where(p => p.Id == playlistId) .SelectMany(p => p.Medias) .Where(e => e != null) .Select(m => m.Video) .ToListAsync(); // Delete all captions. This caused a null pointer exception because some elements were null // the above line and this line now have null filters var captions = videos.SelectMany(v => v.Transcriptions) .Where(e => e != null) .SelectMany(t => t.Captions).ToList(); _context.Captions.RemoveRange(captions); // TODO/TOREVIEW: No need to create in captions. Their IDs should be sufficient // Delete all Transcriptions var transcriptions = videos.SelectMany(v => v.Transcriptions).Where(e => e != null).ToList(); _context.Transcriptions.RemoveRange(transcriptions); videos.ForEach(v => { v.TranscribingAttempts = 0; v.TranscriptionStatus = null; }); await _context.SaveChangesAsync(); videos.ForEach(v => { _transcriptionTask.Publish(v.Id); }); } } }
protected async override Task OnConsume(string videoId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, videoId); // may throw AlreadyInProgress exception Video video; bool videoUpdated = false; using (var _context = CTDbContext.CreateDbContext()) { video = await _context.Videos.Include(v => v.Video1) .Include(v => v.Video2) .Include(v => v.ProcessedVideo1) .Include(v => v.ProcessedVideo2) .Where(v => v.Id == videoId).FirstAsync(); } GetLogger().LogInformation("Consuming" + video); if (video.Duration == null && video.Video1 != null) { var mediaInfoResult = await _rpcClient.PythonServerClient.GetMediaInfoRPCAsync(new CTGrpc.File { FilePath = video.Video1.VMPath }); var mediaJson = JObject.Parse(mediaInfoResult.Json); video.FileMediaInfo = mediaJson; video.UpdateMediaProperties(); videoUpdated = true; } bool runbrokencode = false; if (runbrokencode) { if (video.Video1 != null) { if (video.ProcessedVideo1 == null || taskParameters.Force) { var file = await _rpcClient.PythonServerClient.ProcessVideoRPCAsync(new CTGrpc.File { FilePath = video.Video1.VMPath }); //This does not work video.ProcessedVideo1 = await FileRecord.GetNewFileRecordAsync(file.FilePath, file.Ext); videoUpdated = true; } } if (video.Video2 != null) { if (video.ProcessedVideo2 == null || taskParameters.Force) { var file = await _rpcClient.PythonServerClient.ProcessVideoRPCAsync(new CTGrpc.File { FilePath = video.Video2.VMPath }); //This does not work video.ProcessedVideo2 = await FileRecord.GetNewFileRecordAsync(file.FilePath, file.Ext); videoUpdated = true; } } } if (videoUpdated) { using (var _context = CTDbContext.CreateDbContext()) { _context.Entry(video).State = EntityState.Modified; await _context.SaveChangesAsync(); } } }
public UniversitiesControllerTest() { _context = CTDbContext.CreateDbContext(); _context.Database.EnsureCreated(); _controller = new UniversitiesController(_context); }
// Example Test: curl --insecure "https://localhost/api/Logs/GetAllCourseLogsByDateRange?from=1/1/2020&to=2/2/2020" -H "Authorization: Bearer ..." public async Task <IActionResult> GetAllCourseLogsByDateRange(DateTime from, DateTime to) { DateTime startDump = DateTime.Now; HttpContext.Response.StatusCode = 200; var headers = HttpContext.Response.Headers; headers["Content-Disposition"] = $"attachment; filename=logs-{startDump.ToString("yyyyMMddTHHmmss")}.tsv"; headers["Content-Type"] = "text/tab-separated-values; charset=utf-8"; headers["Cache-Control"] = "no-cache"; CancellationToken cancellationToken = HttpContext.RequestAborted; await HttpContext.Response.StartAsync(cancellationToken); _logger.LogInformation($"GetAllCourseLogsByDateRange({from.ToString("yyyyMMddTHHmmss")},{to.ToString("yyyyMMddTHHmmss")})"); // I doubt we're close to optimal performance (e.g. it is "awaiting" every line, and we're not re-using byte[] objects) // But we're must avoid loading the entire set of events into memory // There's lots of bad example code on the interwebs e.g. examples that load the entire result into memory // This pipe-based version (BodyWriter) appears to be appropriate // On my laptop a complete table scan with no matching results takes 90s // a curl request in a local CMD window returns 143K events in 300s; and seems to be (correctly limited) by the client's download speed. // Local curl test: For all 1,328,947 events from 2019. Processing time: 308.5116288 seconds. 4307 emitted events per second. // Note there can be a long delay (e.g. 40s) before the first event is written // See also- // https://docs.microsoft.com/en-us/aspnet/core/fundamentals/middleware/request-response?view=aspnetcore-5.0 // https://nodogmablog.bryanhogan.net/2019/06/streaming-results-from-entity-framework-core-and-web-api-core/ // Turn off tracking, Do not materialize using (var localcontext = CTDbContext.CreateDbContext()) { localcontext.Database.SetCommandTimeout(DB_LONGTIMEOUT_SECONDS); //default 30s is too short to complete // When developing You can use use this line instead //xxx var logs = localcontext.Logs.AsNoTracking().Take(1000).Select(l => var logs = localcontext.Logs.AsNoTracking().Where(l => l.CreatedAt >= from && l.CreatedAt <= to); var writer = HttpContext.Response.BodyWriter; await writer.WriteAsync( System.Text.Encoding.UTF8.GetBytes( "CreatedAt\tUserId\tOfferingId\tMediaId\tEventType\tJson\n")); int count = 0; const int flushEvery = 25000; // Explicitly flushing occasionally may ensure we don't allow buffer to get infinitely large when the client is slow DateTime lastLogTime = DateTime.Now; foreach (var l in logs) { if (count == 0) { lastLogTime = DateTime.Now; _logger.LogInformation($"Writing first log event after {(lastLogTime - startDump).TotalSeconds} seconds."); } string line = $"{l.CreatedAt}\t{l.UserId}\t{l.OfferingId}\t{l.MediaId}\t{l.EventType}\t\"{JsonConvert.SerializeObject(l.Json).Replace("\"","\\\"")}\"\n"; await writer.WriteAsync(System.Text.Encoding.UTF8.GetBytes(line)); if (((++count) % flushEvery) == 0) { await writer.FlushAsync(cancellationToken); var now = DateTime.Now; _logger.LogInformation($"{count} log events written. Currently {(int)(flushEvery / ((now - lastLogTime).TotalSeconds))} events per second."); lastLogTime = now; } } // Is this the best order for FlushAsync & CompleteAsync? It seems to work for us. // There's bug discussions about the exact implementation even in 2020 // https://github.com/dotnet/aspnetcore/issues/12334 await writer.FlushAsync(cancellationToken); _logger.LogInformation($"Flushed all log events. {count} total log event(s) written."); _logger.LogInformation("Completing"); await writer.CompleteAsync(); // No more writes var duration = (DateTime.Now - startDump).TotalSeconds; _logger.LogInformation($"Complete. Total time: {duration} seconds for {count} events. {(int)(count/duration)} events per second. "); } // Now we've finished writing we can close the database context // Do not attempt to close this before the final flush return(new EmptyResult()); // already processed }
protected override async Task OnConsume(string mediaId, TaskParameters taskParameters, ClientActiveTasks cleanup) { registerTask(cleanup, mediaId); // may throw AlreadyInProgress exception Media media; using (var _context = CTDbContext.CreateDbContext()) { media = await _context.Medias.Where(m => m.Id == mediaId) .Include(m => m.Playlist).FirstAsync(); } GetLogger().LogInformation("Consuming" + media); Video video = new Video(); switch (media.SourceType) { case SourceType.Echo360: video = await DownloadEchoVideo(media); break; case SourceType.Youtube: video = await DownloadYoutubeVideo(media); break; case SourceType.Local: video = await DownloadLocalPlaylist(media); break; case SourceType.Kaltura: video = await DownloadKalturaVideo(media); break; case SourceType.Box: video = await DownloadBoxVideo(media); break; } // If no valid video1, or if a video2 object exists but not a valid file - fail the task. if (video == null || video.Video1 == null || !video.Video1.IsValidFile() || (video.Video2 != null && !video.Video2.IsValidFile())) { throw new Exception("DownloadMediaTask failed for mediaId " + media.Id); } using (var _context = CTDbContext.CreateDbContext()) { var latestMedia = await _context.Medias.FindAsync(media.Id); // Don't add video if there are already videos for the given media. if (latestMedia.Video == null) { // Check if Video already exists, if yes link it with this media item. var file = _context.FileRecords.Where(f => f.Hash == video.Video1.Hash).ToList(); if (!file.Any()) { // Create new video Record await _context.Videos.AddAsync(video); await _context.SaveChangesAsync(); latestMedia.VideoId = video.Id; await _context.SaveChangesAsync(); GetLogger().LogInformation("Downloaded:" + video); _transcriptionTask.Publish(video.Id); _processVideoTask.Publish(video.Id); } else { var existingVideos = await _context.Videos.Where(v => v.Video1Id == file.First().Id).ToListAsync(); // If file exists but video doesn't. if (!existingVideos.Any()) { // Delete existing file Record await file.First().DeleteFileRecordAsync(_context); // Create new video Record await _context.Videos.AddAsync(video); await _context.SaveChangesAsync(); latestMedia.VideoId = video.Id; await _context.SaveChangesAsync(); GetLogger().LogInformation("Downloaded:" + video); _transcriptionTask.Publish(video.Id); _processVideoTask.Publish(video.Id); } // If video and file both exist. else { var existingVideo = await _context.Videos.Where(v => v.Video1Id == file.First().Id).FirstAsync(); latestMedia.VideoId = existingVideo.Id; await _context.SaveChangesAsync(); GetLogger().LogInformation("Existing Video:" + existingVideo); // Deleting downloaded video as it's duplicate. await video.DeleteVideoAsync(_context); } } } } }