/// <summary>
/// Routine to load a slice of chat-logs from our local database that represent the messages sent during this match.
/// </summary>
public void PopulateSegmentChatLogs()
{
foreach (var segment in Segments)
{
if (!segment.IsPopulated)
{
segment.ChatLog = ChatLogParser.GetChatInRange(StartTime, segment.StartTime, segment.EndTime);
segment.IsPopulated = true;
}
}
IsPopulated = true;
}
/// <summary>
/// Processes raw VOD files. Saves their broadcast information to the LocalBroadcastManager and returns the resulting filtered matches.
/// </summary>
/// <param name="videosToProcess"></param>
/// <returns></returns>
public List <FilteredMatches> FilterBroadcasts(List <string> videosToProcess, bool testConfiguration = false)
{
// We iterate over each VOD file, filter out gameplay, get their associated chat-log and create a metadata store in the form of a Broadcast list.
List <Task> videoFilterTasks = new List <Task>();
List <Broadcast> broadcasts = new List <Broadcast>();
bool skip = false;
foreach (var video in videosToProcess)
{
Console.WriteLine($"Found unprocessed video: {video}");
// Extracting metadata from video filename
var videoInfo = video.Substring(video.LastIndexOf("\\", StringComparison.Ordinal) + 1).Split('_');
var recordedDate = videoInfo[0];
var recordedTime = videoInfo[1];
DateTime videoRecordedDateTime = DateTime.ParseExact(recordedDate + recordedTime, "yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture);
var videoId = int.Parse(videoInfo[2]);
if (!testConfiguration)
{
// Check if the Broadcast has already been filtered. Skip this broadcast if true.
foreach (var filteredMatch in FilteredMatchesManager.FilteredMatches)
{
if (filteredMatch.Broadcast.Id == videoId)
{
Console.WriteLine(video + " has already been processed before, skipping.");
skip = true;
break;
}
}
if (skip)
{
skip = false;
continue;
}
}
// Keep track of this broadcast by assigning a corresponding Broadcast object to it.
broadcasts.Add(new Broadcast(videoId, videoRecordedDateTime));
// We put each filtered broadcast into its own test folder.
Directory.CreateDirectory(Helper.BroadcastsPath + $"{videoId}");
string outputPath = Helper.BroadcastsPath + $"{videoId}\\";
// Queue the broadcast filter operation for future parallel filtering.
videoFilterTasks.Add(new Task(() => FilterVideo(video, outputPath, FilterTemplatePath, FilterThreshold, StartingFrame,
FramesToSkip, ConvertToGreyscale, SecondsUntilTimeout, SecondsMinimumMatchLength)));
}
Console.WriteLine("");
Console.WriteLine("Parsing chat-logs to local database.");
Console.WriteLine("");
// Loads in the Twitch chat logs for the day to our local SQL database.
// These will be loaded in filtered by each match's time range to avoid unnecessary ram usage.
foreach (var broadcast in broadcasts)
{
Console.WriteLine($"processing {broadcast.Id}'s chat-log");
if (!testConfiguration)
{
ChatLogParser.GenerateChatRows(broadcast);
}
}
Console.WriteLine("chat-log processing completed.");
Console.WriteLine("");
Console.WriteLine("Starting video filtering.");
Console.WriteLine("");
// Start filtering our queued videos.
foreach (var task in videoFilterTasks)
{
task.Start();
}
// Progress tracking
Regex percentageRegex = new Regex("\\d+%");
double highestPercentage = 0.00;
double prevReportedPercentage = 0.00;
List <int> taskProgressTracker = new List <int>();
// Wait for all videos to be fully processed.
// Regularly filter and report information given by each python process.
while (!Task.WaitAll(videoFilterTasks.ToArray(), 500))
{
Tuple <string, String> progress = null;
lock (_lockProgress)
{
// Take the latest report from a particular task.
if (_taskProgress.Count > 0)
{
progress = _taskProgress[0];
_taskProgress.Remove(progress);
}
}
if (progress != null)
{
// Percentage process of a particular task.
if (progress.Item2.Contains("%"))
{
var match = percentageRegex.Match(progress.Item2);
var percentage = int.Parse(match.Value.Trim('%'));
taskProgressTracker.Add(percentage);
// Rolling average.
if (taskProgressTracker.Count > 5)
{
taskProgressTracker.RemoveAt(0);
}
// Make sure the average progress never goes down.
if (taskProgressTracker.Average() > Math.Ceiling(highestPercentage))
{
highestPercentage += 1;
if (taskProgressTracker.Average() > prevReportedPercentage)
{
// Reports every % or so.
Console.WriteLine(taskProgressTracker.Average() + "% complete.");
}
prevReportedPercentage = taskProgressTracker.Average();
}
}
// Report when a filtered video (match or highlight) has been created.
if (progress.Item2.Contains("video created"))
{
Console.WriteLine(progress.Item2);
}
}
}
Console.WriteLine("All videos processed.");
// Use the csv report generated by each python process to create a reference to each filtered match and highlight.
var filteredMatches = BuildFilteredMatchesFromBroadcasts(broadcasts);
return(filteredMatches);
}
