//Handler for ffmpeg timer to kill the process
static void OnCaptureTimer(object obj)
{
bool killProcess = false;
CaptureProcessInfo captureProcessInfo = obj as CaptureProcessInfo;
//Are we done?
if (DateTime.Now >= captureProcessInfo.timerDone)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: Timer is up. Killing capture process");
}
//Have we been canclled?
if (captureProcessInfo.cancellationToken != null && captureProcessInfo.cancellationToken.IsCancellationRequested)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: Task has been cancelled. Killing capture process");
}
//Make sure file is still growing at a reasonable pace. Otherwise, kill the process
if (!killProcess)
{
//Grab file info
FileInfo fileInfo = new FileInfo(captureProcessInfo.outputPath);
//Make sure file even exists!
if (!fileInfo.Exists)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: ERROR: File {captureProcessInfo.outputPath} doesn't exist. Feed is bad.");
}
else
{
//Make sure file size (rate) is fine
long fileSize = fileInfo.Length;
long kBytesSec = ((fileSize - captureProcessInfo.fileSize) / captureProcessInfo.interval) / 1000;
if (kBytesSec <= captureProcessInfo.acceptableRate)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: ERROR: File size no longer growing. (Current Rate: ({kBytesSec} KB/s) Killing capture process.");
}
captureProcessInfo.fileSize = fileSize;
captureProcessInfo.avgKBytesSec = (captureProcessInfo.avgKBytesSec + kBytesSec) / 2;
}
}
//Kill process if needed
Process p = captureProcessInfo.process;
if (killProcess && p != null && !p.HasExited)
{
p.Kill();
p.WaitForExit();
}
}
public CaptureProcessInfo ExecProcess(TextWriter logWriter, string exe, string cmdLineArgs, int timeout, string outputPath, CancellationToken cancellationToken, bool bestChannelIsSelected)
{
//Create our process
var processInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = cmdLineArgs,
RedirectStandardOutput = true,
RedirectStandardError = true
};
Process process = Process.Start(processInfo);
//Let's build a timer to kill the process when done
CaptureProcessInfo captureProcessInfo = null;
Timer captureTimer = null;
if (timeout > 0)
{
int interval = 10; //# of seconds between timer/file checks
long acceptableRate = Convert.ToInt32(configuration["acceptableRate"]);
//create capture process info
DateTime timerDone = DateTime.Now.AddMinutes(timeout);
captureProcessInfo = new CaptureProcessInfo(process, acceptableRate, interval, timerDone, outputPath, logWriter, cancellationToken, bestChannelIsSelected);
//create timer
TimeSpan intervalTime = new TimeSpan(0, 0, interval);
logWriter.WriteLine($"{DateTime.Now}: Settting Timer for {timeout} minutes in the future to kill process.");
captureTimer = new Timer(OnCaptureTimer, captureProcessInfo, intervalTime, intervalTime);
}
//Now, let's wait for the thing to exit
logWriter.WriteLine(process.StandardError.ReadToEnd());
logWriter.WriteLine(process.StandardOutput.ReadToEnd());
process.WaitForExit();
//Clean up timer
if (timeout > 0 && captureTimer != null)
{
captureTimer.Dispose();
}
return(captureProcessInfo);
}
//Handler for ffmpeg timer to kill the process
static void OnCaptureTimer(object obj)
{
bool killProcess = false;
CaptureProcessInfo captureProcessInfo = obj as CaptureProcessInfo;
//Are we done?
if (DateTime.Now >= captureProcessInfo.timerDone)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: Timer is up. Killing capture process");
}
//Have we been canclled?
if (captureProcessInfo.cancellationToken != null && captureProcessInfo.cancellationToken.IsCancellationRequested)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: Task has been cancelled. Killing capture process");
}
//Make sure file is still growing at a reasonable pace. Otherwise, kill the process
if (!killProcess)
{
//Grab file info
FileInfo fileInfo = new FileInfo(captureProcessInfo.outputPath);
//Make sure file even exists!
if (!fileInfo.Exists)
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: ERROR: File {captureProcessInfo.outputPath} doesn't exist. Feed is bad.");
}
else
{
//Make sure file size (rate) is fine
long fileSize = fileInfo.Length;
long kBytesSec = ((fileSize - captureProcessInfo.fileSize) / captureProcessInfo.interval) / 1000;
if ((kBytesSec <= captureProcessInfo.acceptableRate && !captureProcessInfo.bestChannelIsSelected) || kBytesSec < 5) //If we haven't cycled through all the channels yet, see if we can do better.
{
killProcess = true;
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: ERROR: File growing too slowly or not at all. (Current Rate: ({kBytesSec} KB/s) Killing capture process.");
}
captureProcessInfo.fileSize = fileSize;
if (captureProcessInfo.avgKBytesSec == 0)
{
captureProcessInfo.avgKBytesSec = kBytesSec;
}
else
{
captureProcessInfo.avgKBytesSec = (captureProcessInfo.avgKBytesSec + kBytesSec) / 2;
}
//Log current KB rate every 30 minutes (180 intervals)
if (captureProcessInfo.currentKbLogCount == 0 || captureProcessInfo.currentKbLogCount >= 180)
{
captureProcessInfo.logWriter.WriteLine($"{DateTime.Now}: Current Rate: {kBytesSec} KB/s Avg Rate: {captureProcessInfo.avgKBytesSec} File Size: {fileSize}");
captureProcessInfo.currentKbLogCount = 1;
}
captureProcessInfo.currentKbLogCount++;
}
}
//Kill process if needed
Process p = captureProcessInfo.process;
if (killProcess && p != null && !p.HasExited)
{
p.Kill();
p.WaitForExit();
}
}
//A key member function (along with BuildRecordSchedule) which does the bulk of the work
//
//I'll be a bit more verbose then usual so I can remember later what I was thinking
private void CaptureStream(TextWriter logWriter, string hashValue, ChannelHistory channelHistory, RecordInfo recordInfo, VideoFileManager videoFileManager)
{
//Process manager for ffmpeg
//
//This is there ffmpeg is called, and a watchdog timer created to make sure things are going ok
ProcessManager processManager = new ProcessManager(configuration);
//Test internet connection and get a baseline
//Right now, the baseline is not used for anything other than a number in the logs.
//This could be taken out...
long internetSpeed = TestInternet(logWriter);
//Create servers object
//This is the list of live247 servers we'll use to cycle through, finding the best quality stream
Servers servers = new Servers(configuration["ServerList"]);
//Create the server/channel selector object
//
//This object is what uses the heurstics to determine the right server/channel combo to use.
//Factors like language, quality, and rate factor in.
ServerChannelSelector scs = new ServerChannelSelector(logWriter, channelHistory, servers, recordInfo);
//Marking time we started and when we should be done
DateTime captureStarted = DateTime.Now;
DateTime captureTargetEnd = recordInfo.GetStartDT().AddMinutes(recordInfo.GetDuration());
if (!string.IsNullOrEmpty(configuration["debug"]))
{
captureTargetEnd = DateTime.Now.AddMinutes(1);
}
DateTime lastStartedTime = captureStarted;
TimeSpan duration = (captureTargetEnd.AddMinutes(1)) - captureStarted; //the 1 minute takes care of alignment slop
//Update capture history
//This saves to channelhistory.json file and is used to help determine the initial order of server/channel combos
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).recordingsAttempted += 1;
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).lastAttempt = DateTime.Now;
//Build output file - the resulting capture file.
//See VideoFileManager for specifics around file management (e.g. there can be multiple if errors are encountered etc)
VideoFileInfo videoFileInfo = videoFileManager.AddCaptureFile(configuration["outputPath"]);
//Email that show started
if (recordInfo.emailFlag)
{
new Mailer().SendShowStartedMail(configuration, recordInfo);
}
//Build ffmpeg capture command line with first channel and get things rolling
string cmdLineArgs = BuildCaptureCmdLineArgs(scs.GetServerName(), scs.GetChannelNumber(), hashValue, videoFileInfo.GetFullFile());
logWriter.WriteLine($"=========================================");
logWriter.WriteLine($"{DateTime.Now}: Starting {captureStarted} on server/channel {scs.GetServerName()}/{scs.GetChannelNumber()}. Expect to be done by {captureTargetEnd}.");
logWriter.WriteLine($" {configuration["ffmpegPath"]} {cmdLineArgs}");
CaptureProcessInfo captureProcessInfo = processManager.ExecProcess(logWriter, configuration["ffmpegPath"], cmdLineArgs, (int)duration.TotalMinutes, videoFileInfo.GetFullFile(), recordInfo.cancellationToken, scs.IsBestSelected());
logWriter.WriteLine($"{DateTime.Now}: Exited Capture. Exit Code: {captureProcessInfo.process.ExitCode}");
//Main loop to capture
//
//This loop is never entered if the first capture section completes without incident. However, it is not uncommon
//for errors to occur, and this loop takes care of determining the right next server/channel combo
//as well as making sure that we don't just try forever.
int numRetries = Convert.ToInt32(configuration["numberOfRetries"]);
int retryNum = 0;
for (retryNum = 0; DateTime.Now <= captureTargetEnd && retryNum < numRetries && !recordInfo.cancelledFlag; retryNum++)
{
logWriter.WriteLine($"{DateTime.Now}: Capture Failed for server/channel {scs.GetServerName()}/{scs.GetChannelNumber()}. Retry {retryNum+1} of {configuration["numberOfRetries"]}");
//Let's make sure the interwebs are still there. If not, let's loop until they come back or the show ends.
while (!IsInternetOk() && DateTime.Now <= captureTargetEnd)
{
bool logFlag = false;
if (!logFlag)
{
logWriter.WriteLine($"{DateTime.Now}: Interwebs are down. Checking every minute until back or show ends");
}
TimeSpan oneMinute = new TimeSpan(0, 1, 0);
Thread.Sleep(oneMinute);
}
//Check to see if we need to re-authenticate (most tokens have a lifespan)
int authMinutes = Convert.ToInt16(configuration["authMinutes"]);
if (DateTime.Now > captureStarted.AddMinutes(authMinutes))
{
logWriter.WriteLine($"{DateTime.Now}: It's been more than {authMinutes} authMinutes. Time to re-authenticate");
Task <string> authTask = Authenticate();
hashValue = authTask.Result;
if (string.IsNullOrEmpty(hashValue))
{
Console.WriteLine($"{DateTime.Now}: ERROR: Unable to authenticate. Check username and password?");
throw new Exception("Unable to authenticate during a retry");
}
}
//Log avg streaming rate for channel history
logWriter.WriteLine($"{DateTime.Now}: Avg rate is {captureProcessInfo.avgKBytesSec}KB/s for {scs.GetServerName()}/{scs.GetChannelNumber()}");
//Set new avg streaming rate for channel history
channelHistory.SetServerAvgKBytesSec(scs.GetChannelNumber(), scs.GetServerName(), captureProcessInfo.avgKBytesSec);
//Go to next channel if channel has been alive for less than 15 minutes
//The idea is that if a server/channel has been stable for at least 15 minutes, no sense trying to find another. (could make it worse)
TimeSpan fifteenMin = new TimeSpan(0, 15, 0);
if ((DateTime.Now - lastStartedTime) < fifteenMin)
{
//Set rate for current server/channel pair
scs.SetAvgKBytesSec(captureProcessInfo.avgKBytesSec);
//Get correct server and channel (determined by heuristics)
if (!scs.IsBestSelected())
{
scs.GetNextServerChannel();
retryNum = -1; //reset retries since we haven't got through the server/channel list yet
}
}
else
{
retryNum = -1; //reset retries since it's been more than 15 minutes
}
//Set new started time and calc new timer
TimeSpan timeJustRecorded = DateTime.Now - lastStartedTime;
lastStartedTime = DateTime.Now;
TimeSpan timeLeft = captureTargetEnd - DateTime.Now;
//Update channel history
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).hoursRecorded += timeJustRecorded.TotalHours;
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).recordingsAttempted += 1;
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).lastAttempt = DateTime.Now;
//Build output file
videoFileInfo = videoFileManager.AddCaptureFile(configuration["outputPath"]);
//Now get capture setup and going again
cmdLineArgs = BuildCaptureCmdLineArgs(scs.GetServerName(), scs.GetChannelNumber(), hashValue, videoFileInfo.GetFullFile());
logWriter.WriteLine($"{DateTime.Now}: Starting Capture (again) on server/channel {scs.GetServerName()}/{scs.GetChannelNumber()}");
logWriter.WriteLine($" {configuration["ffmpegPath"]} {cmdLineArgs}");
captureProcessInfo = processManager.ExecProcess(logWriter, configuration["ffmpegPath"], cmdLineArgs, (int)timeLeft.TotalMinutes + 1, videoFileInfo.GetFullFile(), recordInfo.cancellationToken, scs.IsBestSelected());
}
recordInfo.completedFlag = true;
logWriter.WriteLine($"{DateTime.Now}: Done Capturing Stream.");
//Update capture history and save
TimeSpan finalTimeJustRecorded = DateTime.Now - lastStartedTime;
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).hoursRecorded += finalTimeJustRecorded.TotalHours;
channelHistory.GetChannelHistoryInfo(scs.GetChannelNumber()).lastSuccess = DateTime.Now;
channelHistory.SetServerAvgKBytesSec(scs.GetChannelNumber(), scs.GetServerName(), captureProcessInfo.avgKBytesSec);
channelHistory.Save();
//check if actually done and didn't error out early
//We assume too many retries as that's really the only way out of the loop (outside of an exception which is caught elsewhere)
if (DateTime.Now < captureTargetEnd)
{
if (recordInfo.cancelledFlag)
{
logWriter.WriteLine($"{DateTime.Now}: Cancelled {recordInfo.description}");
}
else
{
logWriter.WriteLine($"{DateTime.Now}: ERROR! Too many retries - {recordInfo.description}");
}
//set partial flag
recordInfo.partialFlag = true;
//Send alert mail
string body = recordInfo.description + " partially recorded due to too many retries or cancellation. Time actually recorded is " + finalTimeJustRecorded.TotalHours;
new Mailer().SendErrorMail(configuration, "Partial: " + recordInfo.description, body);
}
}
