private void ProcessFiles(xmds.xmds xmds, string key, string type)
{
// Test for old files
DateTime testDate = DateTime.Now.AddDays(ApplicationSettings.Default.LibraryAgentInterval * -1);
// Track processed files
int filesProcessed = 0;
// Get a list of all the log files waiting to be sent to XMDS.
DirectoryInfo directory = new DirectoryInfo(ApplicationSettings.Default.LibraryPath);
// Loop through each file
foreach (FileInfo fileInfo in directory.GetFiles("*" + type + "*"))
{
if (fileInfo.LastAccessTime < testDate)
{
Trace.WriteLine(new LogMessage("LogAgent - Run", "Deleting old file: " + fileInfo.Name), LogType.Info.ToString());
File.Delete(fileInfo.FullName);
continue;
}
// Only process as many files in one go as configured
if (filesProcessed >= ApplicationSettings.Default.MaxLogFileUploads)
{
break;
}
// construct the log message
StringBuilder builder = new StringBuilder();
builder.Append("<log>");
foreach (string entry in File.ReadAllLines(fileInfo.FullName))
{
builder.Append(entry);
}
builder.Append("</log>");
if (type == ApplicationSettings.Default.StatsLogFile)
{
xmds.SubmitStats(ApplicationSettings.Default.ServerKey, key, builder.ToString());
}
else
{
xmds.SubmitLog(ApplicationSettings.Default.ServerKey, key, builder.ToString());
}
// Delete the file we are on
File.Delete(fileInfo.FullName);
// Increment files processed
filesProcessed++;
}
}
private void ProcessFiles(xmds.xmds xmds, string key, string type)
{
// Get a list of all the log files waiting to be sent to XMDS.
string[] logFiles = Directory.GetFiles(ApplicationSettings.Default.LibraryPath, "*" + type + "*");
// Track processed files
int filesProcessed = 0;
// Loop through each file
foreach (string fileName in logFiles)
{
// Only process as many files in one go as configured
if (filesProcessed >= ApplicationSettings.Default.MaxLogFileUploads)
{
break;
}
// construct the log message
StringBuilder builder = new StringBuilder();
builder.Append("<log>");
foreach (string entry in File.ReadAllLines(fileName))
{
builder.Append(entry);
}
builder.Append("</log>");
if (type == ApplicationSettings.Default.StatsLogFile)
{
xmds.SubmitStats(ApplicationSettings.Default.ServerKey, key, builder.ToString());
}
else
{
xmds.SubmitLog(ApplicationSettings.Default.ServerKey, key, builder.ToString());
}
// Delete the file we are on
File.Delete(fileName);
// Increment files processed
filesProcessed++;
}
}
/// <summary>
/// Runs the agent
/// </summary>
public void Run()
{
Trace.WriteLine(new LogMessage("StatAgent", "Run: Thread Started"), LogType.Info.ToString());
// Assume no backlog when we first start out.
bool isBacklog = false;
int retryAfterSeconds = 0;
int countBacklogBatches = 0;
int processing = 0;
while (!_forceStop)
{
lock (_locker)
{
// What is out processing flag?
processing = (new Random()).Next(1, 1000);
try
{
// If we are restarting, reset
_manualReset.Reset();
// Reset backOff
retryAfterSeconds = 0;
// How many records do we have to send?
int recordsReady = StatManager.Instance.TotalReady();
// Does this mean we're in a backlog situation?
isBacklog = (recordsReady >= 500);
// Check to see if we have anything to send
if (StatManager.Instance.MarkRecordsForSend(processing, isBacklog))
{
HardwareKey key = new HardwareKey();
Trace.WriteLine(new LogMessage("StatAgent", "Run: Thread Woken and Lock Obtained, Key: " + processing), LogType.Audit.ToString());
using (xmds.xmds xmds = new xmds.xmds())
{
xmds.Credentials = null;
xmds.Url = ApplicationSettings.Default.XiboClient_xmds_xmds + "&method=submitStats";
xmds.UseDefaultCredentials = false;
// Get the starts we've marked to process as XML.
xmds.SubmitStats(ApplicationSettings.Default.ServerKey, key.Key, StatManager.Instance.GetXmlForSend(processing));
}
// Update the last send date to indicate we've just done so
StatManager.Instance.LastSendDate = DateTime.Now;
// Remove the ones we've just sent
StatManager.Instance.DeleteSent(processing);
}
}
catch (WebException webEx) when(webEx.Response is HttpWebResponse httpWebResponse && (int)httpWebResponse.StatusCode == 429)
{
// We got a backoff, so none of those records are processed.
StatManager.Instance.UnmarkRecordsForSend(processing);
// Get the header for how long we ought to wait
retryAfterSeconds = webEx.Response.Headers["Retry-After"] != null?int.Parse(webEx.Response.Headers["Retry-After"]) : 120;
// Log it.
Trace.WriteLine(new LogMessage("StatAgent", "Run: 429 received, waiting for " + retryAfterSeconds + " seconds."), LogType.Info.ToString());
}
catch (WebException webEx)
{
// Increment the quantity of XMDS failures and bail out
ApplicationSettings.Default.IncrementXmdsErrorCount();
// Log this message, but dont abort the thread
Trace.WriteLine(new LogMessage("StatAgent", "Run: WebException in Run: " + webEx.Message), LogType.Info.ToString());
// Something went wrong sending those records, set them to be reprocessed
StatManager.Instance.UnmarkRecordsForSend(processing);
}
catch (Exception ex)
{
// Log this message, but dont abort the thread
Trace.WriteLine(new LogMessage("StatAgent", "Run: Exception in Run: " + ex.Message), LogType.Error.ToString());
// Something went wrong sending those records, set them to be reprocessed
StatManager.Instance.UnmarkRecordsForSend(processing);
}
}
if (retryAfterSeconds > 0)
{
// Sleep this thread until we've fulfilled our try after
_manualReset.WaitOne(retryAfterSeconds * 1000);
}
else if (isBacklog)
{
// We've just completed a send in backlog mode, so add to batches.
countBacklogBatches++;
// How many batches have we sent without a cooldown?
if (countBacklogBatches > 2)
{
// Reset batches
countBacklogBatches = 0;
// Come back in 30 seconds
_manualReset.WaitOne(30000);
}
else
{
// Come back much more quickly (10 seconds)
_manualReset.WaitOne(10000);
}
}
else
{
// Reset batches
countBacklogBatches = 0;
// Sleep this thread until the next collection interval
_manualReset.WaitOne((int)(ApplicationSettings.Default.CollectInterval * ApplicationSettings.Default.XmdsCollectionIntervalFactor() * 1000));
}
}
Trace.WriteLine(new LogMessage("StatAgent", "Run: Thread Stopped"), LogType.Info.ToString());
}
