public void SkipLogEventGivenItExceedsBatchSizeLimit()
{
// Arrange
using var stream = new MemoryStream();
const string logEventExceedingBatchSizeLimit = "{ \"foo\": \"This document exceeds the batch size limit\" }";
using var writer = new StreamWriter(stream, Encoding.UTF8WithoutBom);
writer.Write(logEventExceedingBatchSizeLimit + Environment.NewLine);
writer.Write(BarLogEvent + Environment.NewLine);
writer.Flush();
var batchSizeLimit = ByteSize.From(logEventExceedingBatchSizeLimit) - 1;
// Act
var got = BufferFileReader.Read(
stream,
ref nextLineBeginsAtOffset,
null,
null,
batchSizeLimit);
// Assert
got.LogEvents.ShouldBe(new[] { BarLogEvent });
got.HasReachedLimit.ShouldBeFalse();
nextLineBeginsAtOffset.ShouldBe(stream.Length);
}
private static bool TryReadLine(Stream current, ref long nextStart, out string nextLine)
{
var includesBom = nextStart == 0;
if (current.Length <= nextStart)
{
nextLine = null;
return(false);
}
current.Position = nextStart;
nextLine = ReadLine(current);
if (nextLine == null)
{
return(false);
}
nextStart += ByteSize.From(nextLine) + ByteSize.From(Environment.NewLine);
if (includesBom)
{
nextStart += 3;
}
return(true);
}
/// <summary>
/// Checks the size of the log event body.
/// </summary>
/// <returns>true if body size is within acceptable range; otherwise false.</returns>
protected bool CheckEventBodySize(string json)
{
if (eventBodyLimitBytes.HasValue &&
ByteSize.From(json) > eventBodyLimitBytes.Value)
{
SelfLog.WriteLine(
"Event JSON representation exceeds the byte size limit of {0} set for this sink and will be dropped; data: {1}",
eventBodyLimitBytes,
json);
return(false);
}
return(true);
}
public void Post([FromBody] string logEventBatch)
{
var contentType = Request.ContentType;
var contentEncoding = Request.Headers["Content-Encoding"].FirstOrDefault() ?? "";
var contentLength = (int?)Request.ContentLength ?? throw new Exception("No content length on request");
var batchSize = ByteSize.From(logEventBatch);
var logEventSizes = Json
.ParseArray(logEventBatch)
.Select(ByteSize.From)
.ToArray();
_statistics.ReportReceivedBatch(contentType, contentEncoding, contentLength, batchSize, logEventSizes);
}
protected bool CheckEventBodySize(string json)
{
var sizeLimit = _options.EventSizeLimitBytes ?? 255 * ByteSize.KB;
if (ByteSize.From(json) > sizeLimit)
{
SelfLog.WriteLine(
"Event JSON representation exceeds the byte size limit of {0} set for this sink and will be dropped; data: {1}",
sizeLimit,
json);
return(false);
}
return(true);
}
public static Batch Read(LogEventQueue queue, int batchPostingLimit, long batchSizeLimitBytes)
{
var batch = new Batch();
var remainingBatchSizeBytes = batchSizeLimitBytes;
while (true)
{
var result = queue.TryDequeue(remainingBatchSizeBytes, out var logEvent);
if (result == LogEventQueue.DequeueResult.Ok)
{
batch.LogEvents.Add(logEvent);
remainingBatchSizeBytes -= ByteSize.From(logEvent);
// Respect batch posting limit
if (batch.LogEvents.Count == batchPostingLimit)
{
batch.HasReachedLimit = true;
break;
}
}
else if (result == LogEventQueue.DequeueResult.MaxSizeViolation)
{
if (batch.LogEvents.Count == 0)
{
// This single log event exceeds the batch size limit, let's drop it
queue.TryDequeue(long.MaxValue, out var logEventToDrop);
SelfLog.WriteLine(
"Event exceeds the batch size limit of {0} bytes set for this sink and will be dropped; data: {1}",
batchSizeLimitBytes,
logEventToDrop);
}
else
{
batch.HasReachedLimit = true;
break;
}
}
else if (result == LogEventQueue.DequeueResult.QueueEmpty)
{
break;
}
}
return(batch);
}
public void SkipLogEventGivenItExceedsBatchSizeLimit()
{
// Arrange
const string logEventExceedingBatchSizeLimit = "{ \"foo\": \"This document exceeds the batch size limit\" }";
var queue = new LogEventQueue(null);
queue.Enqueue(logEventExceedingBatchSizeLimit);
queue.Enqueue(BarLogEvent);
var batchSizeLimit = ByteSize.From(logEventExceedingBatchSizeLimit) - 1;
// Act
var got = LogEventQueueReader.Read(queue, int.MaxValue, batchSizeLimit);
// Assert
got.LogEvents.ShouldBe(new[] { BarLogEvent });
got.HasReachedLimit.ShouldBeFalse();
}
public void RespectLogEventLimitBytes()
{
// Arrange
using var stream = new MemoryStream();
using var writer = new StreamWriter(stream, Encoding.UTF8WithoutBom);
writer.Write(FooLogEvent + Environment.NewLine);
writer.Flush();
var logEventLimitBytes = ByteSize.From(FooLogEvent) - 1;
// Act
var got = BufferFileReader.Read(
stream,
ref nextLineBeginsAtOffset,
logEventLimitBytes,
null,
null);
// Assert
got.LogEvents.ShouldBeEmpty();
got.HasReachedLimit.ShouldBeFalse();
nextLineBeginsAtOffset.ShouldBe(stream.Length);
}
public static Batch Read(
Stream stream,
ref long nextLineBeginsAtOffset,
int batchPostingLimit,
long batchSizeLimitBytes)
{
var batch = new Batch();
long batchSizeBytes = 0;
while (true)
{
if (stream.Length <= nextLineBeginsAtOffset)
{
break;
}
stream.Position = nextLineBeginsAtOffset;
// Read log event
var line = ReadLine(stream);
if (string.IsNullOrEmpty(line.Text))
{
break;
}
// Calculate the size of the log event
var lineSizeBytes = ByteSize.From(line.Text);
var includeLine = true;
// Respect batch size limit
if (lineSizeBytes > batchSizeLimitBytes)
{
// This single log event exceeds the batch size limit, let's drop it
includeLine = false;
SelfLog.WriteLine(
"Event exceeds the batch size limit of {0} bytes set for this sink and will be dropped; data: {1}",
batchSizeLimitBytes,
line);
}
else if (batchSizeBytes + lineSizeBytes > batchSizeLimitBytes)
{
// The accumulated size of the batch is exceeding the batch size limit
batch.HasReachedLimit = true;
break;
}
// Update cursor
nextLineBeginsAtOffset += lineSizeBytes + ByteSize.From(line.NewLine);
// Add log event
if (includeLine)
{
batch.LogEvents.Add(line.Text);
batchSizeBytes += lineSizeBytes;
}
// Respect batch posting limit
if (batch.LogEvents.Count == batchPostingLimit)
{
batch.HasReachedLimit = true;
break;
}
}
return(batch);
}
private void CreateBindings(ControlListenGroup clg)
{
txtUrl.BindValueTo(v => {
isYoutube = txtUrl.IsValidUrl() && new Uri(v).Host.Contains("youtube.com");
});
txtUrl.BindValueTo(v => api.Arguments.Url = v).Listen(clg);
nudVideoQualityWidth.BindEnableTo(rbVqCustom);
nudVideoQualityHeight.BindEnableTo(rbVqCustom);
nudFpsCustom.BindEnableTo(rbFpsCustom);
gbVideoQuality.BindEnableTo(cbs => !cbs[0].Checked, cbCustomFormatSelector);
gbFrameRate.BindEnableTo(cbs => !cbs[0].Checked, cbCustomFormatSelector);
#region General
cbIgnoreErrors.BindValueTo(v => api.Arguments.General.IgnoreErrors = v).Listen(clg);
cbAbortOnErrors.BindValueTo(v => api.Arguments.General.AbortOnError = v).Listen(clg);
cbMarkWatched.BindValueTo(v => api.Arguments.General.MarkWatched = isYoutube ? v : false, cbUsernamePassword, clg);
#endregion
#region Video Selection
nudPlaylistStart.BindEnableTo(cbPlaylistStart).BindValueTo(v => api.Arguments.VideoSelection.PlaylistStart = (int?)v, cbPlaylistStart, clg);
nudPlaylistEnd.BindEnableTo(cbPlaylistEnd).BindValueTo(v => api.Arguments.VideoSelection.PlaylistEnd = (int?)v, cbPlaylistEnd, clg);
txtPlaylistRange.BindEnableTo(cbPlaylistRange).BindValueTo(v => api.Arguments.VideoSelection.PlaylistItems = v, cbPlaylistRange, clg);
txtMatchTitle.BindEnableTo(cbMatchTitle).BindValueTo(v => api.Arguments.VideoSelection.MatchTitle = v, cbMatchTitle, clg);
txtRejectTitle.BindEnableTo(cbRejectTitle).BindValueTo(v => api.Arguments.VideoSelection.RejectTitle = v, cbRejectTitle, clg);
nudMaxDownloads.BindEnableTo(cbMaxDownloads).BindValueTo(v => api.Arguments.VideoSelection.MaxDownloads = (int?)v, cbMaxDownloads, clg);
txtMatchFilter.BindEnableTo(cbMatchFilter).BindValueTo(v => api.Arguments.VideoSelection.MatchFilter = v, cbMatchFilter, clg);
void filesizeMinEvt(float?v)
{
api.Arguments.VideoSelection.MinFilesize = (v == null || v == 0) ? (ByteSize?)null : ByteSize.From(v.Value, (FilesizeUnit)cbxFilesizeMinUnits.SelectedItem);
}
cbxFilesizeMinUnits.BindEnableTo(cbFilesizeMin).BindValueTo(obj => filesizeMinEvt(nudFilesizeMin.GetValueFloat(cbFilesizeMin))).Listen(clg);
nudFilesizeMin.BindEnableTo(cbFilesizeMin).BindValueTo(filesizeMinEvt, cbFilesizeMin, clg);
void filesizeMaxEvt(float?v)
{
api.Arguments.VideoSelection.MaxFilesize = (v == null || v == 0) ? (ByteSize?)null : ByteSize.From(v.Value, (FilesizeUnit)cbxFilesizeMaxUnits.SelectedItem);
}
cbxFilesizeMaxUnits.BindEnableTo(cbFilesizeMax).BindValueTo(obj => filesizeMaxEvt(nudFilesizeMax.GetValueFloat(cbFilesizeMax))).Listen(clg);
nudFilesizeMax.BindEnableTo(cbFilesizeMax).BindValueTo(filesizeMaxEvt, cbFilesizeMax, clg);
dtpDate.BindEnableTo(cbDate).BindValueTo(v => api.Arguments.VideoSelection.Date = v, cbDate, clg);
dtpDateAfter.BindEnableTo(cbDateAfter).BindValueTo(v => api.Arguments.VideoSelection.DateAfter = v, cbDateAfter, clg);
dtpDateBefore.BindEnableTo(cbDateBefore).BindValueTo(v => api.Arguments.VideoSelection.DateBefore = v, cbDateBefore, clg);
nudViewRangeMin.BindEnableTo(cbMinViews).BindValueTo(v => api.Arguments.VideoSelection.MinViews = (int?)v, cbMinViews, clg);
nudViewRangeMax.BindEnableTo(cbMaxViews).BindValueTo(v => api.Arguments.VideoSelection.MaxViews = (int?)v, cbMaxViews, clg);
//// api.Arguments.VideoSelection.MatchFilter =
rbPlaylistOnly.BindValueTo(v => api.Arguments.VideoSelection.YesPlaylist = v).Listen(clg);
rbVideoOnly.BindValueTo(v => api.Arguments.VideoSelection.NoPlaylist = v).Listen(clg);
nudAgeLimit.BindEnableTo(cbAgeLimit).BindValueTo(v => api.Arguments.VideoSelection.AgeLimit = (int?)v, cbAgeLimit, clg);
btnBrowseDownloadArchive.BindEnableTo(cbDownloadArchive);
txtDownloadArchive.BindEnableTo(cbDownloadArchive).BindValueTo(v => {
api.Arguments.VideoSelection.DownloadArchive = v;
}, cbDownloadArchive, clg);
#endregion
#region Filesystem
txtFileTemplate.BindEnableTo(cbFileTemplate).BindValueTo(v => { api.Arguments.FileSystem.OutputTemplate = v; }, cbFileTemplate, clg);
cbRestrictFilenames.BindValueTo(v => api.Arguments.FileSystem.RestrictFilenames = v).Listen(clg);
cbNoOverwrites.BindValueTo(v => api.Arguments.FileSystem.NoOverwrites = v).Listen(clg);
cbWriteDescription.BindValueTo(v => api.Arguments.FileSystem.WriteDescription = v).Listen(clg);
cbWriteAnnotations.BindValueTo(v => api.Arguments.FileSystem.WriteAnnotations = v).Listen(clg);
#endregion
#region Network
txtProxy.BindEnableTo(cbProxy).BindValueTo(v => api.Arguments.Network.Proxy = v, cbProxy, clg);
txtSourceAddress.BindEnableTo(cbSourceAddress).BindValueTo(v => api.Arguments.Network.SourceAddress = v, cbSourceAddress, clg);
nudSocketTimeout.BindEnableTo(cbSocketTimeout).BindValueTo(v => api.Arguments.Network.SocketTimeout = (int?)v, cbSocketTimeout, clg);
rbForceIpv4.BindValueTo(v => api.Arguments.Network.ForceIpv4 = v).Listen(clg);
rbForceIpv6.BindValueTo(v => api.Arguments.Network.ForceIpv6 = v).Listen(clg);
#endregion
#region Download
void limitRateEvt(float?v)
{
api.Arguments.Download.LimitRate = (v == null || v == 0) ? (ByteSize?)null : ByteSize.From(v.Value, (FilesizeUnit)cbxLimitRateUnits.SelectedItem);
}
cbxLimitRateUnits.BindEnableTo(cbLimitRate).BindValueTo(obj => limitRateEvt(nudLimitRate.GetValueFloat(cbLimitRate))).Listen(clg);
nudLimitRate.BindEnableTo(cbLimitRate).BindValueTo(limitRateEvt, cbLimitRate, clg);
cbRetriesInf.BindEnableTo(cbRetries);
nudRetries.BindEnableTo(cbs => {
return(cbs[0].Checked && !cbs[1].Checked);
}, cbRetries, cbRetriesInf).BindValueTo(v => api.Arguments.Download.Retries = (v == null) ? null : v >= 0 ? v.ToString() : "infinite", cbRetries, clg);
cbFragmentRetriesInf.BindEnableTo(cbFragmentRetries);
nudFragmentRetries.BindEnableTo(cbs => {
return(cbs[0].Checked && !cbs[1].Checked);
}, cbFragmentRetries, cbFragmentRetriesInf).BindValueTo(v => api.Arguments.Download.FragmentRetries = (v == null) ? null : v >= 0 ? v.ToString() : "infinite", cbFragmentRetries, clg);
cbPlaylistRandom.BindValueTo(v => api.Arguments.Download.PlaylistRandom = v).Listen(clg);
cbPlaylistReversed.BindValueTo(v => api.Arguments.Download.PlaylistReverse = v).Listen(clg);
#endregion
cbWriteThumbnail.BindValueTo(v => api.Arguments.Thumbnail.WriteThumbnail = v).Listen(clg);
#region Workarounds
txtReferer.BindEnableTo(cbReferer).BindValueTo(v => api.Arguments.Workarounds.Referer = v, cbReferer, clg);
txtUserAgent.BindEnableTo(cbUserAgent).BindValueTo(v => api.Arguments.Workarounds.UserAgent = v, cbUserAgent, clg);
dgvHeaders.BindValueTo(v => {
api.Arguments.Workarounds.Headers.Clear();
foreach (DataGridViewRow row in v)
{
string key = row.Cells["cKey"].Value as string;
string value = row.Cells["cValue"].Value as string;
if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value))
{
api.Arguments.Workarounds.Headers.Add(key, value);
}
}
}).Listen(clg);
nudSleepValue.BindEnableTo(rbSleepValue).BindValueTo(v => api.Arguments.Workarounds.SleepInterval = (int?)v, rbSleepValue, clg);
nudSleepRangeMin.BindEnableTo(rbSleepRange).BindValueTo(v => api.Arguments.Workarounds.MinSleepInterval = (int?)v, rbSleepRange, clg);
nudSleepRangeMax.BindEnableTo(rbSleepRange).BindValueTo(v => api.Arguments.Workarounds.MaxSleepInterval = (int?)v, rbSleepRange, clg);
#endregion
#region Auth
string user = null, pass = null;
txtPassword.BindEnableTo(cbUsernamePassword).BindValueTo(v => api.Arguments.Authentication.Password = v, cbUsernamePassword, clg);
txtPassword.BindValueTo(v => {
pass = v;
if (string.IsNullOrWhiteSpace(user) || string.IsNullOrWhiteSpace(pass))
{
api.Arguments.Authentication.Username = null;
api.Arguments.Authentication.Password = null;
}
else
{
api.Arguments.Authentication.Username = user;
api.Arguments.Authentication.Password = pass;
}
clg.Trigger();
}, cbUsernamePassword);
txtUsername.BindEnableTo(cbUsernamePassword).BindValueTo(v => {
user = v;
if (string.IsNullOrWhiteSpace(user) || string.IsNullOrWhiteSpace(pass))
{
api.Arguments.Authentication.Username = null;
api.Arguments.Authentication.Password = null;
}
else
{
api.Arguments.Authentication.Username = user;
api.Arguments.Authentication.Password = pass;
}
clg.Trigger();
}, cbUsernamePassword, clg);
txtTwoFactor.BindEnableTo(cbTwoFactor).BindValueTo(v => api.Arguments.Authentication.TwoFactor = v, cbTwoFactor, clg);
txtVideoPassword.BindEnableTo(cbVideoPassword).BindValueTo(v => api.Arguments.Authentication.VideoPassword = v, cbVideoPassword, clg);
#endregion
#region Post Processing
api.Arguments.PostProcessing.PreferFFmpeg = true;
api.Arguments.PostProcessing.FFmpegLocation = ffmpegPath;
cbxRecodeFormat.BindEnableTo(cbRecodeFormat).BindValueTo(v => api.Arguments.PostProcessing.RecodeVideo = (VideoFormatRecode?)v, cbRecodeFormat, clg);
#endregion
#region Video Format
ControlListenGroup formatGroup = ControlListenGroup.New();
txtCustomFormatSelector.BindEnableTo(cbCustomFormatSelector).BindValueTo(v => api.Arguments.VideoFormat.Format = v, cbCustomFormatSelector, formatGroup);
cbxOutputFormat.BindEnableTo(cbOutputFormat).BindValueTo(v => api.Arguments.VideoFormat.MergeOutputFormat = (v == null) ? null : (MergeVideoFormat?)v, cbOutputFormat, clg);
formatGroup.Listen(rbVq2160p, rbVq1440p, rbVq1080p, rbVq720p, rbVq480p, rbVq360p, rbVq240p, rbVqCustom, rbFps144, rbFps120, rbFps60, rbFps30, rbFps25, rbFpsCustom);
formatGroup.Listen(cbVqPreferred, cbFpsPreferred, cbVqFallback, cbFpsFallback);
formatGroup.Listen(nudFpsCustom, nudVideoQualityWidth, nudVideoQualityHeight);
formatGroup.OnChanged += () => {
if (!cbCustomFormatSelector.Checked)
{
api.Arguments.VideoFormat.Format = GetVideoQuality().FormatSelector;
}
VideoQuality vq = GetVideoQuality();
if (!rbVqCustom.Checked)
{
nudVideoQualityWidth.Value = vq.Width;
nudVideoQualityHeight.Value = vq.Height;
}
if (!rbFpsCustom.Checked)
{
nudFpsCustom.Value = vq.Fps;
}
if (!cbCustomFormatSelector.Checked)
{
txtCustomFormatSelector.Text = vq.FormatSelector;
}
clg.Trigger();
};
formatGroup.Trigger();
#endregion
}