public DFFrame(OptionsStruct opt)
{
_frames = new List<DFFrameSub>();
int height = (int)opt.resolution;
int width = height * 16 / 9;
//always need one fullsize subframe;
// DFFrameSub subFrame = new DFFrameSub(width, height);
_frames.Add(new DFFrameSub(width, height));
switch (opt.pxlFmtIn)
{
case PixelFormat.RGB24:
case PixelFormat.YUV444P:
//add 2 fullsize subframes
_frames.Add(new DFFrameSub(width, height));
_frames.Add(new DFFrameSub(width, height));
Size = Size * 3;
break;
case PixelFormat.YUV420P:
//add 2 halfsize subframes
_frames.Add(new DFFrameSub(width / 2, height / 2));
_frames.Add(new DFFrameSub(width / 2, height / 2));
Size = Size + Size / 2;
break;
}
Size = 0;
foreach(var sub in _frames)
{
Size += sub.Size;
}
}
public NodeFrameFromMP4(String path, OptionsStruct opt, IPipe Output, NodeToken token) : base(null, Output)
{
this.opt = opt;
this.token = token;
InitFrameSize();
ffmpeg = new DFffmpeg(GenerateArgs(path, opt));
}
public NodeFrameFromYoutube(String url, OptionsStruct opt, IPipe Output, NodeToken token)
: base(null, Output)
{
this.opt = opt;
this.token = token;
InitFrameSize();
ffmpeg = new DFffmpeg(GenerateArgsFFmpeg(opt));
youtube = new DFYoutube(GenerateArgsYoutube(url));
}
public NodeBlockBase(OptionsStruct options, IPipe input, IPipe output, NodeToken token) : base(input, output)
{
this.options = options;
this.token = token;
valueBounds = new int[2];
snakeIteratorCore = new int[64, 2];
SnakeIteratorInit();
BoundsInit();
blockarray = new byte[64];
dctarray = new float[8, 8];
MaxError = 0;
}
static public void ToPipe(string path, OptionsStruct opt, Pipe pipeOut)
{
int size = 0;
var fi = new FileInfo(path);
long fileSize = fi.Length;
size += sizeof(long); //8
//opt.density
size += 1;
//opt.cellCount
size += 1;
//fileName length
string name = Path.GetFileName(path);
byte[] nameArray = System.Text.UTF8Encoding.UTF8.GetBytes(name);
int nameSize = nameArray.Length;
if (nameSize >= 256 * 256)
{
throw new ArgumentOutOfRangeException("File name length");
}
size += 2;
size += nameSize;
byte[] result = new byte[size];
int i = 0;
while (i < 8)
{
result[i] = (byte)(fileSize % 256);
fileSize = fileSize / 256;
i++;
}
result[i] = opt.density;
i++;
result[i] = opt.cellCount;
i++;
int temp = nameSize;
result[i] = (byte)(temp % 256);
temp /= 256;
i++;
result[i] = (byte)(temp % 256);
i++;
for (int j = 0; j < nameSize; j++)
{
result[i + j] = nameArray[j];
}
pipeOut.Add(result);
pipeOut.Complete();
}
private String GenerateArgsFFmpeg(String path, OptionsStruct opt)
{
string result =
"-y -i "
+ "-"
+ " "
+ "-f image2pipe "
+ "-pix_fmt "
+ opt.pxlFmtIn.ToName()
+ " "
+ "-vcodec rawvideo "
+ "-";
return(result);
}
private String GenerateArgsFFmpeg(OptionsStruct opt)
{
string result =
"-y -i "
+ "- "
// + "http://127.0.0.1:12345 "
+ "-f image2pipe "
+ "-pix_fmt "
+ opt.pxlFmtIn.ToName()
+ " "
+ "-vcodec rawvideo "
+ "-";
return(result);
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await(Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List <LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
await eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}
private String GenerateArgs(String path, OptionsStruct opt)
{
string result =
"-y -f rawvideo -vcodec rawvideo "
+ "-s "
+ ((int)opt.resolution * 16 / 9).ToString()
+ "x"
+ ((int)opt.resolution).ToString()
+ " "
+ "-pix_fmt "
+ opt.pxlFmtIn.ToName()
+ " "
+ "-r 24 " //fps
+ "-i - "
+ "-pix_fmt "
+ opt.pxlFmtOut.ToName()
+ " "
+ "-c:v libx264 ";
if (opt.isEncodingCRF)
{
result += "-crf ";
}
else
{
result += "-b:v ";
}
result +=
opt.videoQuality.ToString()
+ " "
+ "-preset "
+ opt.encodingPreset.ToString()
+ " "
+ "-an "
+ "\""
+ path
+ "\"";
return(result);
}
public NodeBitsToBlock(OptionsStruct opt, IPipe input, IPipe output, NodeToken token)
: base(opt, input, output, token)
{
}
public void EncodeFile(string path, OptionsStruct opt)
{
_tokenSource = new CancellationTokenSource();
token.token = false;
/*
* OptionsStruct opt = new OptionsStruct(0);
* opt.encodingPreset = EncodingPreset.medium;
* opt.cellCount = 1;
* opt.density = 4;
* opt.isEncodingCRF = true;
* opt.videoQuality = 20;
* opt.pxlFmtIn = PixelFormat.YUV420P;
* opt.pxlFmtOut = PixelFormat.YUV420P;
* opt.resolution = ResolutionsEnum.p720;
*/
Monitor.CurrentWork = 0;
string filename =
Path.GetDirectoryName(path) + @"\" +
Path.GetFileName(path) + @".mp4";
OptionsStruct headopt = opt;
headopt.density = 1;
headopt.cellCount = 1;
List <ChainNode> nodes = new List <ChainNode>();
Pipe pipein = new Pipe(_tokenSource.Token);
ChainNode node;
node = new NodeReader(path, pipein, Monitor);
nodes.Add(node);
Pipe pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeECCEncoder(pipeout, pipein);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeToBits(pipeout, pipein);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeBitsToBlock(opt, pipeout, pipein, token);
nodes.Add(node);
Pipe pipebody = pipein;
//HEADER START
pipein = new Pipe(_tokenSource.Token);
DataHeader.ToPipe(path, opt, pipein);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeToBits(pipeout, pipein);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeBitsToBlock(headopt, pipeout, pipein, token);
Pipe pipeheader = pipein;
nodes.Add(node);
//HEADER END
var plist = new List <Pipe>();
plist.Add(pipeheader);
plist.Add(pipebody);
IPipe pipejoiner = new PipeJoiner(_tokenSource.Token, plist);
pipein = new Pipe(_tokenSource.Token);
node = new NodeBlocksToFrame(opt, pipejoiner, pipein);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeFrameToMP4(filename, opt, pipeout);
//node = new NodeDebugRawStorage(pipeout, null);
nodes.Add(node);
Task.Run(() =>
{
Parallel.ForEach(nodes, (n) => n.Start());
Monitor.Add(1);
}
,
_tokenSource.Token
);
}
private void Decode(string path, string url)
{
Monitor.CurrentWork = 0;
_tokenSource = new CancellationTokenSource();
token.token = false;
int width;
if (url == "")
{
width = (new MP4Info()).GetWidthFromPath(path);
}
else
{
width = (new MP4Info()).GetWidthFromUrl(url);
}
OptionsStruct opt = new OptionsStruct(0);
opt.cellCount = 1;
opt.density = 1;
opt.pxlFmtIn = PixelFormat.YUV420P;
opt.pxlFmtOut = PixelFormat.YUV420P;
opt.resolution = (ResolutionsEnum)width;
OptionsStruct headopt = opt;
opt.cellCount = 1;
opt.density = 1;
List <ChainNode> nodes = new List <ChainNode>();
Pipe pipein = new Pipe(_tokenSource.Token);
ChainNode node;
/*
* node = new NodeDebugRawStorage(null,pipein);
*/
if (url == "")
{
node = new NodeFrameFromMP4(path, opt, pipein, token);
}
else
{
node = new NodeFrameFromYoutube(url, opt, pipein, token);
}
nodes.Add(node);
ChainNode n1 = node;
Pipe pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeBlocksFromFrame(opt, pipeout, pipein, token);
ChainNode n2 = node;
nodes.Add(node);
pipeout = pipein;
// Task.Run(()=>Parallel.ForEach(nodes, (n) => n.Start()));
Task.Run(() => { n1.Start(); }, _tokenSource.Token);
Task.Run(() => { n2.Start(); }, _tokenSource.Token);
nodes = new List <ChainNode>();
//HEADER START
string fileName = "";
long fileSize = 0;
DataHeader.FromPipe(ref opt, ref fileName, ref fileSize, pipeout);
fileName = System.IO.Path.GetDirectoryName(path) + @"\" + fileName;
//HEADER END
pipein = new Pipe(_tokenSource.Token);
node = new NodeBitsFromBlock(opt, pipeout, pipein, token);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeFromBits(pipeout, pipein, token);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeECCDecoder(pipeout, pipein);
nodes.Add(node);
pipeout = pipein;
pipein = new Pipe(_tokenSource.Token);
node = new NodeWriter(fileName, fileSize, pipeout, Monitor);
nodes.Add(node);
Task.Run(() =>
{
Parallel.ForEach(nodes, (n) => n.Start());
}
, _tokenSource.Token
);
Task.Run(() =>
{
while (Monitor.TotalWork != (Monitor.CurrentWork + 1))
{
System.Threading.Thread.Sleep(100);
}
Monitor.Add(1);
_tokenSource.Cancel();
token.token = true;
}
);
}
public NodeBlocksToFrame(OptionsStruct opts, IPipe input, IPipe output) : base(input, output)
{
this.opts = opts;
}
public void GetLogFilePathsForMessageFileTime()
{
OptionsStruct options = new OptionsStruct();
options.LogDirectory = LOG_FILE_PATH;
aaLogReader alr = new aaLogReader(options);
List<LogHeader> logHeaders = alr.LogHeaderIndex;
// Loop through all entries in the header index and verify we correctly identify the first, last, and middle message
foreach(LogHeader lh in logHeaders)
{
Assert.That(alr.GetLogFilePathsForMessageFileTime(lh.EndFileTime).Exists(x=>x==lh.LogFilePath),"End FileTime log path not correctly identified");
Assert.That(alr.GetLogFilePathsForMessageFileTime(lh.StartFileTime).Exists(x => x == lh.LogFilePath), "Start FileTime log path not correctly identified");
Assert.That(alr.GetLogFilePathsForMessageFileTime((lh.StartFileTime + lh.EndFileTime)/2).Exists(x => x == lh.LogFilePath), "Middle FileTime log path not correctly identified");
}
}
/*
* private Mat _DCTCoreFirst;
* private Mat _DCTCoreSnake;
*/
public ByteAssembler_(OptionsStruct options)
{
// Func<byte>
}
public void GetLogFilePathsForMessageNumber()
{
OptionsStruct options = new OptionsStruct();
options.LogDirectory = LOG_FILE_PATH;
ulong RandomUlong;
System.Random rnd = new Random(DateTime.Now.Millisecond);
aaLogReader alr = new aaLogReader(options);
List<LogHeader> logHeaders = alr.LogHeaderIndex;
// Loop through all entries in the header index and verify we correctly identify the first, last, and middle message
foreach (LogHeader lh in logHeaders)
{
Assert.That(alr.GetLogFilePathsForMessageNumber(lh.StartMsgNumber).Exists(x => x == lh.LogFilePath), "End Message Number log path not correctly identified");
Assert.That(alr.GetLogFilePathsForMessageNumber(lh.EndMsgNumber).Exists(x => x == lh.LogFilePath), "Start Mesage Number log path not correctly identified");
Assert.That(alr.GetLogFilePathsForMessageNumber((lh.StartMsgNumber + lh.EndMsgNumber) / 2).Exists(x => x == lh.LogFilePath), "Middle Message Number log path not correctly identified");
RandomUlong = (ulong)rnd.Next(Convert.ToInt32(lh.MsgCount)) + lh.StartMsgNumber;
Assert.That(alr.GetLogFilePathsForMessageNumber(RandomUlong).Exists(x => x == lh.LogFilePath), "Random Message Number " + RandomUlong + " log path not correctly identified");
}
}
public NodeFrameFromYoutube(String path, OptionsStruct opt, IPipe Output) : base(null, Output)
{
this.opt = opt;
InitFrameSize();
ffmpeg = new DFffmpeg(GenerateArgsFFmpeg(path, opt));
}
public NodeFrameToMP4(String path, OptionsStruct opt, IPipe Input) : base(Input, null)
{
this.opt = opt;
ffmpeg = new DFffmpeg(GenerateArgs(path, opt));
InitFrameSize();
}
static public void FromPipe(ref OptionsStruct opt,
ref String fileName,
ref long fileSize,
Pipe pipeBlocks)
{
Queue <byte> queue = new Queue <byte>(headLength * 8);
int k = 0;
int todo = headLength;
byte[] header = new byte[headLength];
int nameSize = 0;
//do decoding for params
//density=1
//cellcount=1
while (k < todo)
{
// get 64 bytes = block
byte[] block = pipeBlocks.Take(64);
// average value of block
int sum = 0;
for (byte q = 0; q < 64; q++)
{
sum += block[q];
}
double tmp = (sum / 64.0);
//convert cell value to byte sequence
tmp /= 255;
byte result = (byte)Math.Round(tmp);
queue.Enqueue(result);
if (queue.Count == 8)
{
byte b = 0;
for (int q = 0; q < 8; q++)
{
b *= 2;
b += queue.Dequeue();
}
header[k] = b;
k++;
if (k == headLength)
{
fileSize = 0;
int x;
for (x = 7; x >= 0; x--)
{
fileSize *= 256;
fileSize += header[x];
}
x = 8;
opt.density = header[x];
x++;
opt.cellCount = header[x];
x++;
nameSize = header[x + 1] * 256 + header[x];
todo += nameSize;
Array.Resize <byte>(ref header, headLength + nameSize);
}
}
}
byte[] name = new byte[nameSize];
Array.Copy(header, headLength, name, 0, nameSize);
fileName = System.Text.UTF8Encoding.UTF8.GetString(name);
}
private void Awake()
{
s_instance ??= this;
_data = new OptionsStruct(true);
ApplyOptions();
}
public NodeBlocksFromFrame(OptionsStruct opts, IPipe input, IPipe output, NodeToken token) : base(input, output)
{
this.opts = opts;
this.token = token;
}
static void Main(string[] args)
{
try
{
ulong messageCount = 0;
aaLogReader.OptionsStruct options = new OptionsStruct();
string aaLGXDirectory = "";
log.Info("Starting Log Collection");
// Read the message count as the single argument to the exe
try
{
messageCount = System.Convert.ToUInt32(args[0]);
}
catch (Exception)
{
//Ignore the error and use default value of 1000
log.InfoFormat("No maximum message count specified. Include a single integer argument when calling the application to set the maximum message count retrieved");
messageCount = 1000;
}
// Final check to make sure we don't have a null messageCount
if (messageCount <= 0)
{
messageCount = 1000;
}
try
{
// Read the aaLGX directory command line switch
if (args.Length >= 2)
{
aaLGXDirectory = args[1] ?? "";
}
if (aaLGXDirectory != "")
{
log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory);
if (Directory.Exists(aaLGXDirectory))
{
string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx");
foreach (string fileName in filesList)
{
log.InfoFormat("Processing file {0}", fileName);
var aaLGXRecords = aaLogReader.aaLgxReader.ReadLogRecords(fileName);
log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName);
foreach (LogRecord record in aaLGXRecords)
{
Console.WriteLine(record.ToKVP());
}
log.InfoFormat("Deleting {0} after reading records.", fileName);
File.Delete(fileName);
}
}
else
{
log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory);
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
//Read the options from a local file
try
{
if (System.IO.File.Exists("options.json"))
{
log.DebugFormat("Reading Options from local options.json");
options = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json"));
}
}
catch
{
log.InfoFormat("No options.json file exists. Using default options");
options = new OptionsStruct();
}
log.InfoFormat("Reading records with maximum message count of {0}", messageCount);
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(options);
List <LogRecord> logRecords = logReader.GetUnreadRecords(maximumMessages: messageCount);
log.InfoFormat("{0} unread log records found", logRecords.Count);
// If we have any records then output kvp format to the console
if (logRecords.Count > 0)
{
foreach (LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
public void OpenCurrentLogFileWithOptions()
{
OptionsStruct options = new OptionsStruct();
options.LogDirectory = LOG_FILE_PATH;
aaLogReader alr = new aaLogReader(options);
ReturnCodeStruct rcs = alr.OpenCurrentLogFile();
Assert.That(rcs.Status);
Assert.That(rcs.Message.Length == 0);
}
public NodeMetaHeaderToBlocks(string path, OptionsStruct opt, IPipe output) : base(null, output)
{
this.opt = opt;
}
public void CancelOptions()
{
_data = new OptionsStruct(true);
ApplyOptions();
}
