public override void GenerateBytecode(ClassesContainer container, ByteBlock context)
{
Class targetClass = container[CtorName];
context.Instructions.Add(new StructCreateCode(container.GetClassId(CtorName)));
if (StructCtor != null)
{
foreach (var parameter in Parameters)
{
parameter.GenerateBytecode(container, context);
}
context.Instructions.Add(new CtorCallCode(targetClass.GetCtor().Id, Parameters.Count));
}
}
public void GenerateBytecode(ClassesContainer container, ByteBlock context)
{
for (int i = 0; i < Tree.Count - 1; i++)
{
Tree[i].GenerateLoadBytecode(container, context);
}
if (Store)
{
Tree.Last().GenerateStoreBytecode(container, context);
}
else
{
Tree.Last().GenerateLoadBytecode(container, context);
}
}
public void WordSplitter_ReadLinePerformance()
{
int totalLines = 0;
int totalStrings = 0;
Stopwatch w = Stopwatch.StartNew();
Trace.WriteLine("Splitting all bug strings (compare)");
using (System.IO.StreamReader reader = new System.IO.StreamReader(SampleFilePath))
{
// 100ms just reading
string line;
while ((line = reader.ReadLine()) != null)
{
ByteBlock lineBlock = line;
RangeSet results = this.splitter.Split(lineBlock);
//HashSet<ByteBlock> words = new HashSet<ByteBlock>();
int count = 0;
int i;
for (i = 0; i < results.Count; ++i)
{
Range r = results.Ranges[i];
if (r.Length >= 2 && r.Length <= 50)
{
//ByteBlock word = new ByteBlock();
//word.Array = lineBlock.Array;
//word.Position = r.Index;
//word.Length = r.Length;
//words.Add(word);
count++;
}
}
totalStrings += count;
totalLines++;
}
}
w.Stop();
Trace.WriteLine(String.Format("{0:n0} words found on {1:n0} lines in {2:n0}ms.", totalStrings, totalLines, w.ElapsedMilliseconds));
}
public void GenerateBytecode(ClassesContainer container, ByteBlock context)
{
if (Empty)
{
context.Instructions.Add(new PushNullCode());
}
else
{
int i = 0;
while (i < Tree.Count)
{
Tree[i].GenerateBytecode(container, context);
i++;
}
}
}
public ByteBlock GetSendingBlock()
{
createSendingBlock();
var segment = new ArraySegment <byte>(stream.GetBuffer(), 0, (int)stream.Length);
if (block == null)
{
block = new ByteBlock(segment, null);
}
else
{
block.SetCount(segment);
}
return(block);
}
/// <summary>
/// Обработка входного файла
/// </summary>
protected void ProcessFile()
{
try
{
int currentBlockID = 0;
while (!readCollection.IsCompleted || readCollection.Count > 0)
{
if (!readCollection.TryTake(out ByteBlock block))
{
continue;
}
ByteBlock currentReadBlock = new ByteBlock(block.ID, GetCompressedBytes(block));
currentBlockID = block.ID;
bool isItemAdded = false;
while (!isItemAdded)
{
if (blockCountCursor == currentReadBlock.ID && writeCollection.Count < Constants.maxProcessedBlocks)
{
isItemAdded = writeCollection.TryAdd(currentReadBlock, 100);
if (inputLastBlockID == currentBlockID)
{
readCollection.CompleteAdding();
writeCollection.CompleteAdding();
}
else
{
Interlocked.Increment(ref blockCountCursor);
}
}
else
{
Thread.Sleep(Constants.ThreadDelay);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Error description: {ex.Message}");
}
}
private ByteBlock CheckHash(int blockID, bool finalBlock, ByteBlock input)
{
if (input.Count < (hmac.HashSize / 8))
{
throw new HMACException("Block insufficient size");
}
ByteBlock hash = input.Subset(0, hmac.HashSize / 8);
ByteBlock payload = input.Subset(hmac.HashSize / 8);
ByteBlock computedHash = ComputeHash(blockID, finalBlock, payload);
if (!computedHash.Equals(hash))
{
throw new HMACException("HMAC does not match");
}
return(payload);
}
public ByteBlock GetSendingBlock()
{
if (isCreateSendingBlock())
{
return(null);
}
if (block == null)
{
block = bytePool.Get(bufferSize);
}
createSendingBlock(block);
exitCreateSendingBlock();
return(block);
}
private void Write(ByteBlock byteBlock)
{
CancellationToken.ThrowIfCancellationRequested();
using (var destination = new FileStream(DestinationFilename, FileMode.OpenOrCreate, FileAccess.Write))
{
try
{
destination.Position = byteBlock.Id * (long)BlockSize;
destination.Write(byteBlock.Data, 0, byteBlock.Data.Length);
destination.Position = 0;
}
catch (Exception e)
{
OnException(Command, e);
throw;
}
}
}
public void Enqueue(ByteBlock block)
{
lock (_queue)
{
if (_isStopped)
{
throw new InvalidOperationException("Queue already stopped");
}
while (block.ID != _blockId)
{
Monitor.Wait(_queue);
}
_queue.Enqueue(block);
_blockId++;
Monitor.PulseAll(_queue);
}
}
private unsafe void CompareHashAndCopy(ByteBlock value)
{
ulong originalHash = Hash(value);
int length = value.Length;
byte[] copy = new byte[length];
value.CopyTo(copy);
ulong copyHash;
fixed(byte *key = copy)
{
copyHash = Hashing.MurmurHash3(key, length, 0);
}
Assert.AreEqual(originalHash, copyHash, String.Format("Hash of '{0}' didn't match a copy of itself.", value));
}
public void WordSplitter_ReadBlockPerformance()
{
int totalLines = 0;
int totalStrings = 0;
Stopwatch w = Stopwatch.StartNew();
Trace.WriteLine("Splitting all bug strings (compare)");
using (System.IO.StreamReader reader = new System.IO.StreamReader(SampleFilePath))
{
byte[] buffer = new byte[8 * 1024];
while (true)
{
int length = reader.BaseStream.Read(buffer, 0, buffer.Length);
if (length == 0)
{
break;
}
ByteBlock block = new ByteBlock(buffer);
RangeSet results = this.splitter.Split(block);
int count = 0;
int i;
for (i = 0; i < results.Count; ++i)
{
Range r = results.Ranges[i];
if (r.Length >= 2 && r.Length <= 50)
{
count++;
}
}
totalStrings += count;
totalLines++;
}
}
w.Stop();
System.Console.WriteLine("{0:n0} words found on {1:n0} lines in {2:n0}ms.", totalStrings, totalLines, w.ElapsedMilliseconds);
}
public void Process(object doneEvent)
{
if (!(doneEvent is ManualResetEvent))
{
throw new ArgumentException();
}
try
{
while (!_fileProcessData.Cancelled)
{
ByteBlock block = _queueReader.Dequeue();
if (block == null)
{
return;
}
using (var memoryStream = new MemoryStream())
{
using (var gzs = new GZipStream(memoryStream, CompressionMode.Compress))
{
gzs.Write(block.Buffer, 0, block.Buffer.Length);
}
byte[] compressedData = memoryStream.ToArray();
var bb = new ByteBlock
{
Id = block.Id,
Buffer = compressedData
};
_queueWriter.EnqueueForWriting(bb);
}
((ManualResetEvent)doneEvent).Set();
}
}
catch (Exception ex)
{
_fileProcessData.Cancelled = true;
throw new Exception(ex.Message);
}
}
public override void GenerateBytecode(ClassesContainer container, ByteBlock context)
{
int jumpIndex = context.NextOpIndex;
Condition.GenerateBytecode(container, context);
JumpIfFalseCode jumpIfFalse = new JumpIfFalseCode(-1);
context.Instructions.Add(jumpIfFalse);
foreach (var statement in Statements)
{
statement.GenerateBytecode(container, context);
}
context.Instructions.Add(new JumpCode(jumpIndex));
jumpIfFalse.targetIndex = context.NextOpIndex;
}
public void Receive(ByteBlock block)
{
Ensure.IsNotNull(block);
try
{
receiveSocket.SetBuffer(block.SegmentArray, block.SegmentOffset, block.Length);
receiveSocket.UserToken = block;
if (!socket.ReceiveAsync(receiveSocket))
{
processReceived(receiveSocket);
}
}
catch (Exception)
{
shutdownSocket();
}
}
public void Send(ByteBlock block)
{
Ensure.IsNotNull(block);
try
{
sendingSocket.SetBuffer(block.SegmentArray, block.SegmentOffset, block.Count);
sendingSocket.UserToken = block;
if (!socket.SendAsync(sendingSocket))
{
processSend(sendingSocket);
}
}
catch (Exception)
{
shutdownSocket();
}
}
private void ReadUncompressed()
{
using (var stream = File.Open(_filePath, FileMode.Open, FileAccess.Read))
{
int id = 1;
while (stream.Position < stream.Length)
{
var arraySize = GetBlockSize(stream.Length, stream.Position);
var byteArray = new byte[arraySize];
stream.Read(byteArray, 0, arraySize);
var block = new ByteBlock(id, byteArray);
_queue.Enqueue(block);
id++;
}
_queueDispatcher.CloseQueue();
}
}
public override void StartWork()
{
bool canWork = true;
while (canWork)
{
bool canSend = false;
ByteBlock byteBlock = null;
lock (readLock)
{
if (readQueue.Count > 0)
{
//Read data from queue
byteBlock = readQueue.Dequeue();
canSend = true;
canWork = !byteBlock.lastSubset;
}
}
if (canSend)
{
switch (CompressionType)
{
case Compression.Enums.CompressionType.Compress:
byteBlock.Slice = compression.Compress(byteBlock.Slice);
break;
case Compression.Enums.CompressionType.Decompress:
byteBlock.Slice = compression.Decompress(byteBlock.Slice);
break;
}
lock (writeLock)
{
//write data to queue
writeQueue.Enqueue(byteBlock);
}
}
else
{
Thread.Sleep(1);
}
}
}
public void Read(object sourceFile)
{
try
{
int blockId = 0;
using (FileStream compressedFile = new FileStream(sourceFile.ToString(), FileMode.Open))
{
while (compressedFile.Position < compressedFile.Length)
{
var lengthBuffer = new byte[Constants.FirstByteCountToDecompress];
compressedFile.Read(lengthBuffer, 0, lengthBuffer.Length);
int blockLength = BitConverter.ToInt32(lengthBuffer, 4);
var compressedData = new byte[blockLength];
lengthBuffer.CopyTo(compressedData, 0);
compressedFile.Read(compressedData, Constants.FirstByteCountToDecompress, blockLength - Constants.FirstByteCountToDecompress);
int dataSize = BitConverter.ToInt32(compressedData, blockLength - 4);
byte[] lastBuffer = new byte[dataSize];
ByteBlock block = new ByteBlock
{
Id = blockId,
Buffer = lastBuffer,
CompressedBuffer = compressedData
};
_queueReader.EnqueueForDecompressing(block);
if ((compressedFile.Length - compressedFile.Position) / Constants.BlockSize == 0)
{
_fileProcessData.LastBlockId = blockId;
}
blockId++;
_gZipProgress.ProcessProgress(compressedFile.Position, compressedFile.Length);
}
_queueReader.Stop();
}
}
catch (Exception ex)
{
_fileProcessData.Cancelled = true;
throw new Exception(ex.Message);
}
}
public void ByteBlockAppender_Basic()
{
Highlighter.ByteBlockAppender appender;
ByteBlock rawValue = ByteBlock.TestBlock("active <div class='active'>Sample Value</div> Editors 39400 found within 394000 today");
// If no changes, verify same array returned
appender = new Highlighter.ByteBlockAppender(rawValue);
appender.AppendRemainder();
Assert.AreSame(rawValue.Array, appender.Value().Array);
// Verify AppendTo current position does nothing
appender = new Highlighter.ByteBlockAppender(rawValue);
appender.AppendTo(0);
Assert.AreEqual("", appender.Value().ToString());
// Verify Append, AppendTo properly wrap value
appender.Append("[");
Assert.IsTrue(appender.AppendTo(6));
appender.Append("]");
Assert.AreEqual("[active]", appender.Value().ToString());
// Verify AppendTo properly tracks yet-to-add content
appender.AppendTo(46);
appender.Append("[");
appender.AppendTo(49);
appender.Append("]");
Assert.AreEqual("[active] <div class='active'>Sample Value</div> [Edi]", appender.Value().ToString());
// Verify AppendTo will not re-highlight previous content (for words split multiple ways)
Assert.IsFalse(appender.AppendTo(46));
// Verify AppendTo handles back-to-back wrapping
Assert.IsTrue(appender.AppendTo(49));
appender.Append("[");
appender.AppendTo(53);
appender.Append("]");
Assert.AreEqual("[active] <div class='active'>Sample Value</div> [Edi][tors]", appender.Value().ToString());
// Verify AppendRemainder catches rest
appender.AppendRemainder();
Assert.AreEqual("[active] <div class='active'>Sample Value</div> [Edi][tors] 39400 found within 394000 today", appender.Value().ToString());
}
/// <summary>
/// Чтение входного файла
/// </summary>
private void Read()
{
try
{
long currentPos = 0;
long inputFileLenght = GetFileLength(inputFileName);
int bytesReadLenght;
byte[] currentFileBlock;
int blockIdCursor = 0;
while (currentPos < inputFileLenght)
{
bytesReadLenght = GetLengthToRead(inputFileLenght, currentPos);
currentFileBlock = GetFileBlock(inputFileName, currentPos, bytesReadLenght);
currentPos += bytesReadLenght;
blockIdCursor++;
ByteBlock readBlock = new ByteBlock(blockIdCursor, currentFileBlock);
while (true)
{
if (readCollection.Count() < Constants.maxProcessedBlocks)
{
readCollection.Add(readBlock);
break;
}
else
{
Thread.Sleep(Constants.ThreadDelay);
}
}
}
inputLastBlockID = blockIdCursor;
readCollection.CompleteAdding();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
public void ByteBlock_CompareConstantTime_RandomTests(byte a, byte b, byte c, byte d)
{
try
{
Console.WriteLine("a:{0}, b: {1}, c:{2}", a, b, c);
Assert.AreEqual(ByteBlock.CompareConstantTime(new byte[] { a }, new byte[] { b }), Compare2(a, b));
Assert.AreEqual(ByteBlock.CompareConstantTime(new byte[] { c, a }, new byte[] { c, b }), Compare2(a, b));
if (a != b)
{
Assert.AreEqual(ByteBlock.CompareConstantTime(new byte[] { a, c }, new byte[] { b, c }), Compare2(a, b));
Assert.AreEqual(ByteBlock.CompareConstantTime(new byte[] { a, b }, new byte[] { b, a }), Compare2(a, b));
Assert.AreEqual(ByteBlock.CompareConstantTime(new byte[] { a, c }, new byte[] { b, d }), Compare2(a, b));
}
}
catch (Exception ex)
{
Console.WriteLine("Fail: " + ex.Message);
throw;
}
}
private void Compress(ByteBlock byteBlock)
{
CancellationToken.ThrowIfCancellationRequested();
using (MemoryStream memoryStream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Compress))
{
try
{
gZipStream.Write(byteBlock.Data, 0, byteBlock.Data.Length);
}
catch (Exception e)
{
OnException(Command, e);
}
}
ByteBlock compressedByteBlock = ByteBlock.FromData(byteBlock.Id, byteBlock.Data, memoryStream.GetBuffer());
_writingQueue.Enqueue(compressedByteBlock);
}
}
protected override void DoWorkImpl()
{
while (true)
{
var block = InputQueue.Dequeue();
if (block == null)
{
break;
}
using (var inStream = new MemoryStream(block.Bytes))
using (var gzipStream = new GZipStream(inStream, CompressionMode.Decompress))
using (var outStream = new MemoryStream())
{
gzipStream.CopyTo(outStream);
var decompressedBlock = new ByteBlock(block.ID, outStream.ToArray());
ResultQueue.Enqueue(decompressedBlock, true);
}
}
}
private void ReadCompressed()
{
using (var stream = File.Open(_filePath, FileMode.Open, FileAccess.Read))
{
int id = 1;
while (stream.Position < stream.Length)
{
var blockLengthArray = new byte[8];
stream.Read(blockLengthArray, 0, blockLengthArray.Length);
var blockLength = BitConverter.ToInt32(blockLengthArray, 4);
stream.Position = stream.Position - 8;
var block = new ByteBlock(id, new byte[blockLength]);
stream.Read(block.Bytes, 0, blockLength);
_queue.Enqueue(block);
id++;
}
_queueDispatcher.CloseQueue();
}
}
public override void GenerateBytecode(ClassesContainer container, ByteBlock context)
{
switch (OperatorEnum)
{
case OperatorsEnum.Plus:
context.Instructions.Add(new AddCode());
break;
case OperatorsEnum.Minus:
context.Instructions.Add(new SubCode());
break;
case OperatorsEnum.Multiply:
context.Instructions.Add(new MulCode());
break;
case OperatorsEnum.Divide:
context.Instructions.Add(new DivCode());
break;
case OperatorsEnum.Inferior:
context.Instructions.Add(new ComparaisonCode(OperatorsEnum.Inferior));
break;
case OperatorsEnum.Superior:
context.Instructions.Add(new ComparaisonCode(OperatorsEnum.Superior));
break;
case OperatorsEnum.Different:
context.Instructions.Add(new ComparaisonCode(OperatorsEnum.Different));
break;
case OperatorsEnum.Equals:
context.Instructions.Add(new ComparaisonCode(OperatorsEnum.Equals));
break;
default:
throw new Exception();
}
}
public void WordSplitter_ParallelBlockPerformance()
{
int totalLines = 0;
int totalStrings = 0;
object locker = new object();
Stopwatch w = Stopwatch.StartNew();
Trace.WriteLine("Splitting all bug strings (compare)");
Parallel.ForEach(ReadBlocks(SampleFilePath), (buffer) =>
{
// Wrong - need to figure out how to tell.
int length = 8 * 1024;//reader.BaseStream.Read(buffer, 0, buffer.Length);
ByteBlock block = new ByteBlock(buffer, 0, length);
RangeSet results = this.splitter.Split(block);
int count = 0;
int i;
for (i = 0; i < results.Count; ++i)
{
Range r = results.Ranges[i];
if (r.Length >= 2 && r.Length <= 50)
{
count++;
}
}
lock (locker)
{
totalStrings += count;
totalLines++;
}
});
w.Stop();
System.Console.WriteLine("{0:n0} words found on {1:n0} lines in {2:n0}ms.", totalStrings, totalLines, w.ElapsedMilliseconds);
}
public void Process(object doneEvent)
{
if (!(doneEvent is ManualResetEvent))
{
throw new ArgumentException();
}
try
{
while (!_fileProcessData.Cancelled)
{
ByteBlock byteBlock = _queueReader.Dequeue();
if (byteBlock == null)
{
return;
}
using (MemoryStream ms = new MemoryStream(byteBlock.CompressedBuffer))
{
using (GZipStream gzs = new GZipStream(ms, CompressionMode.Decompress))
{
gzs.Read(byteBlock.Buffer, 0, byteBlock.Buffer.Length);
byte[] decompressedData = byteBlock.Buffer.ToArray();
ByteBlock block = new ByteBlock
{
Id = byteBlock.Id,
Buffer = decompressedData
};
_queueWriter.EnqueueForWriting(block);
}
((ManualResetEvent)doneEvent).Set();
}
}
}
catch (Exception ex)
{
_fileProcessData.Cancelled = true;
throw new Exception(ex.Message);
}
}
public void ByteBlock_Null()
{
ByteBlock nullBlock = default(ByteBlock);
ByteBlock nullBlock2 = (string)null;
ByteBlock nullBlock3 = (byte[])null;
ByteBlock valid = "valid";
Assert.IsTrue(nullBlock.IsZero());
Assert.IsTrue(nullBlock.Equals(nullBlock));
Assert.IsFalse(nullBlock.Equals(valid));
Assert.IsFalse(valid.Equals(nullBlock));
Assert.IsTrue(nullBlock == nullBlock2);
Assert.IsFalse(nullBlock == valid);
Assert.IsFalse(valid == nullBlock);
Assert.IsFalse(nullBlock != nullBlock2);
Assert.IsTrue(nullBlock != valid);
Assert.IsTrue(valid != nullBlock);
Assert.IsTrue(nullBlock.CompareTo(nullBlock2) == 0);
}
/// <summary>
/// Write a ByteBlock [UTF8 text] to the given stream, escaped to be
/// safe within HTML as inner text or attribute values.
/// </summary>
/// <param name="value">ByteBlock to write</param>
/// <param name="stream">Stream to write to</param>
public void WriteAsHtmlText(ByteBlock value)
{
int nextIndexToCopy = value.Index;
int end = value.Index + value.Length;
int length;
for (int i = nextIndexToCopy; i < end; ++i)
{
byte c = value.Array[i];
// Escape: Amperstand, LessThan, GreaterThan, Quote, Apostrophe
if (c == UTF8.Amperstand || c == UTF8.LessThan || c == UTF8.GreaterThan || c == UTF8.DoubleQuote || c == UTF8.Apostrophe)
{
// Copy characters before the one requiring escaping
length = i - nextIndexToCopy;
if (length > 0)
{
stream.Write(value.Array, nextIndexToCopy, length);
}
// Write character as "&#xx;" with the decimal keypoint
stream.WriteByte(UTF8.Amperstand);
stream.WriteByte(UTF8.Pound);
stream.WriteByte((byte)(UTF8.Zero + (c / 10)));
stream.WriteByte((byte)(UTF8.Zero + (c % 10)));
stream.WriteByte(UTF8.Semicolon);
// Set the next character to copy (the one after this one)
nextIndexToCopy = i + 1;
}
}
// Copy the remaining characters
length = end - nextIndexToCopy;
if (length > 0)
{
stream.Write(value.Array, nextIndexToCopy, length);
}
}
