public void ExecuteFinishStepTest()
{
//Arrange
const EncoderState encoderState = EncoderState.Cancelled;
const string fileHash = "fileHash";
const string errorMessage = "errorMessage";
var pipelineStep = new FinishStep(_pipelineMediator.Object, _restClient.Object, _tempFileManager.Object);
var stepData = new UploadStepData()
{
EncoderState = encoderState,
FileHash = fileHash,
ErrorMessage = errorMessage
};
pipelineStep.SetData(stepData);
//Act
pipelineStep.Execute(It.IsAny <CancellationTokenSourceWrapper>());
//Assert
_tempFileManager.Verify(m => m.DeleteAllTempFiles());
_restClient.Verify(m => m.FinishTask(encoderState, fileHash, errorMessage), Times.Once());
}
private void WriteHorizontalCodes(EncoderState state, bool startingColor, int firstSpan, int secondSpan)
{
state.Writer.Write(HorizontalCodes[0], HorizontalCodes[1]);
WriteHorizontalSpan(state, startingColor, firstSpan);
WriteHorizontalSpan(state, !startingColor, secondSpan);
}
private EncoderStatus CreateEncoderStatus(EncoderState encoderState, string errorMessage)
{
return(new EncoderStatus
{
EncoderState = encoderState,
ErrorMessage = errorMessage
});
}
private EncoderStatus CreateEncoderStatus(EncoderState encoderState, string errorMessage)
{
return new EncoderStatus
{
EncoderState = encoderState,
ErrorMessage = errorMessage
};
}
public IRestRequest FinishTaskRequestCreate(string resource, EncoderState encoderState, string fileId, string errorMessage)
{
IRestRequest request = RequestCreate(resource, Method.POST);
request.AddHeader("X-HTTP-Method-Override", "DELETE");
request.AddParameter(EncoderStatusParameters.Result, encoderState);
request.AddParameter(EncoderStatusParameters.FileId, fileId);
request.AddParameter(EncoderStatusParameters.Message, errorMessage);
return(request);
}
private static string FormatIr(MethodBody methodBody)
{
var encoder = new EncoderState();
var encodedImpl = encoder.Encode(methodBody.Implementation);
return(Les2LanguageService.Value.Print(
encodedImpl,
options: new LNodePrinterOptions
{
IndentString = new string(' ', 4)
}));
}
/// <summary>
/// Notify the encoder that the input stream is finished.
/// If the return value is HSER_FINISH_MORE, there is still more output, so call poll() and repeat
/// </summary>
/// <returns>Refer to HSE_finish_res in original C version heatshrink</returns>
public EncoderFinishResult Finish()
{
_flags |= FlagIsFinishing;
if (Constants.EnableLogging)
{
Console.WriteLine("-- setting is_finishing flag");
}
if (_state == EncoderState.NotFull)
{
_state = EncoderState.Filled;
}
return(_state == EncoderState.Done ? EncoderFinishResult.Done : EncoderFinishResult.More);
}
private static string FormatIr(Flame.Compiler.MethodBody methodBody)
{
// TODO: deduplicate this logic (it also appears in IL2LLVM and ILOpt)
var encoder = new EncoderState();
var encodedImpl = encoder.Encode(methodBody.Implementation);
return(Les2LanguageService.Value.Print(
encodedImpl,
options: new LNodePrinterOptions
{
IndentString = new string(' ', 4)
}));
}
/// <summary>
/// Reset the encoder
/// </summary>
public void Reset()
{
Array.Clear(_buffer, 0, _buffer.Length);
_inputSize = 0;
_state = EncoderState.NotFull;
_matchScanIndex = 0;
_flags = 0;
_bitIndex = 0x80;
_currentByte = 0x00;
_matchLength = 0;
_outgoingBits = 0x0000;
_outgoingBitsCount = 0;
}
public void FinishTaskTest()
{
//Arrange
const EncoderState encoderState = EncoderState.Failed;
const string fileHash = "fileHash";
const string errorMessage = "errorMessage";
_restHelper.Setup(m => m.FinishTaskRequestCreate(Resource, encoderState, fileHash, errorMessage)).Returns(_request.Object);
//Act
_encodeWebClient.FinishTask(encoderState, fileHash, errorMessage);
//Assert
_restHelper.Verify(m => m.GetResponse(_request.Object), Times.Once());
}
public int Encode(Stream input, long inputStreamStartOffset, int width, int height, Stream output)
{
var rowLengthInBytes = (((int)width + 31) / 32) * 4; //bmp row data is in a multiple of 4 bytes
// do not .Seek() the output. The caller may want us to append to the end. We store the starting position
//here so we can return how many bytes were written.
var outputStartPosition = output.Position;
var state = new EncoderState()
{
CodingReader = new BitReader(new byte[rowLengthInBytes], width), //this will get swapped to referenceReader on the first run
ReferenceReader = new BitReader(new byte[rowLengthInBytes], width),
Writer = new BitWriter(output),
Width = width
};
//fill the current row with all 1's. It'll be used as the reference row for the first real image row.
for (int i = 0; i < state.CodingReader.Buffer.Length; i++)
{
state.CodingReader.Buffer[i] = 0xff;
}
for (int row = 0; row < height; row++)
{
//swap current into previous and reuse the previous as the next current
//for the first iteration of this loop the current row is all 1's
var tempReader = state.ReferenceReader;
state.ReferenceReader = state.CodingReader;
state.CodingReader = tempReader;
//fill the current row data from the stream
input.Seek(inputStreamStartOffset, SeekOrigin.Begin);
input.Seek(rowLengthInBytes * (height - 1 - row), SeekOrigin.Current);
input.Read(state.CodingReader.Buffer, 0, rowLengthInBytes);
EncodeRowGroup4(state);
}
//EOFB. Not really needed for files but it doesn't hurt and some encoders write it.
state.Writer.Write(0, 8);
state.Writer.Write(16, 8);
state.Writer.Write(1, 8);
state.Writer.Flush();
//return # of bytes written to output stream.
return((int)(output.Position - outputStartPosition));
}
/// <summary>
/// Sink up to size bytes from buffer + offset into the encoder.
/// inputSize is set to the number of bytes actually sunk (in case a buffer was filled)
/// </summary>
/// <param name="buffer">The buffer to be filled into the encoder</param>
/// <param name="offset">The offset of the buffer to be filled</param>
/// <param name="size">Number of bytes to be filled</param>
/// <param name="inputSize">Number of bytes actually sunk</param>
/// <returns>Refer to HSE_sink_res in original C version heatshrink</returns>
public EncoderSinkResult Sink(byte[] buffer, int offset, int size, out int inputSize)
{
inputSize = 0;
if (buffer == null)
{
return(EncoderSinkResult.Null);
}
/* Sinking more content after saying the content is done, tsk tsk */
if (IsFinishing())
{
return(EncoderSinkResult.Misuse);
}
/* Sinking more content before processing is done */
if (_state != EncoderState.NotFull)
{
return(EncoderSinkResult.Misuse);
}
var writeOffset = GetInputOffset() + _inputSize;
var ibs = GetInputBufferSize();
var rem = ibs - _inputSize;
var cpSz = rem < size ? rem : size;
Array.Copy(buffer, offset, _buffer, writeOffset, cpSz);
inputSize = cpSz;
_inputSize += cpSz;
if (Constants.EnableLogging)
{
Console.WriteLine($"-- sunk {cpSz} bytes (of {size}) into encoder at {writeOffset}, input buffer now has {_inputSize}");
}
if (cpSz == rem)
{
if (Constants.EnableLogging)
{
Console.WriteLine("-- internal buffer is now full");
}
_state = EncoderState.Filled;
}
return(EncoderSinkResult.Ok);
}
private void EncodeRowGroup4(EncoderState state)
{
int a0 = -1; //start just before the first element
bool a0Color = true;
while (a0 < state.Width)
{
if (a0 > -1)
{
a0Color = state.CodingReader.GetBit(a0);
}
int a1 = state.CodingReader.GetNextMatchingBit(a0, !a0Color);
int b1 = state.ReferenceReader.GetNextMatchingBit(a0, !a0Color);
int b2 = state.ReferenceReader.GetNextMatchingBit(b1, a0Color);
if (a0 == -1)
{
a0 = 0; // for the rest of the math to work.
}
if (b2 < a1) //pass mode
{
WritePassCode(state);
a0 = b2;
}
else //horizontal or vertical
{
int deltab1a1 = b1 - a1;
if (-3 <= deltab1a1 && deltab1a1 <= 3) //vertical
{
WriteVerticalCode(state, deltab1a1);
a0 = a1;
}
else //horizontal
{
int a2 = state.CodingReader.GetNextMatchingBit(a1, a0Color);
WriteHorizontalCodes(state, a0Color, a1 - a0, a2 - a1);
a0 = a2;
}
}
}
}
public void FinishTaskRequestCreateTest()
{
//Arrange
const string resource = "resource";
const string errorMessage = "errorMessage";
const string fileHash = "fileId";
const EncoderState encoderState = EncoderState.Completed;
//Act
var request = _helper.FinishTaskRequestCreate(resource, encoderState, fileHash, errorMessage);
//Assert
Assert.AreEqual(Method.POST, request.Method);
Assert.AreEqual(resource, request.Resource);
Assert.IsTrue(request.Parameters.Any(p => p.Type == ParameterType.Cookie && p.Name == _settings.CookieName && (string)p.Value == _settings.CookieValue));
Assert.IsTrue(request.Parameters.Any(p => p.Name == EncoderStatusParameters.Result && (EncoderState)p.Value == encoderState));
Assert.IsTrue(request.Parameters.Any(p => p.Name == EncoderStatusParameters.Message && (string)p.Value == errorMessage));
Assert.IsTrue(request.Parameters.Any(p => p.Name == EncoderStatusParameters.FileHash && (string)p.Value == fileHash));
Assert.IsTrue(request.Parameters.Any(p => p.Name == "X-HTTP-Method-Override" && (string)p.Value == "DELETE"));
}
public void PickCorrectElement()
{
var a = new A();
var b = new B();
var c = new D();
var codec = new PiecewiseCodec <object>();
codec = codec.Add(new CodecElement <A, LNode>(
"A",
(obj, state) => LNode.Call((Symbol)"A"),
(node, state) => a));
codec = codec.Add(new CodecElement <B, LNode>(
"B",
(obj, state) => LNode.Call((Symbol)"B"),
(node, state) => b));
codec = codec.Add(new CodecElement <C, LNode>(
"C",
(obj, state) => LNode.Call((Symbol)"C"),
(node, state) => c));
var encoder = new EncoderState();
var decoder = new DecoderState(log, new TypeResolver().ReadOnlyView);
ConstantCodecTest.AssertRoundTrip <object, LNode>(
a,
obj => codec.Encode(obj, encoder),
enc => codec.Decode(enc, decoder));
ConstantCodecTest.AssertRoundTrip <object, LNode>(
b,
obj => codec.Encode(obj, encoder),
enc => codec.Decode(enc, decoder));
ConstantCodecTest.AssertRoundTrip <object, LNode>(
c,
obj => codec.Encode(obj, encoder),
enc => codec.Decode(enc, decoder));
}
public void SetDataWhithStepDataTest()
{
//Arrange
const EncoderState encoderState = EncoderState.Cancelled;
const string message = "message";
var mediator = new Mock <IStepMediator>();
var webClient = new Mock <IEncodeWebClient>();
var pipelineStep = new PipelineStepStub(mediator.Object, webClient.Object);
var stepData = new StepData()
{
EncoderState = encoderState,
ErrorMessage = message
};
//Act
pipelineStep.SetData(stepData);
//Assert
Assert.AreEqual(encoderState, pipelineStep.ProtectedStepData.EncoderState);
Assert.AreEqual(message, pipelineStep.ProtectedStepData.ErrorMessage);
}
private void WriteHorizontalSpan(EncoderState state, bool color, int spanLength)
{
uint[] terminatingCodes = color ? WhiteTerminatingCodes : BlackTerminatingCodes;
uint[] makeUpCodes = color ? WhiteMakeUpCodes : BlackMakeUpCodes;
int count = spanLength;
// The make-up code for 2560 will be written as often as required:
while (count >= 2624)
{
state.Writer.Write(makeUpCodes[39 * 2], makeUpCodes[39 * 2 + 1]); // Magic: 2560
count -= 2560;
}
// A make-up code for a multiple of 64 will be written if required:
if (count > 63)
{
int line = count / 64 - 1;
state.Writer.Write(makeUpCodes[line * 2], makeUpCodes[line * 2 + 1]);
count -= (line + 1) * 64;
}
// And finally the terminating code for the remaining value (0 through 63):
state.Writer.Write(terminatingCodes[count * 2], terminatingCodes[count * 2 + 1]);
}
private void WriteVerticalCode(EncoderState state, int deltab1a1)
{
var verticalIndex = deltab1a1 + 3;
state.Writer.Write(VerticalCodes[verticalIndex * 2], VerticalCodes[verticalIndex * 2 + 1]);
}
public IRestRequest FinishTaskRequestCreate(string resource, EncoderState encoderState, string fileId, string errorMessage)
{
IRestRequest request = RequestCreate(resource, Method.POST);
request.AddHeader("X-HTTP-Method-Override", "DELETE");
request.AddParameter(EncoderStatusParameters.Result, encoderState);
request.AddParameter(EncoderStatusParameters.FileId, fileId);
request.AddParameter(EncoderStatusParameters.Message, errorMessage);
return request;
}
/// <summary>
/// Writes a CIL method body, analyzes it as Flame IR,
/// emits that as CIL and checks that the outcome matches
/// what we'd expect.
/// </summary>
/// <param name="returnType">
/// The return type of the method body.
/// </param>
/// <param name="parameterTypes">
/// The parameter types of the method body.
/// </param>
/// <param name="localTypes">
/// The local variable types of the method body.
/// </param>
/// <param name="emitBody">
/// A function that writes the method body.
/// </param>
/// <param name="oracle">
/// A printed version of the expected method body.
/// </param>
private void RoundtripStaticMethodBody(
TypeReference returnType,
IReadOnlyList <TypeReference> parameterTypes,
IReadOnlyList <TypeReference> localTypes,
Action <Mono.Cecil.Cil.ILProcessor> emitBody,
string oracle)
{
// Define a method.
var methodDef = CreateStaticMethodDef(returnType, parameterTypes);
// Emit the source CIL.
var cilBody = new Mono.Cecil.Cil.MethodBody(methodDef);
foreach (var localType in localTypes)
{
cilBody.Variables.Add(new Mono.Cecil.Cil.VariableDefinition(localType));
}
emitBody(cilBody.GetILProcessor());
cilBody.Optimize();
// Analyze it as Flame IR.
var irBody = ClrMethodBodyAnalyzer.Analyze(
cilBody,
new Parameter(TypeHelpers.BoxIfReferenceType(corlib.Resolve(returnType))),
default(Parameter),
parameterTypes
.Select((type, i) => new Parameter(TypeHelpers.BoxIfReferenceType(corlib.Resolve(type)), "param_" + i))
.ToArray(),
corlib);
// Register analyses.
irBody = new global::Flame.Compiler.MethodBody(
irBody.ReturnParameter,
irBody.ThisParameter,
irBody.Parameters,
irBody.Implementation
.WithAnalysis(LazyBlockReachabilityAnalysis.Instance)
.WithAnalysis(NullabilityAnalysis.Instance)
.WithAnalysis(new EffectfulInstructionAnalysis())
.WithAnalysis(PredecessorAnalysis.Instance)
.WithAnalysis(RelatedValueAnalysis.Instance)
.WithAnalysis(LivenessAnalysis.Instance)
.WithAnalysis(InterferenceGraphAnalysis.Instance)
.WithAnalysis(ValueUseAnalysis.Instance)
.WithAnalysis(ConservativeInstructionOrderingAnalysis.Instance));
// Optimize the IR a tiny bit.
irBody = irBody.WithImplementation(
irBody.Implementation.Transform(
AllocaToRegister.Instance,
CopyPropagation.Instance,
new ConstantPropagation(),
SwitchSimplification.Instance,
DeadValueElimination.Instance,
InstructionSimplification.Instance,
new JumpThreading(true),
DeadBlockElimination.Instance,
new SwitchLowering(corlib.Resolver.TypeEnvironment),
CopyPropagation.Instance,
InstructionSimplification.Instance,
DeadValueElimination.Instance,
InstructionReordering.Instance,
new JumpThreading(false)));
// Turn Flame IR back into CIL.
var newCilBody = ClrMethodBodyEmitter.Compile(irBody, methodDef, corlib.Resolver.TypeEnvironment);
// Check that the resulting CIL matches the expected CIL.
var actual = FormatMethodBody(newCilBody);
actual = actual.Trim().Replace("\r", "");
oracle = oracle.Trim().Replace("\r", "");
if (actual != oracle)
{
var encoder = new EncoderState();
var encodedImpl = encoder.Encode(irBody.Implementation);
var actualIr = Les2LanguageService.Value.Print(
encodedImpl,
options: new LNodePrinterOptions
{
IndentString = new string(' ', 4)
});
log.Log(
new LogEntry(
Severity.Message,
"emitted CIL-oracle mismatch",
"round-tripped CIL does not match the oracle. CIL emit output:",
new Paragraph(new WrapBox(actual, 0, -actual.Length)),
DecorationSpan.MakeBold("remark: Flame IR:"),
new Paragraph(new WrapBox(actualIr, 0, -actualIr.Length))));
}
Assert.AreEqual(oracle, actual);
}
public void FinishTask(EncoderState encoderState, string fileId, string errorMessage)
{
IRestRequest request = _restHelper.FinishTaskRequestCreate(Resource, encoderState, fileId, errorMessage);
_restHelper.GetResponse(request);
}
/// <summary>
/// Writes a CIL method body, analyzes it as Flame IR
/// and checks that the result is what we'd expect.
/// </summary>
/// <param name="returnType">
/// The return type of the method body.
/// </param>
/// <param name="parameterTypes">
/// The parameter types of the method body.
/// </param>
/// <param name="emitBody">
/// A function that writes the method body.
/// </param>
/// <param name="oracle">
/// The expected Flame IR flow graph, as LESv2.
/// </param>
private void AnalyzeStaticMethodBody(
TypeReference returnType,
IReadOnlyList <TypeReference> parameterTypes,
IReadOnlyList <TypeReference> localTypes,
Action <Mono.Cecil.Cil.ILProcessor> emitBody,
string oracle)
{
var methodDef = new MethodDefinition(
"f",
MethodAttributes.Public | MethodAttributes.Static,
returnType);
foreach (var type in parameterTypes)
{
methodDef.Parameters.Add(new ParameterDefinition(type));
int index = methodDef.Parameters.Count - 1;
methodDef.Parameters[index].Name = "param_" + index;
}
var cilBody = new Mono.Cecil.Cil.MethodBody(methodDef);
foreach (var localType in localTypes)
{
cilBody.Variables.Add(new Mono.Cecil.Cil.VariableDefinition(localType));
}
emitBody(cilBody.GetILProcessor());
var irBody = ClrMethodBodyAnalyzer.Analyze(
cilBody,
new Parameter(TypeHelpers.BoxIfReferenceType(corlib.Resolve(returnType))),
default(Parameter),
parameterTypes
.Select((type, i) => new Parameter(TypeHelpers.BoxIfReferenceType(corlib.Resolve(type)), "param_" + i))
.ToArray(),
corlib);
var encoder = new EncoderState();
var encodedImpl = encoder.Encode(irBody.Implementation);
var actual = Les2LanguageService.Value.Print(
encodedImpl,
options: new LNodePrinterOptions
{
IndentString = new string(' ', 4)
});
if (actual.Trim() != oracle.Trim())
{
log.Log(
new LogEntry(
Severity.Message,
"CIL analysis-oracle mismatch",
"analyzed CIL does not match the oracle. CIL analysis output:"));
// TODO: ugly hack to work around wrapping.
Console.Error.WriteLine(actual.Trim());
}
Assert.AreEqual(
actual.Trim(),
oracle.Trim());
}
/// <summary>
/// Poll for output from the encoder, copying at most size bytes into buffer + offset
/// (setting outputSize to the actual amount copied)
/// </summary>
/// <param name="buffer">The buffer to be filled from the encoder</param>
/// <param name="offset">The offset of the buffer to be filled</param>
/// <param name="size">Number of bytes to be filled</param>
/// <param name="outputSize">Number of bytes actually polled</param>
/// <returns>Refer to HSE_poll_res in original C version heatshrink</returns>
public EncoderPollResult Poll(byte[] buffer, int offset, int size, out int outputSize)
{
outputSize = 0;
if (buffer == null)
{
return(EncoderPollResult.Null);
}
if (size == 0)
{
if (Constants.EnableLogging)
{
Console.WriteLine("-- MISUSE: output buffer size is 0");
}
return(EncoderPollResult.Misuse);
}
while (true)
{
if (Constants.EnableLogging)
{
Console.WriteLine($"-- polling, state {Convert.ToInt32(_state)} ({StateNames[_state]}), flags 0x{_flags:x2}");
}
var inState = _state;
switch (inState)
{
case EncoderState.NotFull:
return(EncoderPollResult.Empty);
case EncoderState.Filled:
DoIndexing();
_state = EncoderState.Search;
break;
case EncoderState.Search:
_state = StStepSearch();
break;
case EncoderState.YieldTagBit:
_state = StYieldTagBit(buffer, offset, size, ref outputSize);
break;
case EncoderState.YieldLiteral:
_state = StYieldLiteral(buffer, offset, size, ref outputSize);
break;
case EncoderState.YieldBackrefIndex:
_state = StYieldBackrefIndex(buffer, offset, size, ref outputSize);
break;
case EncoderState.YieldBackrefLength:
_state = StYieldBackrefLength(buffer, offset, size, ref outputSize);
break;
case EncoderState.SaveBacklog:
_state = StSaveBacklog();
break;
case EncoderState.FlushBits:
_state = StFlushBitBuffer(buffer, offset, size, ref outputSize);
return(EncoderPollResult.Empty);
case EncoderState.Done:
return(EncoderPollResult.Empty);
default:
if (Constants.EnableLogging)
{
Console.WriteLine($"-- bad state {StateNames[_state]}");
}
return(EncoderPollResult.Misuse);
}
/* Check if output buffer is exhausted. */
if (_state == inState && outputSize == size)
{
return(EncoderPollResult.More);
}
}
}
private void WritePassCode(EncoderState state)
{
state.Writer.Write(PassCodes[0], PassCodes[1]);
}
