public override byte[] Encode()
{
var frames = EncodeFrames();
var result = new List <byte> {
EncodeTypeField()
};
result.AddRange(ByteUtilities.GetBytes(Version));
if (DestinationConnectionId > 0)
{
DcilScil = (byte)(DcilScil | 0x50);
}
if (SourceConnectionId > 0)
{
DcilScil = (byte)(DcilScil | 0x05);
}
result.Add(DcilScil);
if (DestinationConnectionId > 0)
{
result.Add(DestinationConnectionId);
}
if (SourceConnectionId > 0)
{
result.Add(SourceConnectionId);
}
byte[] tokenLength = new VariableInteger(0);
byte[] length = new VariableInteger(4 + (ulong)frames.Length);
result.AddRange(tokenLength);
result.AddRange(length);
result.AddRange(ByteUtilities.GetBytes(PacketNumber));
result.AddRange(frames);
return(result.ToArray());
}
private static Exception GetCause(
byte[] serializedCause, string originalTaskExceptionToString)
{
// TODO[JIRA REEF-1422]: Distinguish between Java Task Exception and missing Exception.
if (ByteUtilities.IsNullOrEmpty(serializedCause))
{
return(new TaskExceptionMissingException(
"Task failed without an Exception, presumably caused by an Exception failure. Please inspect the FailedTask message."));
}
try
{
return((Exception)ByteUtilities.DeserializeFromBinaryFormat(serializedCause));
}
catch (SerializationException se)
{
Exceptions.Caught(se, Level.Info,
"Exception from Task was not able to be deserialized, returning a NonSerializableTaskException.", Logger);
return(NonSerializableTaskException.UnableToDeserialize(originalTaskExceptionToString, se));
}
}
public override byte[] Encode()
{
byte[] frames = EncodeFrames();
List <byte> result = new List <byte>();
result.Add(Type);
result.AddRange(ByteUtilities.GetBytes(Version));
if (DestinationConnectionId > 0)
{
DCIL_SCIL = (byte)(DCIL_SCIL | 0x50);
}
if (SourceConnectionId > 0)
{
DCIL_SCIL = (byte)(DCIL_SCIL | 0x05);
}
result.Add(DCIL_SCIL);
if (DestinationConnectionId > 0)
{
result.Add(DestinationConnectionId);
}
if (SourceConnectionId > 0)
{
result.Add(SourceConnectionId);
}
byte[] tokenLength = new VariableInteger(0);
byte[] length = new VariableInteger(4 + (UInt64)frames.Length);
result.AddRange(tokenLength);
result.AddRange(length);
result.AddRange(ByteUtilities.GetBytes(PacketNumber));
result.AddRange(frames);
return(result.ToArray());
}
public override byte[] Encode()
{
List <byte> result = new List <byte>();
result.Add(ActualType);
result.AddRange(ErrorCode.ToByteArray());
if (ActualType == 0x1c)
{
result.AddRange(FrameType.ToByteArray());
}
if (string.IsNullOrWhiteSpace(ReasonPhrase) == false)
{
byte[] rpl = new VariableInteger((UInt64)ReasonPhrase.Length);
result.AddRange(rpl);
byte[] reasonPhrase = ByteUtilities.GetBytes(ReasonPhrase);
result.AddRange(reasonPhrase);
}
return(result.ToArray());
}
public Exception AsError()
{
if (Data.IsPresent())
{
Exception inner;
try
{
inner = (Exception)ByteUtilities.DeserializeFromBinaryFormat(Data.Value);
}
catch (SerializationException se)
{
inner = NonSerializableEvaluatorException.UnableToDeserialize(
"Exception from Evaluator was not able to be deserialized, returning a NonSerializableEvaluatorException.",
se);
}
return(new JobException(Id, Message, inner));
}
else
{
return(new JobException(Id, Message));
}
}
public static ExceptionInfo SerializeException(Exception ex)
{
ByteString serializedException;
try
{
serializedException = ByteString.CopyFrom(ByteUtilities.SerializeToBinaryFormat(ex));
}
catch (SerializationException se)
{
Log.Log(Level.Warning, "Unable to serialize exception", ex);
serializedException = ByteString.CopyFrom(ByteUtilities.SerializeToBinaryFormat(
NonSerializableJobException.UnableToSerialize(ex, se)));
}
return(new ExceptionInfo()
{
NoError = false,
Name = ex.GetType().ToString(),
Message = ex.Message,
Data = serializedException
});
}
private void Handle(Exception e)
{
lock (_heartBeatManager)
{
Logger.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "evaluator {0} failed with exception", _evaluatorId), e);
_state = State.FAILED;
string errorMessage = string.Format(
CultureInfo.InvariantCulture,
"failed with error [{0}] with message [{1}] and stack trace [{2}]",
e,
e.Message,
e.StackTrace);
EvaluatorStatusProto evaluatorStatusProto = new EvaluatorStatusProto()
{
evaluator_id = _evaluatorId,
error = ByteUtilities.StringToByteArrays(errorMessage),
state = _state
};
_heartBeatManager.OnNext(evaluatorStatusProto);
_contextManager.Dispose();
}
}
private void saveCodesetgctToolStripMenuItem_Click(object sender, EventArgs e)
{
if (sss.IgnoredMetadata)
{
MessageBox.Show("Warning: extra data was found after the GCT footer (probably code titles placed there by BrawlBox) and will be discarded if you continue to save.");
}
using (var dialog = new SaveFileDialog()) {
dialog.Filter = "Ocarina codes (*.gct)|*.gct";
dialog.OverwritePrompt = true;
if (dialog.ShowDialog() == DialogResult.OK)
{
using (var fs = new FileStream(dialog.FileName, FileMode.Create, FileAccess.Write)) {
foreach (byte[] b in new byte[][] {
sss.DataBefore, ByteUtilities.StringToByteArray(ToCode()), sss.DataAfter
})
{
fs.Write(b, 0, b.Length);
}
}
}
}
}
/// <summary>
/// Creates an Avro HTTP request for unit tests.
/// </summary>
/// <returns>AvroHttpRequest.</returns>
private AvroHttpRequest CreateAvroHttpRequest()
{
AvroHttpRequest r = new AvroHttpRequest();
r.Header = new List <HeaderEntry>();
HeaderEntry he1 = new HeaderEntry();
he1.key = "a";
he1.value = "xxx";
HeaderEntry he2 = new HeaderEntry();
he2.key = "b";
he2.value = "yyy";
r.Header.Add(he1);
r.Header.Add(he2);
r.HttpMethod = "POST";
r.InputStream = ByteUtilities.StringToByteArrays("test binary stream data");
r.PathInfo = "/reef/evaluators";
r.QueryString = "id=12&id=34&a=b";
r.RequestUrl = "http://localhost:8080/reef/evaluators?id=12&id=34&a=b";
return(r);
}
/// <summary>
/// Get runtime configuration
/// </summary>
private static IConfiguration GetRuntimeConfigurationForYarn(string[] args)
{
var token = new SecurityToken(
TrustedApplicationTokenIdentifier,
TrustedApplicationTokenIdentifier,
ByteUtilities.StringToByteArrays(args[0]),
Encoding.ASCII.GetBytes(args[1]));
var clientConfig = YARNClientConfiguration.ConfigurationModule
.Set(YARNClientConfiguration.SecurityTokenStr, JsonConvert.SerializeObject(token))
.Build();
var tcpPortConfig = TcpPortConfigurationModule.ConfigurationModule
.Set(TcpPortConfigurationModule.PortRangeStart, args.Length > 3 ? args[3] : DefaultPortRangeStart)
.Set(TcpPortConfigurationModule.PortRangeCount, args.Length > 4 ? args[4] : DefaultPortRangeCount)
.Build();
var c = TangFactory.GetTang().NewConfigurationBuilder()
.BindIntNamedParam <NumberOfContainers>(args[2])
.Build();
return(Configurations.Merge(clientConfig, tcpPortConfig, c));
}
public TaskStatusProto ToProto()
{
// This is locked because the Task continuation thread which sets the
// result is potentially different from the HeartBeat thread.
lock (_stateLock)
{
Check();
TaskStatusProto taskStatusProto = new TaskStatusProto()
{
context_id = ContextId,
task_id = TaskId,
state = GetProtoState()
};
if (_result.IsPresent())
{
taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value);
}
else if (_lastException.IsPresent())
{
byte[] error = ByteUtilities.StringToByteArrays(_lastException.Value.ToString());
taskStatusProto.result = ByteUtilities.CopyBytesFrom(error);
}
else if (_state == TaskState.Running)
{
foreach (TaskMessage message in GetMessages())
{
TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto()
{
source_id = message.MessageSourceId,
message = ByteUtilities.CopyBytesFrom(message.Message),
};
taskStatusProto.task_message.Add(taskMessageProto);
}
}
return(taskStatusProto);
}
}
public override byte[] Encode()
{
byte[] frames = EncodeFrames();
List <byte> result = new List <byte>();
result.Add(EncodeTypeField());
result.AddRange(ByteUtilities.GetBytes(Version));
result.Add(DCID);
if (DCID > 0)
{
result.Add(DestinationConnectionId);
}
result.Add(SCID);
if (SCID > 0)
{
result.Add(SourceConnectionId);
}
result.AddRange(frames);
return(result.ToArray());
}
public byte[] Call(byte[] memento)
{
_isRunning = true;
while (_isRunning)
{
try
{
Log.Log(Level.Info, "NoopTask.call(): Waiting for the message.");
lock (this)
{
Monitor.Wait(this);
}
}
catch (System.Threading.ThreadInterruptedException ex)
{
Log.Log(Level.Warning, "NoopTask.wait() interrupted.", ex);
}
}
Log.Log(Level.Info,
"NoopTask.call(): Exiting with message {0}",
ByteUtilities.ByteArraysToString(_message.OrElse(InitMessage).Message));
return(_message.OrElse(InitMessage).Message);
}
public void OnNext(IRunningTask runningTask)
{
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "TaskMessegingRunningTaskHandler: {0} is to send message {1}.", runningTask.Id, Message));
runningTask.Send(ByteUtilities.StringToByteArrays(Message));
}
/// <summary>
/// Convert bytes which contains Json string into AvroHttpRequest object
/// </summary>
/// <param name="serializedBytes">The serialized bytes.</param>
/// <returns>AvroHttpRequest.</returns>
public static AvroHttpRequest FromBytesWithJson(byte[] serializedBytes)
{
string s = ByteUtilities.ByteArraysToString(serializedBytes);
return(FromJson(s));
}
public override string ToString()
{
return("DriverMessage [value=" + ByteUtilities.ByteArrarysToString(_value.Value) + "]");
}
/// <summary>
/// helper function mostly used for logging
/// </summary>
/// <returns>seralized string</returns>
public override string ToString()
{
return(ByteUtilities.ByteArrarysToString(new ProcessedResultsCodec().Encode(this)));
}
public void OnNext(byte[] value)
{
Exceptions.Throw(new InvalidOperationException("No handler bound for client Close With Message event:" + ByteUtilities.ByteArraysToString(value)),
Logger.GetLogger(typeof(DefaultClientCloseWithMessageHandler)));
}
public static byte[] ToBytes(T obj)
{
return(ByteUtilities.StringToByteArrays(JsonConvert.SerializeObject(obj)));
}
public byte[] Encode(ProcessedResults results)
{
return(ByteUtilities.StringToByteArrays(results.Loss + "+" + string.Join("@", results.Means.Select(m => m.ToString()))));
}
public RijndaelOfbTransform(Rijndael rijndael, byte[] initializationVector, PaddingMode paddingMode)
: base(rijndael, paddingMode)
{
_LastVector = ByteUtilities.Clone(initializationVector);
}
public void OnHttpRequest(ReefHttpRequest requet, ReefHttpResponse response)
{
LOGGER.Log(Level.Info, "OnHttpRequest in DefaultHttpHandler is called.");
response.Status = HttpStatusCode.OK;
response.OutputStream = ByteUtilities.StringToByteArrays("Byte array returned from DefaultHttpHandler in CLR!!!");
}
public override void Process()
{
Connection.GameAccount.UiSettings = ByteUtilities.ByteArrayToString(UISettings);
DAOManager.accountDAO.SaveAccount(Connection.GameAccount);
}
/// <summary>
/// Handle a context status update
/// </summary>
/// <param name="contextStatusProto"></param>
/// <param name="notifyClientOnNewActiveContext"></param>
private void Handle(ContextStatusProto contextStatusProto, bool notifyClientOnNewActiveContext)
{
string contextId = contextStatusProto.context_id;
Optional <string> parentId = contextStatusProto.parent_id != null ?
Optional <string> .Of(contextStatusProto.parent_id) : Optional <string> .Empty();
if (ContextStatusProto.State.READY == contextStatusProto.context_state)
{
if (!_activeContextIds.Contains(contextId))
{
EvaluatorContext evaluatorContext = new EvaluatorContext(this, contextId, parentId);
AddEvaluatorContext(evaluatorContext);
if (notifyClientOnNewActiveContext)
{
LOGGER.Log(Level.Info, "TODO: REPLACE THIS " + evaluatorContext.ToString());
//TODO
//dispatcher.onNext(ActiveContext.class, context);
}
}
foreach (ContextStatusProto.ContextMessageProto contextMessageProto in contextStatusProto.context_message)
{
byte[] message = contextMessageProto.message;
string sourceId = contextMessageProto.source_id;
LOGGER.Log(Level.Info, "TODO: REPLACE THIS " + sourceId + message);
// this.dispatcher.onNext(ContextMessage.class,
//new ContextMessageImpl(theMessage, contextID, sourceID));
}
}
else
{
if (!_activeContextIds.Contains(contextId))
{
if (ContextStatusProto.State.FAIL == contextStatusProto.context_state)
{
AddEvaluatorContext(new EvaluatorContext(this, contextId, parentId));
}
else
{
var e = new InvalidOperationException("unknown context signaling state " + contextStatusProto.context_state);
Exceptions.Throw(e, LOGGER);
}
}
}
EvaluatorContext context = GetEvaluatorContext(contextId);
EvaluatorContext parentContext = context.ParentId.IsPresent() ?
GetEvaluatorContext(context.ParentId.Value) : null;
RemoveEvaluatorContext(context);
if (ContextStatusProto.State.FAIL == contextStatusProto.context_state)
{
// TODO
Exception reason = new InvalidOperationException(ByteUtilities.ByteArrarysToString(contextStatusProto.error));
Optional <IActiveContext> optionalParentContext = (null == parentContext) ?
Optional <IActiveContext> .Empty() : Optional <IActiveContext> .Of(parentContext);
LOGGER.Log(Level.Info, "TODO: REPLACE THIS " + reason.ToString() + optionalParentContext);
// TODO
//this.dispatcher.onNext(FailedContext.class,
//context.getFailedContext(optionalParentContext, reason));
}
else if (ContextStatusProto.State.DONE == contextStatusProto.context_state)
{
if (null != parentContext)
{
// TODO
//this.dispatcher.onNext(ClosedContext.class, context.getClosedContext(parentContext));
}
else
{
LOGGER.Log(Level.Info, "Root context closed. Evaluator closed will trigger final shutdown.");
}
}
else
{
var e = new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Unknown context state {0} for context {1}", contextStatusProto.context_state, contextId));
Exceptions.Throw(e, LOGGER);
}
}
public void Handle(IRemoteMessage <EvaluatorHeartbeatProto> evaluatorHearBeatProtoMessage)
{
lock (_evaluatorDescriptor)
{
EvaluatorHeartbeatProto heartbeatProto = evaluatorHearBeatProtoMessage.Message;
if (heartbeatProto.evaluator_status != null)
{
EvaluatorStatusProto status = heartbeatProto.evaluator_status;
if (status.error != null)
{
Handle(new EvaluatorException(Id, ByteUtilities.ByteArrarysToString(status.error)));
return;
}
else if (_state == STATE.SUBMITTED)
{
string evaluatorRId = evaluatorHearBeatProtoMessage.Identifier.ToString();
LOGGER.Log(Level.Info, "TODO: REPLACE THIS " + evaluatorRId);
// TODO
// _evaluatorControlHandler = _remoteManager.getHandler(evaluatorRID, EvaluatorRuntimeProtocol.EvaluatorControlProto.class);
_state = STATE.RUNNING;
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Evaluator {0} is running", _evaluatorId));
}
}
LOGGER.Log(Level.Info, "Evaluator heartbeat: " + heartbeatProto);
EvaluatorStatusProto evaluatorStatusProto = heartbeatProto.evaluator_status;
foreach (ContextStatusProto contextStatusProto in heartbeatProto.context_status)
{
Handle(contextStatusProto, heartbeatProto.task_status != null);
}
if (heartbeatProto.task_status != null)
{
Handle(heartbeatProto.task_status);
}
if (evaluatorStatusProto.state == State.FAILED)
{
_state = STATE.FAILED;
EvaluatorException e = evaluatorStatusProto.error != null ?
new EvaluatorException(_evaluatorId, ByteUtilities.ByteArrarysToString(evaluatorStatusProto.error)) :
new EvaluatorException(_evaluatorId, "unknown cause");
LOGGER.Log(Level.Warning, "Failed evaluator: " + Id + e.Message);
Handle(e);
}
else if (evaluatorStatusProto.state == State.DONE)
{
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Evaluator {0} done", Id));
_state = STATE.DONE;
// TODO
// dispatcher.onNext(CompletedEvaluator.class, new CompletedEvaluator() {
//@Override
//public String getId() {
// return EvaluatorManager.this.evaluatorId;
Dispose();
}
}
LOGGER.Log(Level.Info, "DONE with evaluator heartbeat");
}
private void Check()
{
if (_result.IsPresent() && _lastException.IsPresent())
{
LOGGER.Log(Level.Warning, "Both task result and exception are present, the expcetion will take over. Thrown away result:" + ByteUtilities.ByteArraysToString(_result.Value));
State = TaskState.Failed;
_result = Optional <byte[]> .Empty();
}
}
/// <summary>
/// helper function mostly used for logging
/// </summary>
/// <returns>the serialized string</returns>
public override string ToString()
{
return(ByteUtilities.ByteArraysToString(new CentroidsCodec().Encode(this)));
}
public void OnNext(IRemoteMessage <REEFMessage> value)
{
REEFMessage remoteEvent = value.Message;
IRemoteIdentifier id = value.Identifier;
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "receive a ReefMessage from {0} Driver at {1}.", remoteEvent, id));
if (remoteEvent.evaluatorControl != null)
{
if (remoteEvent.evaluatorControl.context_control != null)
{
string context_message = null;
string task_message = null;
if (remoteEvent.evaluatorControl.context_control.context_message != null)
{
context_message = remoteEvent.evaluatorControl.context_control.context_message.ToString();
}
if (remoteEvent.evaluatorControl.context_control.task_message != null)
{
task_message = ByteUtilities.ByteArrarysToString(remoteEvent.evaluatorControl.context_control.task_message);
}
if (!(string.IsNullOrEmpty(context_message) && string.IsNullOrEmpty(task_message)))
{
LOGGER.Log(Level.Info,
string.Format(CultureInfo.InvariantCulture, "Control protobuf with context message [{0}] and task message [{1}]", context_message, task_message));
}
else if (remoteEvent.evaluatorControl.context_control.remove_context != null)
{
LOGGER.Log(Level.Info,
string.Format(CultureInfo.InvariantCulture, "Control protobuf to remove context {0}", remoteEvent.evaluatorControl.context_control.remove_context.context_id));
}
else if (remoteEvent.evaluatorControl.context_control.add_context != null)
{
LOGGER.Log(Level.Info,
string.Format(CultureInfo.InvariantCulture, "Control protobuf to add a context on top of {0}", remoteEvent.evaluatorControl.context_control.add_context.parent_context_id));
}
else if (remoteEvent.evaluatorControl.context_control.start_task != null)
{
LOGGER.Log(Level.Info,
string.Format(CultureInfo.InvariantCulture, "Control protobuf to start an task in {0}", remoteEvent.evaluatorControl.context_control.start_task.context_id));
}
else if (remoteEvent.evaluatorControl.context_control.stop_task != null)
{
LOGGER.Log(Level.Info, "Control protobuf to stop task");
}
else if (remoteEvent.evaluatorControl.context_control.suspend_task != null)
{
LOGGER.Log(Level.Info, "Control protobuf to suspend task");
}
}
}
if (_count == 0)
{
_begin = DateTime.Now;
_origBegin = _begin;
}
var count = Interlocked.Increment(ref _count);
int printBatchSize = 100000;
if (count % printBatchSize == 0)
{
DateTime end = DateTime.Now;
var diff = (end - _begin).TotalMilliseconds;
double seconds = diff / 1000.0;
long eventsPerSecond = (long)(printBatchSize / seconds);
_begin = DateTime.Now;
}
var observer = _observer;
if (observer != null)
{
observer.OnNext(value);
}
}
public AlternateStageLoaderData(byte[] data)
{
AlternatesByStage = new Dictionary <string, AlternateStageEntry>();
for (int index = 0; index < data.Length; index += 8)
{
if (ByteUtilities.ByteArrayEquals(data, index, HEADER_BRAWL, 0, HEADER_BRAWL.Length) ||
ByteUtilities.ByteArrayEquals(data, index, HEADER_PM36, 0, HEADER_PM36.Length))
{
int countbyte = data[index + 19];
index += 24;
int endIndex = index + 8 * countbyte - 8;
while (index < endIndex)
{
char[] name = new char[4];
name[0] = (char)data[index];
name[1] = (char)data[index + 1];
name[2] = (char)data[index + 2];
name[3] = (char)data[index + 3];
if (name[0] == '\0')
{
throw new Exception("Invalid stage name in alternate stage loader data: " +
string.Join("", name.Select(c => ((int)c).ToString("X2"))));
}
Console.WriteLine(new string(name) + " " + data[index + 3].ToString("X2"));
int buttonActivatedCount = data[index + 4];
if (buttonActivatedCount > 26)
{
throw new Exception("There are more than 26 button activated alternate stages for stage " +
new string(name) + ". This is probably incorrect.");
}
int randomCount = data[index + 5];
if (randomCount > 26)
{
throw new Exception("There are more than 26 random alternate stages for stage " +
new string(name) + ". This is probably incorrect.");
}
index += 8;
List <AlternateStageEntry.Alternate> buttonActivated = new List <AlternateStageEntry.Alternate>();
for (int j = 0; j < buttonActivatedCount; j++)
{
int buttonMask = data[index] << (8 + data[index + 1]);
char letter = (char)('A' + data[index + 3]);
buttonActivated.Add(new AlternateStageEntry.Alternate
{
ButtonMask = (ushort)buttonMask,
Letter = letter
});
index += 8;
}
List <AlternateStageEntry.Alternate> random = new List <AlternateStageEntry.Alternate>();
for (int j = 0; j < randomCount; j++)
{
char letter = (char)('A' + j);
random.Add(new AlternateStageEntry.Alternate
{
Letter = letter,
ButtonMask = 0
});
}
AlternatesByStage.Add(new string(name), new AlternateStageEntry
{
Random = random,
ButtonActivated = buttonActivated
});
}
break;
}
}
}
public void OnNext(IContextMessage value)
{
LOGGER.Log(Level.Info, "Received ContextMessage: " + ByteUtilities.ByteArrarysToString(value.Message));
}