/// <summary>
/// Performs a single environment update of the Academy and Agent
/// objects within the environment.
/// </summary>
public void EnvironmentStep()
{
if (!m_HadFirstReset)
{
ForcedFullReset();
}
AgentPreStep?.Invoke(m_StepCount);
m_StepCount += 1;
m_TotalStepCount += 1;
AgentIncrementStep?.Invoke();
using (TimerStack.Instance.Scoped("AgentSendState"))
{
AgentSendState?.Invoke();
}
using (TimerStack.Instance.Scoped("DecideAction"))
{
DecideAction?.Invoke();
}
// If the communicator is not on, we need to clear the SideChannel sending queue
if (!IsCommunicatorOn)
{
SideChannelManager.GetSideChannelMessage();
}
using (TimerStack.Instance.Scoped("AgentAct"))
{
AgentAct?.Invoke();
}
}
public void TestRawBytesSideChannel()
{
var str1 = "Test string";
var str2 = "Test string, second";
var strSender = new RawBytesChannel(new Guid("9a5b8954-4f82-11ea-b238-784f4387d1f7"));
var strReceiver = new RawBytesChannel(new Guid("9a5b8954-4f82-11ea-b238-784f4387d1f7"));
var dictSender = new Dictionary <Guid, SideChannel> {
{ strSender.ChannelId, strSender }
};
var dictReceiver = new Dictionary <Guid, SideChannel> {
{ strReceiver.ChannelId, strReceiver }
};
strSender.SendRawBytes(Encoding.ASCII.GetBytes(str1));
strSender.SendRawBytes(Encoding.ASCII.GetBytes(str2));
byte[] fakeData = SideChannelManager.GetSideChannelMessage(dictSender);
SideChannelManager.ProcessSideChannelData(dictReceiver, fakeData);
var messages = strReceiver.GetAndClearReceivedMessages();
Assert.AreEqual(messages.Count, 2);
Assert.AreEqual(Encoding.ASCII.GetString(messages[0]), str1);
Assert.AreEqual(Encoding.ASCII.GetString(messages[1]), str2);
}
/// <summary>
/// Shut down the Academy.
/// </summary>
public void Dispose()
{
DisableAutomaticStepping();
// Signal to listeners that the academy is being destroyed now
DestroyAction?.Invoke();
Communicator?.Dispose();
Communicator = null;
m_EnvironmentParameters.Dispose();
m_StatsRecorder.Dispose();
SideChannelManager.UnregisterAllSideChannels(); // unregister custom side channels
if (m_ModelRunners != null)
{
foreach (var mr in m_ModelRunners)
{
mr.Dispose();
}
m_ModelRunners = null;
}
// Clear out the actions so we're not keeping references to any old objects
ResetActions();
// TODO - Pass worker ID or some other identifier,
// so that multiple envs won't overwrite each others stats.
TimerStack.Instance.SaveJsonTimers();
m_Initialized = false;
// Reset the Lazy instance
s_Lazy = new Lazy <Academy>(() => new Academy());
}
/// <summary>
/// Performs a single environment update of the Academy and Agent
/// objects within the environment.
/// </summary>
public void EnvironmentStep()
{
// Check whether we're already in the middle of a step.
// This shouldn't happen generally, but could happen if user code (e.g. CollectObservations)
// that is called by EnvironmentStep() also calls EnvironmentStep(). This would result
// in an infinite loop and/or stack overflow, so stop it before it happens.
if (m_IsStepping)
{
throw new UnityAgentsException(
"Academy.EnvironmentStep() called recursively. " +
"This might happen if you call EnvironmentStep() from custom code such as " +
"CollectObservations() or OnActionReceived()."
);
}
m_IsStepping = true;
try
{
if (!m_HadFirstReset)
{
ForcedFullReset();
}
AgentPreStep?.Invoke(m_StepCount);
m_StepCount += 1;
m_TotalStepCount += 1;
AgentIncrementStep?.Invoke();
using (TimerStack.Instance.Scoped("AgentSendState"))
{
AgentSendState?.Invoke();
}
using (TimerStack.Instance.Scoped("DecideAction"))
{
DecideAction?.Invoke();
}
// If the communicator is not on, we need to clear the SideChannel sending queue
if (!IsCommunicatorOn)
{
SideChannelManager.GetSideChannelMessage();
}
using (TimerStack.Instance.Scoped("AgentAct"))
{
AgentAct?.Invoke();
}
}
finally
{
// Reset m_IsStepping when we're done (or if an exception occurred).
m_IsStepping = false;
}
}
public void OnDestroy()
{
// De-register the Debug.Log callback
Application.logMessageReceived -= stringChannel.SendDebugStatementToPython;
if (Academy.IsInitialized)
{
SideChannelManager.UnregisterSideChannel(stringChannel);
}
}
public SamplerTests()
{
m_Channel = SideChannelManager.GetSideChannel <EnvironmentParametersChannel>();
// if running test on its own
if (m_Channel == null)
{
m_Channel = new EnvironmentParametersChannel();
SideChannelManager.RegisterSideChannel(m_Channel);
}
}
public void Awake()
{
// We create the Side Channel
stringChannel = new StringLogSideChannel();
// When a Debug.Log message is created, we send it to the stringChannel
Application.logMessageReceived += stringChannel.SendDebugStatementToPython;
// The channel must be registered with the SideChannelManager class
SideChannelManager.RegisterSideChannel(stringChannel);
}
public void TestFloatPropertiesSideChannel()
{
var k1 = "gravity";
var k2 = "length";
int wasCalled = 0;
var propA = new FloatPropertiesChannel();
var propB = new FloatPropertiesChannel();
var dictReceiver = new Dictionary <Guid, SideChannel> {
{ propA.ChannelId, propA }
};
var dictSender = new Dictionary <Guid, SideChannel> {
{ propB.ChannelId, propB }
};
propA.RegisterCallback(k1, f => { wasCalled++; });
var tmp = propB.GetWithDefault(k2, 3.0f);
Assert.AreEqual(tmp, 3.0f);
propB.Set(k2, 1.0f);
tmp = propB.GetWithDefault(k2, 3.0f);
Assert.AreEqual(tmp, 1.0f);
byte[] fakeData = SideChannelManager.GetSideChannelMessage(dictSender);
SideChannelManager.ProcessSideChannelData(dictReceiver, fakeData);
tmp = propA.GetWithDefault(k2, 3.0f);
Assert.AreEqual(tmp, 1.0f);
Assert.AreEqual(wasCalled, 0);
propB.Set(k1, 1.0f);
Assert.AreEqual(wasCalled, 0);
fakeData = SideChannelManager.GetSideChannelMessage(dictSender);
SideChannelManager.ProcessSideChannelData(dictReceiver, fakeData);
Assert.AreEqual(wasCalled, 1);
var keysA = propA.Keys();
Assert.AreEqual(2, keysA.Count);
Assert.IsTrue(keysA.Contains(k1));
Assert.IsTrue(keysA.Contains(k2));
var keysB = propA.Keys();
Assert.AreEqual(2, keysB.Count);
Assert.IsTrue(keysB.Contains(k1));
Assert.IsTrue(keysB.Contains(k2));
}
public void TestAcademyDispose()
{
var envParams1 = SideChannelManager.GetSideChannel <EnvironmentParametersChannel>();
var engineParams1 = SideChannelManager.GetSideChannel <EngineConfigurationChannel>();
var statsParams1 = SideChannelManager.GetSideChannel <StatsSideChannel>();
Academy.Instance.Dispose();
Academy.Instance.LazyInitialize();
var envParams2 = SideChannelManager.GetSideChannel <EnvironmentParametersChannel>();
var engineParams2 = SideChannelManager.GetSideChannel <EngineConfigurationChannel>();
var statsParams2 = SideChannelManager.GetSideChannel <StatsSideChannel>();
Academy.Instance.Dispose();
Assert.AreNotEqual(envParams1, envParams2);
Assert.AreNotEqual(engineParams1, engineParams2);
Assert.AreNotEqual(statsParams1, statsParams2);
}
public void GaussianSamplerTest()
{
float mean = 3.0f;
float stddev = 0.2f;
string parameter = "parameter2";
using (var outgoingMsg = new OutgoingMessage())
{
outgoingMsg.WriteString(parameter);
// 1 indicates this meessage is a Sampler
outgoingMsg.WriteInt32(1);
outgoingMsg.WriteInt32(k_Seed);
outgoingMsg.WriteInt32((int)SamplerType.Gaussian);
outgoingMsg.WriteFloat32(mean);
outgoingMsg.WriteFloat32(stddev);
byte[] message = GetByteMessage(m_Channel, outgoingMsg);
SideChannelManager.ProcessSideChannelData(message);
}
Assert.AreEqual(2.936162f, m_Channel.GetWithDefault(parameter, 1.0f), k_Epsilon);
Assert.AreEqual(2.951348f, m_Channel.GetWithDefault(parameter, 1.0f), k_Epsilon);
}
public void UniformSamplerTest()
{
float min_value = 1.0f;
float max_value = 2.0f;
string parameter = "parameter1";
using (var outgoingMsg = new OutgoingMessage())
{
outgoingMsg.WriteString(parameter);
// 1 indicates this meessage is a Sampler
outgoingMsg.WriteInt32(1);
outgoingMsg.WriteInt32(k_Seed);
outgoingMsg.WriteInt32((int)SamplerType.Uniform);
outgoingMsg.WriteFloat32(min_value);
outgoingMsg.WriteFloat32(max_value);
byte[] message = GetByteMessage(m_Channel, outgoingMsg);
SideChannelManager.ProcessSideChannelData(message);
}
Assert.AreEqual(1.208888f, m_Channel.GetWithDefault(parameter, 1.0f), k_Epsilon);
Assert.AreEqual(1.118017f, m_Channel.GetWithDefault(parameter, 1.0f), k_Epsilon);
}
public void TestIntegerSideChannel()
{
var intSender = new TestSideChannel();
var intReceiver = new TestSideChannel();
var dictSender = new Dictionary <Guid, SideChannel> {
{ intSender.ChannelId, intSender }
};
var dictReceiver = new Dictionary <Guid, SideChannel> {
{ intReceiver.ChannelId, intReceiver }
};
intSender.SendInt(4);
intSender.SendInt(5);
intSender.SendInt(6);
byte[] fakeData = SideChannelManager.GetSideChannelMessage(dictSender);
SideChannelManager.ProcessSideChannelData(dictReceiver, fakeData);
Assert.AreEqual(intReceiver.messagesReceived[0], 4);
Assert.AreEqual(intReceiver.messagesReceived[1], 5);
Assert.AreEqual(intReceiver.messagesReceived[2], 6);
}
public void MultiRangeUniformSamplerTest()
{
float[] intervals = new float[4];
intervals[0] = 1.2f;
intervals[1] = 2f;
intervals[2] = 3.2f;
intervals[3] = 4.1f;
string parameter = "parameter3";
using (var outgoingMsg = new OutgoingMessage())
{
outgoingMsg.WriteString(parameter);
// 1 indicates this meessage is a Sampler
outgoingMsg.WriteInt32(1);
outgoingMsg.WriteInt32(k_Seed);
outgoingMsg.WriteInt32((int)SamplerType.MultiRangeUniform);
outgoingMsg.WriteFloatList(intervals);
byte[] message = GetByteMessage(m_Channel, outgoingMsg);
SideChannelManager.ProcessSideChannelData(message);
}
Assert.AreEqual(3.387999f, m_Channel.GetWithDefault(parameter, 1.0f), k_Epsilon);
Assert.AreEqual(1.294413f, m_Channel.GetWithDefault(parameter, 1.0f), k_Epsilon);
}
public void TestAcademy()
{
Assert.AreEqual(false, Academy.IsInitialized);
var aca = Academy.Instance;
Assert.AreEqual(true, Academy.IsInitialized);
// Check that init is idempotent
aca.LazyInitialize();
aca.LazyInitialize();
Assert.AreEqual(0, aca.EpisodeCount);
Assert.AreEqual(0, aca.StepCount);
Assert.AreEqual(0, aca.TotalStepCount);
Assert.AreNotEqual(null, SideChannelManager.GetSideChannel <EnvironmentParametersChannel>());
Assert.AreNotEqual(null, SideChannelManager.GetSideChannel <EngineConfigurationChannel>());
Assert.AreNotEqual(null, SideChannelManager.GetSideChannel <StatsSideChannel>());
// Check that Dispose is idempotent
aca.Dispose();
Assert.AreEqual(false, Academy.IsInitialized);
aca.Dispose();
}
void UpdateEnvironmentWithInput(UnityRLInputProto rlInput)
{
SideChannelManager.ProcessSideChannelData(rlInput.SideChannel.ToArray());
SendCommandEvent(rlInput.Command);
}
/// <summary>
/// Initializes the environment, configures it and initializes the Academy.
/// </summary>
void InitializeEnvironment()
{
TimerStack.Instance.AddMetadata("communication_protocol_version", k_ApiVersion);
TimerStack.Instance.AddMetadata("com.unity.ml-agents_version", k_PackageVersion);
EnableAutomaticStepping();
SideChannelManager.RegisterSideChannel(new EngineConfigurationChannel());
SideChannelManager.RegisterSideChannel(new TrainingAnalyticsSideChannel());
m_EnvironmentParameters = new EnvironmentParameters();
m_StatsRecorder = new StatsRecorder();
// Try to launch the communicator by using the arguments passed at launch
var port = ReadPortFromArgs();
if (port > 0)
{
Communicator = CommunicatorFactory.Create();
}
if (Communicator != null)
{
// We try to exchange the first message with Python. If this fails, it means
// no Python Process is ready to train the environment. In this case, the
// environment must use Inference.
bool initSuccessful = false;
var communicatorInitParams = new CommunicatorInitParameters
{
port = port,
unityCommunicationVersion = k_ApiVersion,
unityPackageVersion = k_PackageVersion,
name = "AcademySingleton",
CSharpCapabilities = new UnityRLCapabilities()
};
try
{
initSuccessful = Communicator.Initialize(
communicatorInitParams,
out var unityRlInitParameters
);
if (initSuccessful)
{
UnityEngine.Random.InitState(unityRlInitParameters.seed);
// We might have inference-only Agents, so set the seed for them too.
m_InferenceSeed = unityRlInitParameters.seed;
TrainerCapabilities = unityRlInitParameters.TrainerCapabilities;
TrainerCapabilities.WarnOnPythonMissingBaseRLCapabilities();
}
else
{
Debug.Log($"Couldn't connect to trainer on port {port} using API version {k_ApiVersion}. Will perform inference instead.");
Communicator = null;
}
}
catch (Exception ex)
{
Debug.Log($"Unexpected exception when trying to initialize communication: {ex}\nWill perform inference instead.");
Communicator = null;
}
}
if (Communicator != null)
{
Communicator.QuitCommandReceived += OnQuitCommandReceived;
Communicator.ResetCommandReceived += OnResetCommand;
}
// If a communicator is enabled/provided, then we assume we are in
// training mode. In the absence of a communicator, we assume we are
// in inference mode.
ResetActions();
}
/// <summary>
/// Helper method that sends the current UnityRLOutput, receives the next UnityInput and
/// Applies the appropriate AgentAction to the agents.
/// </summary>
void SendBatchedMessageHelper()
{
var message = new UnityOutputProto
{
RlOutput = m_CurrentUnityRlOutput,
};
var tempUnityRlInitializationOutput = GetTempUnityRlInitializationOutput();
if (tempUnityRlInitializationOutput != null)
{
message.RlInitializationOutput = tempUnityRlInitializationOutput;
}
byte[] messageAggregated = SideChannelManager.GetSideChannelMessage();
message.RlOutput.SideChannel = ByteString.CopyFrom(messageAggregated);
var input = Exchange(message);
UpdateSentActionSpec(tempUnityRlInitializationOutput);
foreach (var k in m_CurrentUnityRlOutput.AgentInfos.Keys)
{
m_CurrentUnityRlOutput.AgentInfos[k].Value.Clear();
}
var rlInput = input?.RlInput;
if (rlInput?.AgentActions == null)
{
return;
}
UpdateEnvironmentWithInput(rlInput);
foreach (var brainName in rlInput.AgentActions.Keys)
{
if (!m_OrderedAgentsRequestingDecisions[brainName].Any())
{
continue;
}
if (!rlInput.AgentActions[brainName].Value.Any())
{
continue;
}
var agentActions = rlInput.AgentActions[brainName].ToAgentActionList();
var numAgents = m_OrderedAgentsRequestingDecisions[brainName].Count;
for (var i = 0; i < numAgents; i++)
{
var agentAction = agentActions[i];
var agentId = m_OrderedAgentsRequestingDecisions[brainName][i];
if (m_LastActionsReceived[brainName].ContainsKey(agentId))
{
m_LastActionsReceived[brainName][agentId] = agentAction;
}
}
}
foreach (var brainName in m_OrderedAgentsRequestingDecisions.Keys)
{
m_OrderedAgentsRequestingDecisions[brainName].Clear();
}
}
/// <summary>
/// Constructor.
/// </summary>
internal StatsRecorder()
{
m_Channel = new StatsSideChannel();
SideChannelManager.RegisterSideChannel(m_Channel);
}
internal void Dispose()
{
SideChannelManager.UnregisterSideChannel(m_Channel);
}
/// <summary>
/// Constructor.
/// </summary>
internal EnvironmentParameters()
{
m_Channel = new EnvironmentParametersChannel();
SideChannelManager.RegisterSideChannel(m_Channel);
}