/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
var expectedObservations = m_Shape[0];
if (m_Observations.Count > expectedObservations)
{
// Too many observations, truncate
Debug.LogWarningFormat(
"More observations ({0}) made than vector observation size ({1}). The observations will be truncated.",
m_Observations.Count, expectedObservations
);
m_Observations.RemoveRange(expectedObservations, m_Observations.Count - expectedObservations);
}
else if (m_Observations.Count < expectedObservations)
{
// Not enough observations; pad with zeros.
Debug.LogWarningFormat(
"Fewer observations ({0}) made than vector observation size ({1}). The observations will be padded.",
m_Observations.Count, expectedObservations
);
for (int i = m_Observations.Count; i < expectedObservations; i++)
{
m_Observations.Add(0);
}
}
writer.AddList(m_Observations);
return(expectedObservations);
}
/// <summary>
/// Writes a Texture2D into a ObservationWriter.
/// </summary>
/// <param name="obsWriter">
/// Writer to fill with Texture data.
/// </param>
/// <param name="texture">
/// The texture to be put into the tensor.
/// </param>
/// <param name="grayScale">
/// If set to <c>true</c> the textures will be converted to grayscale before
/// being stored in the tensor.
/// </param>
/// <returns>The number of floats written</returns>
public static int WriteTexture(
this ObservationWriter obsWriter,
Texture2D texture,
bool grayScale)
{
var width = texture.width;
var height = texture.height;
var texturePixels = texture.GetPixels32();
// During training, we convert from Texture to PNG before sending to the trainer, which has the
// effect of flipping the image. We need another flip here at inference time to match this.
for (var h = height - 1; h >= 0; h--)
{
for (var w = 0; w < width; w++)
{
var currentPixel = texturePixels[(height - h - 1) * width + w];
if (grayScale)
{
obsWriter[h, w, 0] =
(currentPixel.r + currentPixel.g + currentPixel.b) / 3f / 255.0f;
}
else
{
// For Color32, the r, g and b values are between 0 and 255.
obsWriter[h, w, 0] = currentPixel.r / 255.0f;
obsWriter[h, w, 1] = currentPixel.g / 255.0f;
obsWriter[h, w, 2] = currentPixel.b / 255.0f;
}
}
}
return(height * width * (grayScale ? 1 : 3));
}
/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
for (int i = 0; i < m_ObsSize * m_MaxNumObs; i++)
{
writer[i] = m_ObservationBuffer[i];
}
return(m_ObsSize * m_MaxNumObs);
}
/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("RenderTextureSensor.Write"))
{
ObservationToTexture(m_RenderTexture, m_Texture);
var numWritten = writer.WriteTexture(m_Texture, m_Grayscale);
return(numWritten);
}
}
/// <summary>
/// Writes out the generated, uncompressed image to the provided <see cref="ObservationWriter"/>.
/// </summary>
/// <param name="writer">Where the observation is written to.</param>
/// <returns></returns>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor"))
{
ObservationToTexture(m_Camera, m_Texture, m_Width, m_Height);
var numWritten = writer.WriteTexture(m_Texture, m_Grayscale);
return(numWritten);
}
}
/// <summary>
/// Writes out the generated, uncompressed image to the provided <see cref="ObservationWriter"/>.
/// </summary>
/// <param name="writer">Where the observation is written to.</param>
/// <returns></returns>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor"))
{
var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
var numWritten = Utilities.TextureToTensorProxy(texture, writer, m_Grayscale);
DestroyTexture(texture);
return(numWritten);
}
}
/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("RenderTextureSensor.Write"))
{
var texture = ObservationToTexture(m_RenderTexture);
var numWritten = Utilities.TextureToTensorProxy(texture, writer, m_Grayscale);
DestroyTexture(texture);
return(numWritten);
}
}
/// <summary>
/// Generates the observations for the provided sensor, and returns true if they equal the
/// expected values. If they are unequal, errorMessage is also set.
/// This should not generally be used in production code. It is only intended for
/// simplifying unit tests.
/// </summary>
/// <param name="sensor"></param>
/// <param name="expected"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
public static bool CompareObservation(ISensor sensor, float[,,] expected, out string errorMessage)
{
var tensorShape = new TensorShape(0, expected.GetLength(0), expected.GetLength(1), expected.GetLength(2));
var numExpected = tensorShape.height * tensorShape.width * tensorShape.channels;
const float fill = -1337f;
var output = new float[numExpected];
for (var i = 0; i < numExpected; i++)
{
output[i] = fill;
}
if (numExpected > 0)
{
if (fill != output[0])
{
errorMessage = "Error setting output buffer.";
return(false);
}
}
ObservationWriter writer = new ObservationWriter();
writer.SetTarget(output, sensor.GetObservationShape(), 0);
// Make sure ObservationWriter didn't touch anything
if (numExpected > 0)
{
if (fill != output[0])
{
errorMessage = "ObservationWriter.SetTarget modified a buffer it shouldn't have.";
return(false);
}
}
sensor.Write(writer);
for (var h = 0; h < tensorShape.height; h++)
{
for (var w = 0; w < tensorShape.width; w++)
{
for (var c = 0; c < tensorShape.channels; c++)
{
if (expected[h, w, c] != output[tensorShape.Index(0, h, w, c)])
{
errorMessage = $"Expected and actual differed in position [{h}, {w}, {c}]. " +
$"Expected: {expected[h, w, c]} Actual: {output[tensorShape.Index(0, h, w, c)]} ";
return(false);
}
}
}
}
errorMessage = null;
return(true);
}
/// <summary>
/// Generates the observations for the provided sensor, and returns true if they equal the
/// expected values. If they are unequal, errorMessage is also set.
/// This should not generally be used in production code. It is only intended for
/// simplifying unit tests.
/// </summary>
/// <param name="sensor"></param>
/// <param name="expected"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
public static bool CompareObservation(ISensor sensor, float[] expected, out string errorMessage)
{
var numExpected = expected.Length;
const float fill = -1337f;
var output = new float[numExpected];
for (var i = 0; i < numExpected; i++)
{
output[i] = fill;
}
if (numExpected > 0)
{
if (fill != output[0])
{
errorMessage = "Error setting output buffer.";
return(false);
}
}
ObservationWriter writer = new ObservationWriter();
writer.SetTarget(output, sensor.GetObservationShape(), 0);
// Make sure ObservationWriter didn't touch anything
if (numExpected > 0)
{
if (fill != output[0])
{
errorMessage = "ObservationWriter.SetTarget modified a buffer it shouldn't have.";
return(false);
}
}
sensor.Write(writer);
for (var i = 0; i < output.Length; i++)
{
if (expected[i] != output[i])
{
errorMessage = $"Expected and actual differed in position {i}. Expected: {expected[i]} Actual: {output[i]} ";
return(false);
}
}
errorMessage = null;
return(true);
}
/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("GridSensor.Write"))
{
int index = 0;
for (var h = m_GridSize.z - 1; h >= 0; h--)
{
for (var w = 0; w < m_GridSize.x; w++)
{
for (var d = 0; d < m_CellObservationSize; d++)
{
writer[h, w, d] = m_PerceptionBuffer[index];
index++;
}
}
}
return(index);
}
}
/// <summary>
/// Computes the ray perception observations and saves them to the provided
/// <see cref="ObservationWriter"/>.
/// </summary>
/// <param name="writer">Where the ray perception observations are written to.</param>
/// <returns></returns>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("RayPerceptionSensor.Perceive"))
{
Array.Clear(m_Observations, 0, m_Observations.Length);
var numRays = m_RayPerceptionInput.Angles.Count;
var numDetectableTags = m_RayPerceptionInput.DetectableTags.Count;
// For each ray, write the information to the observation buffer
for (var rayIndex = 0; rayIndex < numRays; rayIndex++)
{
m_RayPerceptionOutput.RayOutputs[rayIndex].ToFloatArray(numDetectableTags, rayIndex, m_Observations);
}
// Finally, add the observations to the ObservationWriter
writer.AddList(m_Observations);
}
return(m_Observations.Length);
}
/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
// First, call the wrapped sensor's write method. Make sure to use our own writer, not the passed one.
var wrappedShape = m_WrappedSensor.GetObservationShape();
m_LocalWriter.SetTarget(m_StackedObservations[m_CurrentIndex], wrappedShape, 0);
m_WrappedSensor.Write(m_LocalWriter);
// Now write the saved observations (oldest first)
var numWritten = 0;
for (var i = 0; i < m_NumStackedObservations; i++)
{
var obsIndex = (m_CurrentIndex + 1 + i) % m_NumStackedObservations;
writer.AddList(m_StackedObservations[obsIndex], numWritten);
numWritten += m_UnstackedObservationSize;
}
return(numWritten);
}
/// <inheritdoc/>
public int Write(ObservationWriter writer)
{
// First, call the wrapped sensor's write method. Make sure to use our own writer, not the passed one.
m_LocalWriter.SetTarget(m_StackedObservations[m_CurrentIndex], m_WrappedSpec, 0);
m_WrappedSensor.Write(m_LocalWriter);
// Now write the saved observations (oldest first)
var numWritten = 0;
if (m_WrappedSpec.Rank == 1)
{
for (var i = 0; i < m_NumStackedObservations; i++)
{
var obsIndex = (m_CurrentIndex + 1 + i) % m_NumStackedObservations;
writer.AddList(m_StackedObservations[obsIndex], numWritten);
numWritten += m_UnstackedObservationSize;
}
}
else
{
for (var i = 0; i < m_NumStackedObservations; i++)
{
var obsIndex = (m_CurrentIndex + 1 + i) % m_NumStackedObservations;
for (var h = 0; h < m_WrappedSpec.Shape[0]; h++)
{
for (var w = 0; w < m_WrappedSpec.Shape[1]; w++)
{
for (var c = 0; c < m_WrappedSpec.Shape[2]; c++)
{
writer[h, w, i *m_WrappedSpec.Shape[2] + c] = m_StackedObservations[obsIndex][m_tensorShape.Index(0, h, w, c)];
}
}
}
}
numWritten = m_WrappedSpec.Shape[0] * m_WrappedSpec.Shape[1] * m_WrappedSpec.Shape[2] * m_NumStackedObservations;
}
return(numWritten);
}
internal static int WriteTextureRGB24(
this ObservationWriter obsWriter,
Texture2D texture,
bool grayScale
)
{
var width = texture.width;
var height = texture.height;
var rawBytes = texture.GetRawTextureData <byte>();
// During training, we convert from Texture to PNG before sending to the trainer, which has the
// effect of flipping the image. We need another flip here at inference time to match this.
for (var h = height - 1; h >= 0; h--)
{
for (var w = 0; w < width; w++)
{
var offset = (height - h - 1) * width + w;
var r = rawBytes[3 * offset];
var g = rawBytes[3 * offset + 1];
var b = rawBytes[3 * offset + 2];
if (grayScale)
{
obsWriter[h, w, 0] = (r + g + b) / 3f / 255.0f;
}
else
{
// For Color32, the r, g and b values are between 0 and 255.
obsWriter[h, w, 0] = r / 255.0f;
obsWriter[h, w, 1] = g / 255.0f;
obsWriter[h, w, 2] = b / 255.0f;
}
}
}
return(height * width * (grayScale ? 1 : 3));
}
/// <summary>
/// Computes the ray perception observations and saves them to the provided
/// <see cref="ObservationWriter"/>.
/// </summary>
/// <param name="writer">Where the ray perception observations are written to.</param>
/// <returns></returns>
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("RayPerceptionSensor.Perceive"))
{
Array.Clear(m_Observations, 0, m_Observations.Length);
var numRays = m_RayPerceptionInput.Angles.Count;
var numDetectableTags = m_RayPerceptionInput.DetectableTags.Count;
if (m_DebugDisplayInfo != null)
{
// Reset the age information, and resize the buffer if needed.
m_DebugDisplayInfo.Reset();
if (m_DebugDisplayInfo.rayInfos == null || m_DebugDisplayInfo.rayInfos.Length != numRays)
{
m_DebugDisplayInfo.rayInfos = new DebugDisplayInfo.RayInfo[numRays];
}
}
// For each ray, do the casting, and write the information to the observation buffer
for (var rayIndex = 0; rayIndex < numRays; rayIndex++)
{
DebugDisplayInfo.RayInfo debugRay;
var rayOutput = PerceiveSingleRay(m_RayPerceptionInput, rayIndex, out debugRay);
if (m_DebugDisplayInfo != null)
{
m_DebugDisplayInfo.rayInfos[rayIndex] = debugRay;
}
rayOutput.ToFloatArray(numDetectableTags, rayIndex, m_Observations);
}
// Finally, add the observations to the ObservationWriter
writer.AddRange(m_Observations);
}
return(m_Observations.Length);
}