public void TestStackingMapping()
{
// Test grayscale stacked mapping with CameraSensor
var cameraSensor = new CameraSensor(new Camera(), 64, 64,
true, "grayscaleCamera", SensorCompressionType.PNG);
var stackedCameraSensor = new StackingSensor(cameraSensor, 2);
Assert.AreEqual(stackedCameraSensor.GetCompressionSpec().CompressedChannelMapping, new[] { 0, 0, 0, 1, 1, 1 });
// Test RGB stacked mapping with RenderTextureSensor
var renderTextureSensor = new RenderTextureSensor(new RenderTexture(24, 16, 0),
false, "renderTexture", SensorCompressionType.PNG);
var stackedRenderTextureSensor = new StackingSensor(renderTextureSensor, 2);
Assert.AreEqual(stackedRenderTextureSensor.GetCompressionSpec().CompressedChannelMapping, new[] { 0, 1, 2, 3, 4, 5 });
// Test mapping with number of layers not being multiple of 3
var dummySensor = new Dummy3DSensor();
dummySensor.ObservationSpec = ObservationSpec.Visual(2, 2, 4);
dummySensor.Mapping = new[] { 0, 1, 2, 3 };
var stackedDummySensor = new StackingSensor(dummySensor, 2);
Assert.AreEqual(stackedDummySensor.GetCompressionSpec().CompressedChannelMapping, new[] { 0, 1, 2, 3, -1, -1, 4, 5, 6, 7, -1, -1 });
// Test mapping with dummy layers that should be dropped
var paddedDummySensor = new Dummy3DSensor();
paddedDummySensor.ObservationSpec = ObservationSpec.Visual(2, 2, 4);
paddedDummySensor.Mapping = new[] { 0, 1, 2, 3, -1, -1 };
var stackedPaddedDummySensor = new StackingSensor(paddedDummySensor, 2);
Assert.AreEqual(stackedPaddedDummySensor.GetCompressionSpec().CompressedChannelMapping, new[] { 0, 1, 2, 3, -1, -1, 4, 5, 6, 7, -1, -1 });
}
/// <summary>
/// Create a sensor for the board with the specified observation type.
/// </summary>
/// <param name="board"></param>
/// <param name="obsType"></param>
/// <param name="name"></param>
public Match3Sensor(AbstractBoard board, Match3ObservationType obsType, string name)
{
m_Board = board;
m_Name = name;
m_Rows = board.Rows;
m_Columns = board.Columns;
m_NumCellTypes = board.NumCellTypes;
m_NumSpecialTypes = board.NumSpecialTypes;
m_ObservationType = obsType;
m_ObservationSpec = obsType == Match3ObservationType.Vector
? ObservationSpec.Vector(m_Rows * m_Columns * (m_NumCellTypes + SpecialTypeSize))
: ObservationSpec.Visual(m_Rows, m_Columns, m_NumCellTypes + SpecialTypeSize);
// See comment in GetCompressedObservation()
var cellTypePaddedSize = 3 * ((m_NumCellTypes + 2) / 3);
m_SparseChannelMapping = new int[cellTypePaddedSize + SpecialTypeSize];
// If we have 4 cell types and 2 special types (3 special size), we'd have
// [0, 1, 2, 3, -1, -1, 4, 5, 6]
for (var i = 0; i < m_NumCellTypes; i++)
{
m_SparseChannelMapping[i] = i;
}
for (var i = m_NumCellTypes; i < cellTypePaddedSize; i++)
{
m_SparseChannelMapping[i] = -1;
}
for (var i = 0; i < SpecialTypeSize; i++)
{
m_SparseChannelMapping[cellTypePaddedSize + i] = i + m_NumCellTypes;
}
}
/// <summary>
/// Create a GridSensorBase with the specified configuration.
/// </summary>
/// <param name="name">The sensor name</param>
/// <param name="cellScale">The scale of each cell in the grid</param>
/// <param name="gridNum">Number of cells on each side of the grid</param>
/// <param name="detectableTags">Tags to be detected by the sensor</param>
/// <param name="compression">Compression type</param>
public GridSensorBase(
string name,
Vector3 cellScale,
Vector3Int gridNum,
string[] detectableTags,
SensorCompressionType compression
)
{
m_Name = name;
m_CellScale = cellScale;
m_GridSize = gridNum;
m_DetectableTags = detectableTags;
CompressionType = compression;
if (m_GridSize.y != 1)
{
throw new UnityAgentsException("GridSensor only supports 2D grids.");
}
m_NumCells = m_GridSize.x * m_GridSize.z;
m_CellObservationSize = GetCellObservationSize();
m_ObservationSpec = ObservationSpec.Visual(m_GridSize.x, m_GridSize.z, m_CellObservationSize);
m_PerceptionTexture = new Texture2D(m_GridSize.x, m_GridSize.z, TextureFormat.RGB24, false);
ResetPerceptionBuffer();
}
public Float2DSensor(float[,] floatData, string name)
{
this.floatData = floatData;
Height = floatData.GetLength(0);
Width = floatData.GetLength(1);
m_Name = name;
m_ObservationSpec = ObservationSpec.Visual(Height, Width, 1);
}
public GameStateSensor(Board board)
{
this.board = board;
this.name = "GameStateSensor";
// This might be 85 or 85*3
observationSpec = ObservationSpec.Vector(2);
}
public BoardSensor(Board board)
{
this.board = board;
this.name = "BoardSensor";
// This might be 85 or 85*3
observationSpec = ObservationSpec.Vector(85 * 3);
}
public Float2DSensor(int width, int height, string name)
{
Width = width;
Height = height;
m_Name = name;
m_ObservationSpec = ObservationSpec.Visual(height, width, 1);
floatData = new float[Height, Width];
}
public ReflectionSensorBase(ReflectionSensorInfo reflectionSensorInfo, int size)
{
m_Object = reflectionSensorInfo.Object;
m_FieldInfo = reflectionSensorInfo.FieldInfo;
m_PropertyInfo = reflectionSensorInfo.PropertyInfo;
m_ObservableAttribute = reflectionSensorInfo.ObservableAttribute;
m_SensorName = reflectionSensorInfo.SensorName;
m_ObservationSpec = ObservationSpec.Vector(size);
m_NumFloats = size;
}
public TestTextureSensor(
Texture2D texture, string name, SensorCompressionType compressionType)
{
m_Texture = texture;
var width = texture.width;
var height = texture.height;
m_Name = name;
m_ObservationSpec = ObservationSpec.Visual(height, width, 3);
m_CompressionType = compressionType;
}
/// <summary>Gets the observation shape</summary>
/// <returns>int[] of the observation shape</returns>
public ObservationSpec GetObservationSpec()
{
// Lazy update
var shape = m_ObservationSpec.Shape;
if (shape[0] != GridNumSideX || shape[1] != GridNumSideZ || shape[2] != ObservationPerCell)
{
m_ObservationSpec = ObservationSpec.Visual(GridNumSideX, GridNumSideZ, ObservationPerCell);
}
return(m_ObservationSpec);
}
/// <summary>
/// Create a sensor for the GridValueProvider with the specified observation type.
/// </summary>
/// <remarks>
/// Use Match3Sensor.CellTypeSensor() or Match3Sensor.SpecialTypeSensor() instead of calling
/// the constructor directly.
/// </remarks>
/// <param name="board">The abstract board. This is only used to get the size.</param>
/// <param name="gvp">The GridValueProvider, should be either board.GetCellType or board.GetSpecialType.</param>
/// <param name="oneHotSize">The number of possible values that the GridValueProvider can return.</param>
/// <param name="obsType">Whether to produce vector or visual observations</param>
/// <param name="name">Name of the sensor.</param>
public Match3Sensor(AbstractBoard board, GridValueProvider gvp, int oneHotSize, Match3ObservationType obsType, string name)
{
m_Name = name;
m_Rows = board.Rows;
m_Columns = board.Columns;
m_GridValues = gvp;
m_OneHotSize = oneHotSize;
m_ObservationType = obsType;
m_ObservationSpec = obsType == Match3ObservationType.Vector
? ObservationSpec.Vector(m_Rows * m_Columns * oneHotSize)
: ObservationSpec.Visual(m_Rows, m_Columns, oneHotSize);
}
public void TestStackingSensorBuiltInSensorType()
{
var dummySensor = new Dummy3DSensor();
dummySensor.ObservationSpec = ObservationSpec.Visual(2, 2, 4);
dummySensor.Mapping = new[] { 0, 1, 2, 3 };
var stackedDummySensor = new StackingSensor(dummySensor, 2);
Assert.AreEqual(stackedDummySensor.GetBuiltInSensorType(), BuiltInSensorType.Unknown);
var vectorSensor = new VectorSensor(4);
var stackedVectorSensor = new StackingSensor(vectorSensor, 4);
Assert.AreEqual(stackedVectorSensor.GetBuiltInSensorType(), BuiltInSensorType.VectorSensor);
}
/// <summary>
/// Create a sensor for the GridValueProvider with the specified observation type.
/// </summary>
/// <remarks>
/// Use Match3Sensor.CellTypeSensor() or Match3Sensor.SpecialTypeSensor() instead of calling
/// the constructor directly.
/// </remarks>
/// <param name="board">The abstract board.</param>
/// <param name="gvp">The GridValueProvider, should be either board.GetCellType or board.GetSpecialType.</param>
/// <param name="oneHotSize">The number of possible values that the GridValueProvider can return.</param>
/// <param name="obsType">Whether to produce vector or visual observations</param>
/// <param name="name">Name of the sensor.</param>
public Match3Sensor(AbstractBoard board, GridValueProvider gvp, int oneHotSize, Match3ObservationType obsType, string name)
{
var maxBoardSize = board.GetMaxBoardSize();
m_Name = name;
m_MaxBoardSize = maxBoardSize;
m_GridValues = gvp;
m_OneHotSize = oneHotSize;
m_Board = board;
m_ObservationType = obsType;
m_ObservationSpec = obsType == Match3ObservationType.Vector
? ObservationSpec.Vector(maxBoardSize.Rows * maxBoardSize.Columns * oneHotSize)
: ObservationSpec.Visual(maxBoardSize.Rows, maxBoardSize.Columns, oneHotSize);
}
public GridSensor(
string name,
Vector3 cellScale,
Vector3Int gridNum,
bool rotateWithAgent,
int[] channelDepths,
string[] detectableObjects,
LayerMask colliderMask,
GridDepthType depthType,
GameObject rootReference,
SensorCompressionType compression,
int maxColliderBufferSize,
int initialColliderBufferSize
)
{
m_Name = name;
m_CellScale = cellScale;
m_GridSize = gridNum;
m_RotateWithAgent = rotateWithAgent;
m_RootReference = rootReference;
m_MaxColliderBufferSize = maxColliderBufferSize;
m_InitialColliderBufferSize = initialColliderBufferSize;
m_ColliderMask = colliderMask;
m_GridDepthType = depthType;
m_ChannelDepths = channelDepths;
m_DetectableObjects = detectableObjects;
m_CompressionType = compression;
if (m_GridSize.y != 1)
{
throw new UnityAgentsException("GridSensor only supports 2D grids.");
}
if (m_GridDepthType == GridDepthType.Counting && m_DetectableObjects.Length != m_ChannelDepths.Length)
{
throw new UnityAgentsException("The channels of a CountingGridSensor is equal to the number of detectableObjects");
}
InitGridParameters();
InitDepthType();
InitCellPoints();
ResetPerceptionBuffer();
m_ObservationSpec = ObservationSpec.Visual(m_GridSize.x, m_GridSize.z, m_CellObservationSize);
m_PerceptionTexture = new Texture2D(m_GridSize.x, m_GridSize.z, TextureFormat.RGB24, false);
m_ColliderBuffer = new Collider[Math.Min(m_MaxColliderBufferSize, m_InitialColliderBufferSize)];
}
/// <summary>
/// Creates a <see cref="GridSensor"/> instance.
/// </summary>
/// <param name="buffer">The <see cref="GridBuffer"/> instance to wrap</param>
/// <param name="compressionType">The <see cref="SensorCompressionType"/>
/// to apply to the generated image</param>
/// <param name="observationType">The <see cref="ObservationType"/>
/// (default or goal signal) of the sensor</param>
/// <param name="name">Name of the sensor</param>
public GridSensor(
string name,
GridBuffer buffer,
SensorCompressionType compressionType,
ObservationType observationType)
{
m_Name = name;
buffer.GetShape().Validate();
m_GridBuffer = buffer;
m_CompressionType = compressionType;
HandleCompressionType();
m_ObservationSpec = ObservationSpec.Visual(
m_GridBuffer.Height, m_GridBuffer.Width, m_GridBuffer.NumChannels, observationType);
}
public void Test3DStacking()
{
var wrapped = new Dummy3DSensor();
wrapped.ObservationSpec = ObservationSpec.Visual(2, 1, 2);
var sensor = new StackingSensor(wrapped, 2);
// Check the stacking is on the last dimension
wrapped.CurrentObservation = new[, , ] {
{ { 1f, 2f } }, { { 3f, 4f } }
};
SensorTestHelper.CompareObservation(sensor, new[, , ] {
{ { 0f, 0f, 1f, 2f } }, { { 0f, 0f, 3f, 4f } }
});
sensor.Update();
wrapped.CurrentObservation = new[, , ] {
{ { 5f, 6f } }, { { 7f, 8f } }
};
SensorTestHelper.CompareObservation(sensor, new[, , ] {
{ { 1f, 2f, 5f, 6f } }, { { 3f, 4f, 7f, 8f } }
});
sensor.Update();
wrapped.CurrentObservation = new[, , ] {
{ { 9f, 10f } }, { { 11f, 12f } }
};
SensorTestHelper.CompareObservation(sensor, new[, , ] {
{ { 5f, 6f, 9f, 10f } }, { { 7f, 8f, 11f, 12f } }
});
// Check that if we don't call Update(), the same observations are produced
SensorTestHelper.CompareObservation(sensor, new[, , ] {
{ { 5f, 6f, 9f, 10f } }, { { 7f, 8f, 11f, 12f } }
});
// Test reset
sensor.Reset();
wrapped.CurrentObservation = new[, , ] {
{ { 13f, 14f } }, { { 15f, 16f } }
};
SensorTestHelper.CompareObservation(sensor, new[, , ] {
{ { 0f, 0f, 13f, 14f } }, { { 0f, 0f, 15f, 16f } }
});
}
public void TestVectorObsSpec()
{
var obsSpec = ObservationSpec.Vector(5);
Assert.AreEqual(1, obsSpec.Rank);
var shape = obsSpec.Shape;
Assert.AreEqual(1, shape.Length);
Assert.AreEqual(5, shape[0]);
var dimensionProps = obsSpec.DimensionProperties;
Assert.AreEqual(1, dimensionProps.Length);
Assert.AreEqual(DimensionProperty.None, dimensionProps[0]);
Assert.AreEqual(ObservationType.Default, obsSpec.ObservationType);
}
/// <summary>
/// Calls the initialization methods. Creates the data storing properties used to send the data
/// Establishes
/// </summary>
public virtual void Start()
{
InitGridParameters();
InitDepthType();
InitCellPoints();
InitPerceptionBuffer();
m_ColliderBuffer = new Collider[Math.Min(MaxColliderBufferSize, InitialColliderBufferSize)];
// Default root reference to current game object
if (rootReference == null)
{
rootReference = gameObject;
}
m_ObservationSpec = ObservationSpec.Visual(GridNumSideX, GridNumSideZ, ObservationPerCell);
compressedImgs = new List <byte[]>();
byteSizesBytesList = new List <byte[]>();
m_perceptionTexture2D = new Texture2D(GridNumSideX, GridNumSideZ, TextureFormat.RGB24, false);
}
public void TestVariableLengthObsSpec()
{
var obsSpec = ObservationSpec.VariableLength(5, 6);
Assert.AreEqual(2, obsSpec.Rank);
var shape = obsSpec.Shape;
Assert.AreEqual(2, shape.Length);
Assert.AreEqual(5, shape[0]);
Assert.AreEqual(6, shape[1]);
var dimensionProps = obsSpec.DimensionProperties;
Assert.AreEqual(2, dimensionProps.Length);
Assert.AreEqual(DimensionProperty.VariableSize, dimensionProps[0]);
Assert.AreEqual(DimensionProperty.None, dimensionProps[1]);
Assert.AreEqual(ObservationType.Default, obsSpec.ObservationType);
}
public void TestStackedGetCompressedObservation()
{
var wrapped = new Dummy3DSensor();
wrapped.ObservationSpec = ObservationSpec.Visual(1, 1, 3);
var sensor = new StackingSensor(wrapped, 2);
wrapped.CurrentObservation = new[, , ] {
{ { 1f, 2f, 3f } }
};
var expected1 = sensor.CreateEmptyPNG();
expected1 = expected1.Concat(Array.ConvertAll(new[] { 1f, 2f, 3f }, (z) => (byte)z)).ToArray();
Assert.AreEqual(sensor.GetCompressedObservation(), expected1);
sensor.Update();
wrapped.CurrentObservation = new[, , ] {
{ { 4f, 5f, 6f } }
};
var expected2 = Array.ConvertAll(new[] { 1f, 2f, 3f, 4f, 5f, 6f }, (z) => (byte)z);
Assert.AreEqual(sensor.GetCompressedObservation(), expected2);
sensor.Update();
wrapped.CurrentObservation = new[, , ] {
{ { 7f, 8f, 9f } }
};
var expected3 = Array.ConvertAll(new[] { 4f, 5f, 6f, 7f, 8f, 9f }, (z) => (byte)z);
Assert.AreEqual(sensor.GetCompressedObservation(), expected3);
// Test reset
sensor.Reset();
wrapped.CurrentObservation = new[, , ] {
{ { 10f, 11f, 12f } }
};
var expected4 = sensor.CreateEmptyPNG();
expected4 = expected4.Concat(Array.ConvertAll(new[] { 10f, 11f, 12f }, (z) => (byte)z)).ToArray();
Assert.AreEqual(sensor.GetCompressedObservation(), expected4);
}
public void TestVisualObsSpec()
{
var obsSpec = ObservationSpec.Visual(5, 6, 7);
Assert.AreEqual(3, obsSpec.Rank);
var shape = obsSpec.Shape;
Assert.AreEqual(3, shape.Length);
Assert.AreEqual(5, shape[0]);
Assert.AreEqual(6, shape[1]);
Assert.AreEqual(7, shape[2]);
var dimensionProps = obsSpec.DimensionProperties;
Assert.AreEqual(3, dimensionProps.Length);
Assert.AreEqual(DimensionProperty.TranslationalEquivariance, dimensionProps[0]);
Assert.AreEqual(DimensionProperty.TranslationalEquivariance, dimensionProps[1]);
Assert.AreEqual(DimensionProperty.None, dimensionProps[2]);
Assert.AreEqual(ObservationType.Default, obsSpec.ObservationType);
}
public PhysicsBodySensor(ArticulationBody rootBody, PhysicsSensorSettings settings, string sensorName = null)
{
var poseExtractor = new ArticulationBodyPoseExtractor(rootBody);
m_PoseExtractor = poseExtractor;
m_SensorName = string.IsNullOrEmpty(sensorName) ? $"ArticulationBodySensor:{rootBody?.name}" : sensorName;
m_Settings = settings;
var numJointExtractorObservations = 0;
m_JointExtractors = new List <IJointExtractor>(poseExtractor.NumEnabledPoses);
foreach (var articBody in poseExtractor.GetEnabledArticulationBodies())
{
var jointExtractor = new ArticulationBodyJointExtractor(articBody);
numJointExtractorObservations += jointExtractor.NumObservations(settings);
m_JointExtractors.Add(jointExtractor);
}
var numTransformObservations = m_PoseExtractor.GetNumPoseObservations(settings);
m_ObservationSpec = ObservationSpec.Vector(numTransformObservations + numJointExtractorObservations);
}
/// <summary>
/// Construct a new PhysicsBodySensor
/// </summary>
/// <param name="poseExtractor"></param>
/// <param name="settings"></param>
/// <param name="sensorName"></param>
public PhysicsBodySensor(
RigidBodyPoseExtractor poseExtractor,
PhysicsSensorSettings settings,
string sensorName
)
{
m_PoseExtractor = poseExtractor;
m_SensorName = sensorName;
m_Settings = settings;
var numJointExtractorObservations = 0;
m_JointExtractors = new List <IJointExtractor>(poseExtractor.NumEnabledPoses);
foreach (var rb in poseExtractor.GetEnabledRigidbodies())
{
var jointExtractor = new RigidBodyJointExtractor(rb);
numJointExtractorObservations += jointExtractor.NumObservations(settings);
m_JointExtractors.Add(jointExtractor);
}
var numTransformObservations = m_PoseExtractor.GetNumPoseObservations(settings);
m_ObservationSpec = ObservationSpec.Vector(numTransformObservations + numJointExtractorObservations);
}
public ObservationSpec GetObservationSpec()
{
return(ObservationSpec.Visual(m_Height, m_Width, m_Channels));
}
public DummySensor(int dim1, int dim2)
{
m_ObservationSpec = ObservationSpec.VariableLength(dim1, dim2);
}
/// <inheritdoc/>
public override ObservationSpec GetObservationSpec()
{
return(ObservationSpec.Vector(BasicController.k_Extents));
}
public DummySensor(int dim1)
{
m_ObservationSpec = ObservationSpec.Vector(dim1);
}
public DummySensor(int dim1, int dim2, int dim3)
{
m_ObservationSpec = ObservationSpec.Visual(dim1, dim2, dim3);
}
