public ShadowShaderRenderer(Model model, PictureBox box)
{
pictureBox = box;
this.model = model;
width = pictureBox.Width;
height = pictureBox.Height;
light_dir = new Vector3(1, 1, -0.4f);
eye = new Vector3(0, 0, 3);
center = new Vector3(0, 0, 0);
up = new Vector3(0, 1, 0);
Matrix4x4 Rot_X = Matrix4x4Extensions.Rotation_X(20);
Matrix4x4 Rot_Y = Matrix4x4Extensions.Rotation_Y(15);
eye = (Rot_X * Rot_Y).MultiplyByVector3_V3(eye);
z_buffer = new float[width * height];
shadow_buffer = new float[width * height];
for (int i = 0; i < z_buffer.Length; i++)
{
z_buffer[i] = shadow_buffer[i] = float.MinValue;
}
light_dir = light_dir.Normalize();
}
public NoToneShaderRenderer(Model model, PictureBox box)
{
pictureBox = box;
this.model = model;
width = pictureBox.Width;
height = pictureBox.Height;
light_dir = new Vector3(0, 0, -1);
eye = new Vector3(0, 0, 3);
center = new Vector3(0, 0, 0);
up = new Vector3(0, 1, 0);
Matrix4x4 Rot_X = Matrix4x4Extensions.Rotation_X(20);
Matrix4x4 Rot_Y = Matrix4x4Extensions.Rotation_Y(15);
eye = (Rot_X * Rot_Y).MultiplyByVector3_V3(eye);
ModelView = Matrix4x4Extensions.LookAt(eye, center, up);
Projection = Matrix4x4.Identity;
ViewPort = Matrix4x4Extensions.Viewport(width / 8, height / 8, width * 3 / 4, height * 3 / 4);
Projection.M43 = -1.0f / (eye - center).Norm();
z_buffer = new float[pictureBox.Width * pictureBox.Height];
for (int i = 0; i < z_buffer.Length; i++)
{
z_buffer[i] = -float.MaxValue;
}
light_dir.Normalize();
}
public Matrix4x4 GetPivotTransformation(Frame frame, bool mirrored)
{
Matrix4x4 matrix = frame.GetBoneTransformation(Sensor, mirrored);
Matrix4x4Extensions.SetPosition(ref matrix, matrix.GetPosition() + matrix.GetRotation() * Offset);
return(matrix);
}
private void ApplyStaticGoal(Vector3 position, Vector3 direction, float[] actions)
{
//Transformations
for (int i = 0; i < TimeSeries.Samples.Length; i++)
{
float weight = TimeSeries.GetWeight1byN1(i, 2f);
float positionBlending = weight * UserControl;
float directionBlending = weight * UserControl;
Matrix4x4Extensions.SetPosition(ref GoalSeries.Transformations[i], Vector3.Lerp(GoalSeries.Transformations[i].GetPosition(), position, positionBlending));
Matrix4x4Extensions.SetRotation(ref GoalSeries.Transformations[i], Quaternion.LookRotation(Vector3.Slerp(GoalSeries.Transformations[i].GetForward(), direction, directionBlending), Vector3.up));
}
//Actions
for (int i = TimeSeries.Pivot; i < TimeSeries.Samples.Length; i++)
{
float w = (float)(i - TimeSeries.Pivot) / (float)(TimeSeries.FutureSampleCount);
w = Utility.Normalise(w, 0f, 1f, 1f / TimeSeries.FutureKeyCount, 1f);
for (int j = 0; j < GoalSeries.Actions.Length; j++)
{
float weight = GoalSeries.Values[i][j];
weight = 2f * (0.5f - Mathf.Abs(weight - 0.5f));
weight = Utility.Normalise(weight, 0f, 1f, UserControl, 1f - UserControl);
if (actions[j] != GoalSeries.Values[i][j])
{
GoalSeries.Values[i][j] = Mathf.Lerp(
GoalSeries.Values[i][j],
Mathf.Clamp(GoalSeries.Values[i][j] + weight * UserControl * Mathf.Sign(actions[j] - GoalSeries.Values[i][j]), 0f, 1f),
w);
}
}
}
}
public void Fix()
{
for (int i = 0; i < Transformations.Length; i++)
{
Matrix4x4Extensions.SetScale(ref Transformations[i], new Vector3(1.5f, 1f, 1f));
}
}
private void PlaceCamera(Box box)
{
var max = Math.Max(box.size.X * 1.3f, box.size.Y * 1.3f * camera.aspectRatio);
var tan = (float)Math.Tan(Const.Deg2Rad * camera.fov / 2);
var dist = max / tan;
camera.transform.localToWorldMatrix = Matrix4x4Extensions.LookAt(new Vector3(0, 0.5f, dist), box.center);
}
private void ApplyDynamicGoal(Matrix4x4 root, Vector3 move, float turn, float[] actions)
{
//Transformations
Vector3[] positions_blend = new Vector3[TimeSeries.Samples.Length];
Vector3[] directions_blend = new Vector3[TimeSeries.Samples.Length];
float time = 2f;
for (int i = 0; i < TimeSeries.Samples.Length; i++)
{
float weight = TimeSeries.GetWeight1byN1(i, 0.5f);
float bias_pos = 1.0f - Mathf.Pow(1.0f - weight, 0.75f);
float bias_dir = 1.0f - Mathf.Pow(1.0f - weight, 0.75f);
directions_blend[i] = Quaternion.AngleAxis(bias_dir * turn, Vector3.up) * Vector3.ProjectOnPlane(root.GetForward(), Vector3.up).normalized;
if (i == 0)
{
positions_blend[i] = root.GetPosition() +
Vector3.Lerp(
GoalSeries.Transformations[i + 1].GetPosition() - GoalSeries.Transformations[i].GetPosition(),
time / (TimeSeries.Samples.Length - 1f) * (Quaternion.LookRotation(directions_blend[i], Vector3.up) * move),
bias_pos
);
}
else
{
positions_blend[i] = positions_blend[i - 1] +
Vector3.Lerp(
GoalSeries.Transformations[i].GetPosition() - GoalSeries.Transformations[i - 1].GetPosition(),
time / (TimeSeries.Samples.Length - 1f) * (Quaternion.LookRotation(directions_blend[i], Vector3.up) * move),
bias_pos
);
}
}
for (int i = 0; i < TimeSeries.Samples.Length; i++)
{
Matrix4x4Extensions.SetPosition(ref GoalSeries.Transformations[i], Vector3.Lerp(GoalSeries.Transformations[i].GetPosition(), positions_blend[i], UserControl));
Matrix4x4Extensions.SetRotation(ref GoalSeries.Transformations[i], Quaternion.Slerp(GoalSeries.Transformations[i].GetRotation(), Quaternion.LookRotation(directions_blend[i], Vector3.up), UserControl));
}
//Actions
for (int i = TimeSeries.Pivot; i < TimeSeries.Samples.Length; i++)
{
float w = (float)(i - TimeSeries.Pivot) / (float)(TimeSeries.FutureSampleCount);
w = Utility.Normalise(w, 0f, 1f, 1f / TimeSeries.FutureKeyCount, 1f);
for (int j = 0; j < GoalSeries.Actions.Length; j++)
{
float weight = GoalSeries.Values[i][j];
weight = 2f * (0.5f - Mathf.Abs(weight - 0.5f));
weight = Utility.Normalise(weight, 0f, 1f, UserControl, 1f - UserControl);
if (actions[j] != GoalSeries.Values[i][j])
{
GoalSeries.Values[i][j] = Mathf.Lerp(
GoalSeries.Values[i][j],
Mathf.Clamp(GoalSeries.Values[i][j] + weight * UserControl * Mathf.Sign(actions[j] - GoalSeries.Values[i][j]), 0f, 1f),
w);
}
}
}
}
protected override async Task OnRenderAsync(bool firstRender, int width, int height)
{
if (firstRender)
{
await _context.EnableAsync(EnableCap.DEPTH_TEST);
// set up world matrix
await _context.UniformMatrixAsync(_matLocWorld, false, _matWorld.Values1D());
// set up view matrix
var mView = Matrix4x4.CreateLookAt(new Vector3(0f, 0f, -25f), new Vector3(0f, 0f, 0f), new Vector3(0f, 1f, 0f));
await _context.UniformMatrixAsync(_matLocView, false, mView.Values1D());
// this will set up projection matrix
await UpdateSizeAsync(width, height);
}
if (SpectrumBuffer == null || SpectrumBuffer.Length != 25)
{
return;
} // must match x/y iterations below
await _context.ClearColorAsync(0.184f, 0.310f, 0.310f, 1f);
await _context.ClearAsync(BufferBits.COLOR_BUFFER_BIT | BufferBits.DEPTH_BUFFER_BIT);
// Please note: I have no idea why I am transposing the matrix's so much. I obvioulsy don't
// understand how the matrix is stored in System.Numerics.Matrix4x4
var rotate = (float)((Environment.TickCount - _tickStart) / 1000.0 / 6.0 * 2.0 * Math.PI);
var matRotate = Matrix4x4.Transpose(Matrix4x4Extensions.CreateRotate(rotate, new Vector3(.5f, 1f, -.2f)));
var matWorld = Matrix4x4.Transpose(_matWorld);
var ind = 0;
await _context.BeginBatchAsync();
for (var y = -2; y < 3; ++y)
{
for (var x = -2; x < 3; ++x)
{
var mFin =
matWorld *
matRotate *
Matrix4x4.Transpose(Matrix4x4.CreateTranslation(
new Vector3(2.1f * x, 2.1f * y, 0f))) *
Matrix4x4.Transpose(Matrix4x4.CreateScale(
new Vector3(1f, 1f, (float)SpectrumBuffer[_dicMap[new Point(x, y)]] * .5f)));
await _context.UniformMatrixAsync(_matLocWorld, false, mFin.Values1D(true));
await _context.DrawElementsAsync(
Primitive.TRIANGLES, _indexBuffer.ElementCount, _indexBuffer.DataType, 0);
++ind;
}
}
await _context.EndBatchAsync();
}
void OnTrackerMove(Vector3 pos, Quaternion rot)
{
// Convert the tracker's position and rotation to a Matrix4x4 format.
Matrix4x4 trackerMat = Matrix4x4.TRS(pos, rot, Vector3.one);
if ((buttonPressed) && (selected == 0))
{
// Dragging while holding onto the menu title bar,
// move the menu based on motion of the tracker
// Get the menu's Transform in Matrix4x4 format
Matrix4x4 origMenuMat = transform.localToWorldMatrix;
// Calc change in tracker pos and rot from the last frame until now
Matrix4x4 deltaTracker = trackerMat * lastTrackerMat.inverse;
// Apply this change to the menu's current transformation to find its new transform
Matrix4x4 newMenuMat = deltaTracker * origMenuMat;
// Save the result, converting from Matrix4x4 back to unity's Transform class.
transform.position = Matrix4x4Extensions.GetTranslation(newMenuMat);
transform.rotation = Matrix4x4Extensions.GetRotation(newMenuMat);
}
else if (!buttonPressed)
{
// Clear selection and highlighting
selected = -1;
titleBoxObj.GetComponent <Renderer>().material.color = titleBGColor;
for (int i = 0; i < labelBoxes.Count; i++)
{
labelBoxes[i].GetComponent <Renderer>().material.color = itemBGColor;
}
// Update selection
if (InsideTransformedCube(pos, titleBoxObj))
{
selected = 0;
titleBoxObj.GetComponent <Renderer>().material.color = titleHighColor;
}
else
{
for (int i = 0; i < labelBoxes.Count; i++)
{
if (InsideTransformedCube(pos, labelBoxes[i]))
{
selected = i + 1;
labelBoxes[i].GetComponent <Renderer>().material.color = itemHighColor;
}
}
}
}
lastTrackerMat = trackerMat;
}
public void SetDirection(int index, Vector3 direction)
{
Matrix4x4Extensions.SetRotation(ref Transformations[index], Quaternion.LookRotation(direction == Vector3.zero ? Vector3.forward : direction, Vector3.up));
}
public void SetRotation(int index, Quaternion rotation)
{
Matrix4x4Extensions.SetRotation(ref Transformations[index], rotation);
}
public void SetPosition(int index, Vector3 position)
{
Matrix4x4Extensions.SetPosition(ref Transformations[index], position);
}
public void SetRotation(Quaternion rotation)
{
Matrix4x4Extensions.SetRotation(ref Transformation, rotation);
}
public void SetPosition(Vector3 position)
{
Matrix4x4Extensions.SetPosition(ref Transformation, position);
}
public void Render()
{
Stopwatch sw = new Stopwatch();
sw.Start();
DirectBitmap bitmap = new DirectBitmap(width, height);
Rasterizer rasterizer = new Rasterizer();
//// start of shadow buffer rendering
Matrix4x4 SB_ModelView = Matrix4x4Extensions.LookAt(light_dir, center, up);
Matrix4x4 SB_Projection = Matrix4x4.Identity;
Matrix4x4 ViewPort = Matrix4x4Extensions.Viewport(width / 8, height / 8, width * 3 / 4, height * 3 / 4);
SB_Projection.M43 = 0;
var shader = new DepthShader(model, ViewPort, SB_Projection, SB_ModelView);
for (int i = 0; i < model.FacesV.Count; i++)
{
Vector4[] screen_coords = new Vector4[3];
for (int j = 0; j < model.FacesV[i].Length; j++)
{
screen_coords[j] = shader.Vertex(i, j);
}
rasterizer.Triangle(screen_coords, shader, bitmap, pictureBox, shadow_buffer, true);
}
//// end of shadow buffer rendering
//// start of picture render by shadow shader ////
Matrix4x4 M = ViewPort * SB_Projection * SB_ModelView;
Matrix4x4 ModelView = Matrix4x4Extensions.LookAt(eye, center, up);
Matrix4x4 Projection = Matrix4x4.Identity;
Projection.M43 = -1.0f / (eye - center).Norm();
var sh_shader = new ShadowShader(model, ViewPort, Projection, ModelView, light_dir, shadow_buffer, width);
sh_shader.uniform_m = Projection * ModelView;
sh_shader.uniform_mit = sh_shader.uniform_m.Inverse().Transpose();
sh_shader.uniform_m_shadow = M * (ViewPort * Projection * ModelView).Inverse();
for (int i = 0; i < model.FacesV.Count; i++)
{
Vector4[] screen_coords = new Vector4[3];
for (int j = 0; j < model.FacesV[i].Length; j++)
{
screen_coords[j] = sh_shader.Vertex(i, j);
}
rasterizer.Triangle(screen_coords, sh_shader, bitmap, pictureBox, z_buffer);
}
//// end of picture render by shadow shader ////
sw.Stop();
model.Texture.Dispose();
bitmap.Bitmap.RotateFlip(RotateFlipType.Rotate180FlipNone);
InfoViewer.ShowDiagnosticInfo
($"Render Time: {sw.Elapsed.TotalMilliseconds:f2} ms", new PointF(10, 30), Graphics.FromImage(bitmap.Bitmap));
// output final render
pictureBox.Image = bitmap.Bitmap;
}
public Matrix4x4[] CleanupBallTransformations(bool detectInvalid)
{
bool invalid = IsInvalid();
if (!detectInvalid && invalid)
{
invalid = false;
Debug.LogWarning("Ignored invalid detection in asset " + Data.GetName() + ".");
}
Matrix4x4[] motion = new Matrix4x4[Data.Frames.Length];
for (int i = 0; i < Data.Frames.Length; i++)
{
Matrix4x4 ball = GetTransformation(Ball, Data.Frames[i].Timestamp);
Matrix4x4 root = GetRoot(Data.Frames[i].Timestamp, false);
float radius = GetControlRadius();
if (invalid)
{
//Project Surface
Matrix4x4 head = GetTransformation(Head, Data.Frames[i].Timestamp);
float offset = (head.GetPosition() - root.GetPosition()).ZeroY().magnitude;
Vector3 projection = head.GetPosition() + (radius - offset) * (head.GetRotation() * Axis.GetAxis());
Vector3 position = (root.GetPosition().ZeroY() + radius * (projection - root.GetPosition()).ZeroY().normalized).SetY(projection.y);
position.y = Mathf.Max(position.y, root.GetPosition().y + Radius);
Matrix4x4Extensions.SetPosition(ref ball, position);
}
else
{
//Clamp Magnitude
Matrix4x4Extensions.SetPosition(ref ball, ball.GetPosition().ClampMagnitudeXZ(radius, root.GetPosition()));
}
motion[i] = ball;
}
return(motion);
Matrix4x4 GetTransformation(int bone, float timestamp)
{
float start = Data.GetFirstValidFrame().Timestamp;
float end = Data.GetLastValidFrame().Timestamp;
if (timestamp < start || timestamp > end)
{
float boundary = Mathf.Clamp(timestamp, start, end);
float pivot = 2f * boundary - timestamp;
float clamped = Mathf.Clamp(pivot, start, end);
Matrix4x4 reference = Data.GetFrame(clamped).GetBoneTransformation(bone, false);
Vector3 position = 2f * Data.GetFrame(boundary).GetBoneTransformation(bone, false).GetPosition() - reference.GetPosition();
position.y = reference.GetPosition().y;
Quaternion rotation = reference.GetRotation();
return(Matrix4x4.TRS(position, rotation, Vector3.one));
}
else
{
return(Data.GetFrame(timestamp).GetBoneTransformation(bone, false));
}
}
bool IsInvalid()
{
//Get Range
int start = Data.GetFirstValidFrame().Index - 1;
int end = Data.GetLastValidFrame().Index - 1;
//Check contacts
ContactModule contact = GetContactModule();
if (contact == null)
{
return(false);
}
ContactModule.Sensor left = contact.GetSensor(Data.Source.Bones[LeftHand].Name);
ContactModule.Sensor right = contact.GetSensor(Data.Source.Bones[RightHand].Name);
if (left != null && right != null)
{
float[] leftContacts = left.Contacts.GatherByPivots(start, end);
float[] rightContacts = left.Contacts.GatherByPivots(start, end);
if (leftContacts.All(1f, 0.9f) ^ rightContacts.All(1f, 0.9f) && !ArrayExtensions.Equal(leftContacts, rightContacts).All(true, 0.1f))
{
Debug.LogWarning("Stuck-to-hand invalidation in file " + Data.GetName() + ".");
return(true);
}
}
//Check transformations
Matrix4x4[] transformations = new Matrix4x4[Data.Frames.Length];
for (int i = 0; i < Data.Frames.Length; i++)
{
transformations[i] = Data.Frames[i].GetBoneTransformation(Ball, false);
}
if (transformations.GatherByPivots(start, end).Repeating(0.9f))
{
Debug.LogWarning("Repeating ball transformation invalidation in file " + Data.GetName() + ".");
return(true);
}
//Check velocities
Vector3[] velocities = new Vector3[Data.Frames.Length];
for (int i = 0; i < Data.Frames.Length; i++)
{
velocities[i] = Data.Frames[i].GetBoneVelocity(Ball, false);
}
if (velocities.GatherByPivots(start, end).Repeating(0.9f))
{
Debug.LogWarning("Ball velocity invalidation in file " + Data.GetName() + ".");
return(true);
}
// //Check out-of-bounds
// bool[] bounds = new bool[Data.Frames.Length];
// for(int i=0; i<Data.Frames.Length; i++) {
// bounds[i] = Vector3.Distance(GetRoot(Data.Frames[i].Timestamp, false).GetPosition(), Data.Frames[i].GetBoneTransformation(Ball, false).GetPosition()) > 0.5f*GetInteractionArea();
// }
// if(bounds.All(true, 0.9f)) {
// return true;
// }
//All seems good
return(false);
}
}