public ImagePair(TangoEnums.TangoCameraId cameraId, TangoUnityImageData imageBuffer)
{
this.cameraId = cameraId;
this.imageBuffer = imageBuffer;
if (cameraId == TangoEnums.TangoCameraId.TANGO_CAMERA_FISHEYE)
{
cameraName = "fisheye";
}
else if (cameraId == TangoEnums.TangoCameraId.TANGO_CAMERA_COLOR)
{
cameraName = "colorcamera";
}
else if (cameraId == TangoEnums.TangoCameraId.TANGO_CAMERA_DEPTH)
{
cameraName = "depthcamera";
}
else if (cameraId == TangoEnums.TangoCameraId.TANGO_CAMERA_RGBIR)
{
cameraName = "rgbircamera";
}
else
{
cameraName = "unknowncamera";
}
}
public void OnTangoImageAvailableEventHandler(TangoEnums.TangoCameraId cameraId, TangoUnityImageData imageBuffer)
{
if (imageIsRecording)
{
ImagePair ip = new ImagePair(cameraId, imageBuffer);
image_queue.Enqueue(ip);
}
}
void Start()
{
m_tangoApplication = FindObjectOfType <TangoApplication>();
m_pointCloud = FindObjectOfType <TangoPointCloud>();
m_imagebuffer = new TangoUnityImageData();
if (m_tangoApplication != null)
{
m_tangoApplication.Register(this);
}
else
{
Debug.Log("No Tango Manager found in scene.");
}
}
/// <summary>
/// Computes the spherical harmonic diffuse coefficients for a given
/// TangoImageBuffer.
/// </summary>
/// <param name="imageBuffer">The TangoImageBuffer to sample.</param>
private void _ComputeDiffuseCoefficients(TangoUnityImageData imageBuffer)
{
if (m_enableEnvironmentalLighting)
{
// Compute SH Coefficients.
float weight = 4.0f * Mathf.PI;
int numSamples = m_samples.Length;
int numCoefficients = m_coefficients.Length;
for (int coeffIdx = 0; coeffIdx < numCoefficients; ++coeffIdx)
{
m_coefficients[coeffIdx] = Vector3.zero;
}
for (int sampleIdx = 0; sampleIdx < numSamples; ++sampleIdx)
{
float theta = m_samples[sampleIdx].sph.x;
float phi = m_samples[sampleIdx].sph.y;
// Normalize between 0 and 1.
float x = 1.0f - Mathf.Pow(Mathf.Cos(theta / 2.0f), 2.0f);
float y = phi / Mathf.PI / 2.0f;
int i = (int)(imageBuffer.height * x);
int j = (int)(imageBuffer.width * y);
Vector3 rgb = _GetRgbFromImageBuffer(imageBuffer, i, j);
for (int coeffIdx = 0; coeffIdx < numCoefficients; ++coeffIdx)
{
m_coefficients[coeffIdx] += rgb * m_samples[sampleIdx].coeff[coeffIdx];
}
}
// Divide the result by weight and number of samples.
float factor = weight / numSamples;
for (int coeffIdx = 0; coeffIdx < numCoefficients; ++coeffIdx)
{
m_coefficients[coeffIdx] *= factor;
}
Shader.SetGlobalMatrix("_TangoLightingSphericalHarmonicMatrixR", _SetShmMatrix(0));
Shader.SetGlobalMatrix("_TangoLightingSphericalHarmonicMatrixG", _SetShmMatrix(1));
Shader.SetGlobalMatrix("_TangoLightingSphericalHarmonicMatrixB", _SetShmMatrix(2));
Shader.SetGlobalFloat("_TangoLightingExposure", m_coefficients[0].magnitude);
}
}
/// <summary>
/// Detect one or more markers in the input image.
/// </summary>
/// <param name="cameraId">
/// Returned camera ID.
/// </param>
/// <param name="imageBuffer">
/// Color camera image buffer.
/// </param>
public void OnTangoImageAvailableEventHandler(TangoEnums.TangoCameraId cameraId,
TangoUnityImageData imageBuffer)
{
TangoSupport.DetectMarkers(imageBuffer, cameraId,
TangoSupport.MarkerType.ARTAG, MARKER_SIZE, m_markerList);
for (int i = 0; i < m_markerList.Count; ++i)
{
TangoSupport.Marker marker = m_markerList[i];
int markerId = Convert.ToInt32(marker.m_content);
int planetIndex = (markerId - 1) % m_planets.Length;
if (m_planets[planetIndex] == null)
{
m_planets[planetIndex] = Instantiate <GameObject>(m_planetsPrefabs[planetIndex]);
}
m_planets[planetIndex].transform.position = marker.m_translation;
m_planets[planetIndex].transform.rotation = marker.m_orientation;
}
}
/// <summary>
/// Detect one or more markers in the input image.
/// </summary>
/// <param name="cameraId">
/// Returned camera ID.
/// </param>
/// <param name="imageBuffer">
/// Color camera image buffer.
/// </param>
public void OnTangoImageAvailableEventHandler(TangoEnums.TangoCameraId cameraId,
TangoUnityImageData imageBuffer)
{
TangoSupport.DetectMarkers(imageBuffer, cameraId,
TangoSupport.MarkerType.ARTAG, MARKER_SIZE, m_markerList);
for (int i = 0; i < m_markerList.Count; ++i)
{
TangoSupport.Marker marker = m_markerList[i];
if (m_markerObjects.ContainsKey(marker.m_content))
{
GameObject markerObject = m_markerObjects[marker.m_content];
markerObject.GetComponent <MarkerVisualizationObject>().SetMarker(marker);
}
else
{
GameObject markerObject = Instantiate <GameObject>(m_markerPrefab);
m_markerObjects.Add(marker.m_content, markerObject);
markerObject.GetComponent <MarkerVisualizationObject_Custom>().SetMarker(marker);
}
}
}
void saveImage(TangoUnityImageData im, string scan_name)
{
//double t = im->timestamp;
double t = im.timestamp;
//size_t height = im->height+im->height/2;
//size_t width = im->width;
uint height = im.height + im.height / 2;
uint width = im.width;
if (scan_name == "fisheye")
{
//height = im->height;
//width = im->stride; // stride > width for fisheye
height = im.height;
width = im.stride; // stride > width for fisheye
}
// Path to the file
string filePath = imagePath + "/" + scan_name + "_" + t + ".yv12";
// Create a file and write all bytes to it
File.WriteAllBytes(filePath, im.data);
}
void ITangoVideoOverlay.OnTangoImageAvailableEventHandler(TangoEnums.TangoCameraId cameraId, TangoUnityImageData imageBuffer)
{
m_imagebuffer = imageBuffer;
//Debug.Log("<<<<<<<<<< In Image Available Event Handler: " + imageBuffer.data.GetValue(0));
}
/// <summary>
/// This will be called when a new frame is available from the camera.
/// </summary>
/// <param name="cameraId">Camera identifier.</param>
/// <param name="imageBuffer">Tango camera image buffer.</param>
public void OnTangoImageAvailableEventHandler(TangoEnums.TangoCameraId cameraId, TangoUnityImageData imageBuffer)
{
if (fCount == 0)
{
TangoEnums.TangoImageFormatType enumDisplayStatus = (TangoEnums.TangoImageFormatType)imageBuffer.format;
string stringValue = enumDisplayStatus.ToString();
JLog("TangoImageAvailable on cameraId: " + cameraId + " Format: " + imageBuffer.format + "(" + stringValue + "), length: " + imageBuffer.data.Length + " (" + imageBuffer.width + "x" + imageBuffer.height + "), timestamp: " + imageBuffer.timestamp + ", Stride: " + imageBuffer.stride);
}
fCount++;
// if (_FileStream!= null && _FileStream.Length >= 600464512) {
// JLog ("Closed File: " + _FileStream.Name);
// _FileStream.Close();
// _FileStream = null;
// } else if ( _FileStream.CanWrite) {
// JLog ("Write File: " + _FileStream.Length);
// _FileStream.Write (imageBuffer.data, 0, imageBuffer.data.Length);
// }
// //For each YV12 frame BROKEN
// for (int i = 0; i < imageBuffer.width * imageBuffer.height; i+=36) {
// for (int y = 0; y < 24; y++)
// JLog ("Y: " + imageBuffer.data[i+y]);
// for (int u = 24; u < 30; u++)
// JLog ("U: " + imageBuffer.data[i+u]);
// for (int v = 30; v < 36; v++)
// JLog ("V: " + imageBuffer.data[i+v]);
// JLog ("--------------");
// size.total = size.width * size.height;
// y = yuv [position.y * size.width + position.x];
// u = yuv [(position.y / 2) * (size.width / 2) + (position.x / 2) + size.total];
// v = yuv [(position.y / 2) * (size.width / 2) + (position.x / 2) + size.total + (size.total / 4)];
// }
// if(fCount%100==0)
// System.Diagnostics.Stopwatch timer = System.Diagnostics.Stopwatch.StartNew();
callUnityTangoARPlayer("arwAcceptVideoImage", new object[] {
halveYUV420(
halveYUV420(imageBuffer.data, imageBuffer.width, imageBuffer.height),
imageBuffer.width / 2,
imageBuffer.height / 2
)
}
);
//
// timer.Stop();
// TimeSpan timespan = timer.Elapsed;
////
// JLog( String.Format("{0:00}:{1:00}:{2:00}", timespan.Minutes, timespan.Seconds, timespan.Milliseconds / 10) );
//We are calling ARToolkit Library native function, through JNI, to feed in a frame of Tango video to the ARToolkit processing.
/**
* Passes a video frame to the native library for processing.
*
* @param image Buffer containing the video frame
* @param width Width of the video frame in pixels
* @param height Height of the video frame in pixels
* @param cameraIndex Zero-based index of the camera in use. If only one camera is present, will be 0.
* @param cameraIsFrontFacing false if camera is rear-facing (the default) or true if camera is facing toward the user.
* @return true if no error occurred, otherwise false
*/
//public static native boolean arwAcceptVideoImage(byte[] image, int width, int height, int cameraIndex, boolean cameraIsFrontFacing);
//public static native boolean arwAcceptVideoImage(byte[] image);
}
/// <summary>
/// This will be called when a new frame is available from the camera.
/// </summary>
/// <param name="cameraId">Camera identifier.</param>
/// <param name="imageBuffer">Tango camera image buffer.</param>
public void OnTangoImageAvailableEventHandler(TangoEnums.TangoCameraId cameraId, TangoUnityImageData imageBuffer)
{
_ComputeDiffuseCoefficients(imageBuffer);
}
/// <summary>
/// Minji Kim 2017.09.25
/// Find Edge Wrapper
/// call FindEdgesNearPoint
/// TangoSupport_findEdgesNearPoint
/// </summary>
/// <param name="cam"></param>
/// <param name="pos"></param>
/// <returns></returns>
public bool FindEdges(TangoUnityImageData imageBuffer,
Camera cam, Vector2 pos, out TangoSupport.TangoSupportEdge[] edges, out int num_edges)
//out Vector3[] end_points, out Vector3 nearest_on_edge)
{
if (m_pointsCount == 0)
{
// No points to check, maybe not connected to the service yet
edges = new TangoSupport.TangoSupportEdge[1];
num_edges = 0;
return(false);
}
Vector2 normalizedPos = cam.ScreenToViewportPoint(pos);
// If the camera has a TangoARScreen attached, it is not displaying the entire color camera image. Correct
// the normalized coordinates by taking the clipping into account.
TangoARScreen arScreen = cam.gameObject.GetComponent <TangoARScreen>();
if (arScreen != null)
{
normalizedPos = arScreen.ViewportPointToCameraImagePoint(normalizedPos);
}
// if the image data has not been updated, update it manually
if (imageBuffer.data == null)
{
GetTangoUnityImageData(ref imageBuffer);
}
bool returnValue = TangoSupport.FindEdgesNearPoint(
imageBuffer,
m_mostRecentPointCloud,
arScreen.m_screenUpdateTime,
normalizedPos,
out edges,
out num_edges
);
Vector3[] start = new Vector3[num_edges];
Vector3[] end = new Vector3[num_edges];
Vector3[] near = new Vector3[num_edges];
TangoSupport.TangoSupportEdge[] n_edges = new TangoSupport.TangoSupportEdge[num_edges];
for (int j = 0; j < num_edges; j++)
{
start[j] = new Vector3(edges[j].end_points_x1, edges[j].end_points_y1, edges[j].end_points_z1);
end[j] = new Vector3(edges[j].end_points_x2, edges[j].end_points_y2, edges[j].end_points_z2);
near[j] = new Vector3(edges[j].closest_point_on_edge_x, edges[j].closest_point_on_edge_y,
edges[j].closest_point_on_edge_z);
start[j] = m_mostRecentUnityWorldTDepthCamera.MultiplyVector(start[j]);
end[j] = m_mostRecentUnityWorldTDepthCamera.MultiplyVector(end[j]);
near[j] = m_mostRecentUnityWorldTDepthCamera.MultiplyVector(near[j]);
Vector3.Normalize(start[j]);
Vector3.Normalize(end[j]);
Vector3.Normalize(near[j]);
n_edges[j].end_points_x1 = start[j][0];
n_edges[j].end_points_y1 = start[j][1];
n_edges[j].end_points_z1 = start[j][2];
n_edges[j].end_points_x2 = end[j][0];
n_edges[j].end_points_y2 = end[j][1];
n_edges[j].end_points_z2 = end[j][2];
n_edges[j].closest_point_on_edge_x = near[j][0];
n_edges[j].closest_point_on_edge_y = near[j][1];
n_edges[j].closest_point_on_edge_z = near[j][2];
}
edges = n_edges;
return(returnValue);
}
/// <summary>
/// Minji Kim
/// from VideoOverlayListener
/// get tango unity image data
/// </summary>
/// <param name="colorImageData"></param>
private void GetTangoUnityImageData(ref TangoUnityImageData colorImageData)
{
const int EMULATED_CAMERA_WIDTH = 1280;
const int EMULATED_CAMERA_HEIGHT = 720;
const int EMULATED_CAMERA_PACKED_WIDTH = 1280 / 4;
const int EMULATED_CAMERA_PACKED_Y_HEIGHT = 720;
const int EMULATED_CAMERA_PACKED_UV_HEIGHT = 720 / 2;
const string EMULATED_RGB2YUV_Y_SHADERNAME = "Hidden/Tango/RGB2YUV_Y";
const string EMULATED_RGB2YUV_CBCR_SHADERNAME = "Hidden/Tango/RGB2YUV_CbCr";
Material m_yuvFilterY = null;
Material m_yuvFilterCbCr = null;
RenderTexture m_emulatedColorRenderTexture = null;
Texture2D[] m_emulationByteBufferCaptureTextures = null;
float m_lastColorEmulationTime = ((float)DateTime.Now.Hour + ((float)DateTime.Now.Minute * 0.01f));
CreateMaterial(EMULATED_RGB2YUV_Y_SHADERNAME, out m_yuvFilterY);
CreateMaterial(EMULATED_RGB2YUV_CBCR_SHADERNAME, out m_yuvFilterCbCr);
m_emulatedColorRenderTexture = new RenderTexture(EMULATED_CAMERA_WIDTH, EMULATED_CAMERA_HEIGHT, 24,
RenderTextureFormat.ARGB32);
m_emulationByteBufferCaptureTextures = new Texture2D[2];
m_emulationByteBufferCaptureTextures[0] = new Texture2D(EMULATED_CAMERA_PACKED_WIDTH,
EMULATED_CAMERA_PACKED_Y_HEIGHT,
TextureFormat.ARGB32, false);
m_emulationByteBufferCaptureTextures[1] = new Texture2D(EMULATED_CAMERA_PACKED_WIDTH,
EMULATED_CAMERA_PACKED_UV_HEIGHT,
TextureFormat.ARGB32, false);
int yDataSize = EMULATED_CAMERA_WIDTH * EMULATED_CAMERA_HEIGHT;
int cbCrDataSize = yDataSize / 2;
int dataSize = yDataSize + cbCrDataSize;
if (colorImageData.data == null || colorImageData.data.Length != dataSize)
{
colorImageData.data = new byte[dataSize];
}
RenderTexture yRT = RenderTexture.GetTemporary(EMULATED_CAMERA_PACKED_WIDTH,
EMULATED_CAMERA_PACKED_Y_HEIGHT,
0, RenderTextureFormat.ARGB32);
RenderTexture cbCrRT = RenderTexture.GetTemporary(EMULATED_CAMERA_PACKED_WIDTH,
EMULATED_CAMERA_PACKED_UV_HEIGHT,
0, RenderTextureFormat.ARGB32);
Graphics.Blit(m_emulatedColorRenderTexture, yRT, m_yuvFilterY);
m_emulationByteBufferCaptureTextures[0].ReadPixels(new Rect(0, 0, yRT.width, yRT.height), 0, 0);
Graphics.Blit(m_emulatedColorRenderTexture, cbCrRT, m_yuvFilterCbCr);
m_emulationByteBufferCaptureTextures[1].ReadPixels(new Rect(0, 0, cbCrRT.width, cbCrRT.height), 0, 0);
Color32[] colors = m_emulationByteBufferCaptureTextures[0].GetPixels32();
for (int i = 0; i < yDataSize / 4; i++)
{
colorImageData.data[(i * 4)] = colors[i].r;
colorImageData.data[(i * 4) + 1] = colors[i].g;
colorImageData.data[(i * 4) + 2] = colors[i].b;
colorImageData.data[(i * 4) + 3] = colors[i].a;
}
int startOffset = colors.Length * 4;
colors = m_emulationByteBufferCaptureTextures[1].GetPixels32();
for (int i = 0; i < cbCrDataSize / 4; i++)
{
colorImageData.data[(i * 4) + startOffset] = colors[i].r;
colorImageData.data[(i * 4) + startOffset + 1] = colors[i].g;
colorImageData.data[(i * 4) + startOffset + 2] = colors[i].b;
colorImageData.data[(i * 4) + startOffset + 3] = colors[i].a;
}
RenderTexture.ReleaseTemporary(yRT);
RenderTexture.ReleaseTemporary(cbCrRT);
colorImageData.format = TangoEnums.TangoImageFormatType.TANGO_HAL_PIXEL_FORMAT_YV12;
colorImageData.width = EMULATED_CAMERA_WIDTH;
colorImageData.height = EMULATED_CAMERA_HEIGHT;
colorImageData.stride = EMULATED_CAMERA_WIDTH;
colorImageData.timestamp = m_lastColorEmulationTime;
}
