public Task <bool> UploadImageAsync(ImageBody body)
{
Task <bool> task = new Task <bool>(() => UploadImage(body));
task.Start();
return(task);
}
/// <summary>
/// Retrieve an image and its metadata from a camera.
/// </summary>
/// <returns>The image and its metadata</returns>
public ImageBody GetImage()
{
DateTime now = BlobStorageHelper.GetImageUtcTime();
string nowString = BlobStorageHelper.FormatImageUtcTime(now);
byte[] imageBytes = this.ImageSource.RequestImage();
if (imageBytes != null)
{
ByteString image = ByteString.CopyFrom(imageBytes);
ByteString smallImage = Camera.ShrinkJpegTo300x300(imageBytes);
ImageBody result = new ImageBody
{
CameraId = CameraId,
Time = nowString,
Type = "jpg",
Image = image,
SmallImageRGB = smallImage,
SkipMlProcessing = this.SkipMLProcessing
};
return(result);
}
else
{
return(null);
}
}
private ImageProcessor(BlobStorageHelper blobHelper, ModuleClient client, string processorType, ImageBody body)
{
this.moduleClient = client;
this.blobHelper = blobHelper;
this.processorType = processorType;
this.body = body;
}
/// <summary>
/// Performs the main processing.
/// </summary>
private static void MainLoop()
{
do
{
lock (s_twinUpdateLocker)
{
// Send images from each camera every ten seconds
foreach (Camera camera in Camera.s_cameras)
{
try
{
ImageBody body = camera.GetImage();
if (body != null)
{
Console.WriteLine($"Sending a {body.Image.Length} byte image from {body.CameraId} at {body.Time}");
s_grpcClient.UploadImage(body);
}
}
catch (Exception ex)
{
Console.WriteLine($"Error sending grpc message to VideoProcessor module: {ex.Message}");
}
}
}
Task.Delay(10000).Wait();
} while (s_mainLoopRunning);
}
public void SendImageMsgTest()
{
var mediaId = m_appClient.UploadFile(AppClient.FileTypeImage, ImageName, ImagePath);
var body = new ImageBody(mediaId);
var msg = new Message(ToUsers, Message.MessageTypeImage, body);
m_appClient.SendMsg(msg);
}
public override Task <ImageReply> SubmitImage(ImageBody request, ServerCallContext context)
{
// Start the callback task before responding
Task task = new Task(() => callback(request));
task.Start();
return(Task.FromResult(new ImageReply {
Error = string.Empty
}));
}
/// <summary>
///
/// </summary>
/// <param name="image"></param>
/// <returns>Null on error</returns>
public List <ImageFeature> Process(ByteString image)
{
bool succeeded = false;
try
{
List <ImageFeature> result = new List <ImageFeature>();
EnsureConnection();
ImageBody body = new ImageBody
{
Image = image
};
ImageReply reply = grpcClient.SubmitImage(body);
for (int i = 0; i < reply.Classes.Count; i++)
{
ImageFeature feature = new ImageFeature(reply.Classes[i], reply.Scores[i],
reply.Boxes[i].YMin, reply.Boxes[i].XMin, reply.Boxes[i].YMax, reply.Boxes[i].XMax);
result.Add(feature);
}
succeeded = true;
return(result);
}
catch (Grpc.Core.RpcException ex)
{
if (ex.StatusCode == StatusCode.Unavailable)
{
Console.WriteLine($"Error sending grpc message: Grpc server for FPGA is unavailable");
return(null);
}
else
{
Console.WriteLine($"Error sending grpc for FPGA message");
Console.WriteLine(ex);
return(null);
}
}
catch (Exception ex)
{
Console.WriteLine($"Error sending grpc for FPGA message");
Console.WriteLine(ex);
return(null);
}
finally
{
if (!succeeded)
{
Disconnect();
}
}
}
/// <summary>
/// Retrieve an image and its metadata from a camera.
/// </summary>
/// <returns>The image and its metadata</returns>
public ImageBody GetImage()
{
DateTime now = BlobStorageHelper.GetImageUtcTime();
string nowString = BlobStorageHelper.FormatImageUtcTime(now);
ImageBody result = this.ImageSource.RequestImages();
if (result != null)
{
result.CameraId = CameraId;
result.Time = nowString;
result.Type = "jpg";
}
return(result);
}
static void Main(string[] args)
{
CameraServerClient client = new CameraServerClient("PURL2//700");
Console.WriteLine("Press 's' to resend, any other key to exit...");
while (true)
{
try
{
ImageBody reply = ((IImageSource)client).RequestImages();
if (reply != null)
{
Console.WriteLine($"Received {reply.Image.Length} byte jpeg plus {reply.SmallImage.Length} 300x300 png");
using (MemoryStream stream = new MemoryStream(reply.SmallImage.ToByteArray()))
using (Bitmap bitmap = new Bitmap(stream))
{
bitmap.Save(@"..\..\..\smallImage.bmp", ImageFormat.Bmp);
bitmap.Save(@"..\..\..\smallImage.png", ImageFormat.Png);
}
File.WriteAllBytes(@"..\..\..\fullImage.jpeg", reply.Image.ToByteArray());
}
else
{
Console.WriteLine($"CameraServer returned null");
}
}
catch (Exception ex)
{
Console.WriteLine("Failed: " + ex.Message);
}
ConsoleKeyInfo key = Console.ReadKey();
if (key.KeyChar != 's')
{
((IImageSource)client).Disconnect();
return;
}
else
{
Console.Write("\b");
}
}
}
/// <summary>
/// Add a received image to the processing buffer
/// </summary>
/// <param name="body"></param>
public static void Add(ImageBody body)
{
lock (s_bufferLock)
{
s_entryDictionary.TryGetValue(body.CameraId, out BufferEntry found);
if (found == null)
{
BufferEntry entry = new BufferEntry()
{
CameraId = body.CameraId, Body = body
};
s_entryDictionary.Add(body.CameraId, entry);
s_entryList.Add(entry);
found = entry;
}
// Keep only the latest Body for processing, discarding any unprocessed older ones
found.Body = body;
}
}
private void ImageLoop()
{
while (!isDone)
{
try
{
ImageBody body = this.GetImage();
if (body != null)
{
Console.WriteLine($"Sending a {body.Image.Length} byte image from {body.CameraId} at {body.Time}");
this.grpcClient.UploadImage(body);
}
}
catch (Exception ex)
{
Console.WriteLine($"Error sending grpc message to VideoProcessor module: {ex.Message}");
}
Task.Delay(MillisecondsBetweenImages).Wait();
}
}
/// <summary>
/// Upload an image
/// </summary>
/// <param name="body"></param>
/// <returns>True for success</returns>
public bool UploadImage(ImageBody body)
{
lock (uploadLock)
{
bool succeeded = false;
try
{
EnsureConnection();
grpcClient.SubmitImage(body);
succeeded = true;
return(true);
}
catch (Grpc.Core.RpcException ex)
{
if (ex.StatusCode == StatusCode.Unavailable)
{
Console.WriteLine($"Error sending grpc message: Grpc server is unavailable");
return(false);
}
else
{
Console.WriteLine($"Error sending grpc message");
Console.WriteLine(ex);
return(false);
}
}
catch (Exception ex)
{
Console.WriteLine($"Error sending grpc message");
Console.WriteLine(ex);
return(false);
}
finally
{
if (!succeeded)
{
Disconnect();
}
}
}
}
/// <summary>
/// Return the next image to process
/// </summary>
/// <returns>May be null</returns>
public static ImageBody GetNext()
{
lock (s_bufferLock)
{
// Walk the list looking for a non-null ImageBody to process
for (int i = 0; i < s_entryList.Count; i++)
{
// s_entryListIndex is left pointing at the last camera whose image was processed
s_entryListIndex++;
s_entryListIndex = (int)(s_entryListIndex % s_entryList.Count);
if (s_entryList[s_entryListIndex].Body != null)
{
ImageBody result = s_entryList[s_entryListIndex].Body;
s_entryList[s_entryListIndex].Body = null;
return(result);
}
}
return(null);
}
}
/// <summary>
/// Retrieve images from a camera.
/// </summary>
/// <returns>The large and small images</returns>
ImageBody IImageSource.RequestImages()
{
// If we're not cycling then this.imageIndex is what we want. Otherwise
// we need to cycle.
int simulatedImageIndex = this.imageIndex;
if (cycleId != null)
{
List <int> selectedCycle = cycles[cycleId.Value];
simulatedImageIndex = selectedCycle[this.imageIndex];
this.imageIndex++;
this.imageIndex = this.imageIndex % selectedCycle.Count;
}
DateTime now = BlobStorageHelper.GetImageUtcTime();
string nowString = BlobStorageHelper.FormatImageUtcTime(now);
string filename = simulatedImageNames[simulatedImageIndex];
string smallImageFilename = smallSimulatedImageNames[simulatedImageIndex];
if (simulatedImageIndex >= simulatedImageNames.Length)
{
simulatedImageIndex = 0;
}
byte[] content = File.ReadAllBytes(filename);
byte[] smallContent = File.ReadAllBytes(smallImageFilename);
Google.Protobuf.ByteString image = Google.Protobuf.ByteString.CopyFrom(content);
Google.Protobuf.ByteString smallImage = Google.Protobuf.ByteString.CopyFrom(smallContent);
ImageBody body = new ImageBody
{
Time = nowString,
Type = "jpg",
Image = image,
SmallImage = smallImage
};
return(body);
}
/// <summary>
/// 接收图片消息
/// </summary>
/// <param name="imgBody"></param>
private void RecieveImageMessage(ImageBody imgBody)
{
/*
* [{"type":"img","url":"https://a1.easemob.com/easemob-demo/chatdemoui/chatfiles/cd6f8050-81f7-11e5-a16a-05187e341cb0",
* "secret":"zW-AWoH3EeWmJevV5n4Fpkxnnu3e5okMLIhENE0QHaZbvqg5",
* "filename":"原生+自定义.jpg",
* "thumb":"https://a1.easemob.com/easemob-demo/chatdemoui/chatfiles/cd6f8050-81f7-11e5-a16a-05187e341cb0",
* "thumb_secret":"","size":{"width":952,"height":671}}]
*/
string url = imgBody.url;
string secret = imgBody.secret;
string filename = imgBody.filename;
string thumb = imgBody.thumb;
string thumb_secret = imgBody.thumb_secret;
int width = 0;
int height = 0;
byte[] bytes = null;
if (!string.IsNullOrEmpty(thumb))
{
width = imgBody.size.width;
height = imgBody.size.height;
bytes = SdkUtils.DownloadThumbnail(thumb, secret, conn.TokenData.access_token);
}
if (bytes == null || bytes.Length == 0)
{
}
else
{
//edChat.AppendImageBytes(bytes);
}
// edChat.AppendNewLine(2);
chatBrower.AppendLeftImage(toUserName, thumb);
}
/// <summary>
/// Wake up every second and do all the work
/// </summary>
private static void ProcessingLoop()
{
while (true)
{
lock (s_twinUpdateLocker)
{
ImageBody found = InputBuffer.GetNext();
while (found != null)
{
try
{
ImageProcessor.Process(s_blobHelper, s_moduleClient, s_currentModel, found);
}
catch (Exception ex)
{
Console.WriteLine($"Error from ImageProcessor.Process: {ex.Message}");
}
found = InputBuffer.GetNext();
}
}
Task.Delay(1000).Wait();
}
}
/// <summary>
/// Process the image. The image is given to the model ML and if a
/// valid recognition is found then the image is uploaded to storage.
/// Processing times are tracked.
/// </summary>
/// <param name="body">
/// An ImageBody object containing the image from the camera.
/// </param>
public static void Process(BlobStorageHelper blobHelper, ModuleClient client, IProcessImage imageProc, ImageBody body)
{
ImageProcessor proc = new ImageProcessor(blobHelper, client, imageProc.ProcessorType, body);
// Perform and measure elapsed time for the ML model work
DateTime startTime = DateTime.Now;
proc.features = imageProc.Process(body.SmallImage);
proc.recognitionDuration = DateTime.Now - startTime;
// Loop to the next recognition task without waiting for the report to process
if (proc.features != null)
{
Task reportTask = new Task(() => proc.Report());
reportTask.Start();
}
}
private static void OnImageReceived(ImageBody body)
{
InputBuffer.Add(body);
}
/// <summary>
/// Process the image. The image is given to the model ML and if a
/// valid recognition is found then the image is uploaded to storage.
/// Processing times are tracked.
/// </summary>
/// <param name="body">
/// An ImageBody object containing the image from the camera.
/// </param>
public static void Process(BlobStorageHelper blobHelper, ModuleClient client, IProcessImage imageProc, ImageBody body)
{
ImageProcessor proc = new ImageProcessor(blobHelper, client, imageProc.ProcessorType, body);
if (!body.SkipMlProcessing)
{
// Perform and measure elapsed time for the ML model work
DateTime startTime = DateTime.Now;
proc.features = imageProc.Process(body.SmallImageRGB);
proc.recognitionDuration = DateTime.Now - startTime;
// Loop to the next recognition task without waiting for the report to process
if (proc.features != null)
{
Task reportTask = new Task(() => proc.Report());
reportTask.Start();
}
}
else
{
// Don't process ML or send any messages; just upload to BLOB
try
{
proc.UploadToBlob();
}
catch (Exception ex)
{
Console.WriteLine($" Failed to upload image to BLOB store: {ex.Message}");
}
}
}
/// <summary>
/// 发送文件
/// </summary>
/// <param name="toUser">发给谁</param>
/// <param name="postedfile">已经上传的文件返回信息</param>
public void SendFile(string toUser, PostedFileResp postedfile)
{
Jid to = new Jid(BuildJid(toUser));
Jid from = new Jid(BuildJid(UserName));
BodyBase[] bodies = new BodyBase[postedfile.entities.Length];
// 构建发送文件的 message 消息
for (int i = 0; i < postedfile.entities.Length; i++)
{
PostFileEntity entity = postedfile.entities[i];
// 文件类型 img audio
string otype = SdkUtils.GetFileType(entity.filename);
// 文件的url
string ourl = postedfile.uri + "/" + entity.uuid;
string osecret = entity.share_secret;
string ofilename = entity.filename;
/*
* 传图片
* ReceivedData:
* <message xmlns='jabber:client' from='easemob-demo#[email protected]/webim'
* to='easemob-demo#[email protected]' id='124420481838219668' type='chat'>
* <body>
* {"from":"weigang75","to":"march3","bodies":
* [{"type":"img","url":"https://a1.easemob.com/easemob-demo/chatdemoui/chatfiles/cd6f8050-81f7-11e5-a16a-05187e341cb0",
* "secret":"zW-AWoH3EeWmJevV5n4Fpkxnnu3e5okMLIhENE0QHaZbvqg5",
* "filename":"原生+自定义.jpg",
* "thumb":"https://a1.easemob.com/easemob-demo/chatdemoui/chatfiles/cd6f8050-81f7-11e5-a16a-05187e341cb0",
* "thumb_secret":"","size":{"width":952,"height":671}}],"ext":{}}</body></message>
*
* 传语音
* ReceivedData:
* <message xmlns='jabber:client' from='easemob-demo#[email protected]/webim'
* to='easemob-demo#[email protected]' id='124421298246910356' type='chat'>
* <body>
* {"from":"weigang75","to":"march3","bodies":
* [{"type":"audio",
* "url":"https://a1.easemob.com/easemob-demo/chatdemoui/chatfiles/3ec2bb50-81f8-11e5-8e7c-1fa6315dec2d",
* "secret":"PsK7WoH4EeWwmIkeyVsexnkK-Rmqu2X_N2qqK9FQYmUkko8W",
* "filename":"环信通讯 - 副本.mp3",
* "length":3}],"ext":{}}</body></message>
*
*/
// 如果是文件，需要多一些字段 thumb、thumb_secret、size
if ("img".Equals(otype))
{
bodies[i] = new ImageBody
{
type = otype,
url = ourl,
secret = osecret,
filename = ofilename,
thumb = ourl,
thumb_secret = "",
size = new ImageSize
{
width = entity.imageSize.Width,
height = entity.imageSize.Height
}
};
}
else if ("audio".Equals(otype))
{
bodies[i] = new AudioBody
{
type = otype,
url = ourl,
secret = osecret,
filename = ofilename
};
}
else
{
bodies[i] = new FileBody
{
type = otype,
url = ourl,
secret = osecret,
filename = ofilename
};
}
}
MsgData data = new MsgData()
{
from = UserName, to = toUser, bodies = bodies, ext = new { }
};
WSMessage msgNode = new WSMessage(to, from);
msgNode.Type = MessageType.chat;
msgNode.SetBodyData(data);
Send(msgNode);
}
