private SpeechAppBody RobotAction(RbHeader _rbh, JObject _appInfo, JObject _appParams)
{
var speechAppBody = new SpeechAppBody();
var actionClient = new HwsRobotBehaviorApi.Person.Action.Client(_appInfo["SqlConnectionString"].ToString());
var actionResult = actionClient.Get(
_rbh.SourceDeviceId,
_rbh.SourceDeviceId,
_appParams["PersonId"].ToString(),
_appParams["PersonReply"].ToString(),
_appParams["PersonTalk"].ToString(),
null,
"晴れ", // OpenWeatherMap API等を利用して天気情報を取得してください.
false
);
if (actionResult.apiResult.IsSuccessStatusCode)
{
var luisService = new LuisService(_rbh.SourceDeviceId, _appInfo);
speechAppBody.IsNoReaction = actionResult.appBody.IsNoReaction;
if (_appParams["PersonReply"].ToString() != "" && speechAppBody.IsNoReaction)
{
speechAppBody.Behavior = luisService.CreateRobotBehavior(_appParams["PersonTalk"].ToString());
speechAppBody.Behavior.NaturalTalkText = TextOverflow(speechAppBody.Behavior.NaturalTalkText);
}
else
{
speechAppBody.Behavior = luisService.CreateRobotBehaviorDirectSpeech(actionResult.appBody.RobotTalk, 108, 165);
}
}
return(speechAppBody);
}
/// <summary>
/// Get application routing information.
/// </summary>
/// <returns>RbAppRouterCache</returns>
private RbAppRouterCache GetAppRoutingInfo(RbHeader rbh)
{
AppRouter ar = null;
bool ar_action = true;
string cachekey = rbh.AppId + "_" + rbh.AppProcessingId;
RbAppRouterCache rbapprc = null;
if (rbAppRouterCacheDic.ContainsKey(cachekey))
{
rbapprc = (RbAppRouterCache)rbAppRouterCacheDic[cachekey];
if (rbapprc.CacheExpiredDatetime >= DateTime.Now)
{
ar_action = false;
}
}
if (ar_action)
{
ar = new AppRouter(rbh.AppId, rbh.AppProcessingId, sqlConnectionString, rbCacheExpiredTimeSec);
rbapprc = ar.GetAppRouting();
if (rbapprc != null)
{
lock (thisLock)
{
rbAppRouterCacheDic[cachekey] = rbapprc;
}
}
else
{
throw new ApplicationException("Error ** GetAppRouting() returns Null Object");
}
}
return(rbapprc);
}
private RbMessage DeleteFace(RbHeader rbh, JObject appInfo, JObject jo_input)
{
var storageAccount = (appInfo["StorageAccount"] ?? "").ToString();
var storageKey = (appInfo["StorageKey"] ?? "").ToString();
var storageContainer = (appInfo["StorageContainer"] ?? "").ToString();
var faceApiKey = (appInfo["FaceApiKey"] ?? "").ToString();
AppBody appbody = new AppBody();
appbody.PersonId = (jo_input["PersonId"] ?? "").ToString();
appbody.PersonGroupId = (jo_input["PersonGroupId"] ?? "").ToString();
var facePerson = new HwsMicrosoftFaceApi.Person.Client(faceApiKey);
var personDeleteResult = facePerson.Delete(appbody.PersonGroupId, appbody.PersonId);
if (personDeleteResult.apiResult.IsSuccessStatusCode)
{
var personGroup = new HwsMicrosoftFaceApi.PersonGroup.Client(faceApiKey);
personGroup.Train(appbody.PersonGroupId);
PersonInfoModel.Find(appbody.PersonId).Delete();
}
else
{
appbody.Code = ERROR_CODE;
}
return(GetRbMessage(rbh, appbody));
}
/// <summary>
/// Get application master information.
/// </summary>
/// <returns>RbAppMasterCache</returns>
private RbAppMasterCache GetAppMasterInfo(RbHeader rbh)
{
AppMaster am = null;
bool am_action = true;
RbAppMasterCache rbappmc = null;
if (rbAppMasterCacheDic.ContainsKey(rbh.AppId))
{
rbappmc = (RbAppMasterCache)rbAppMasterCacheDic[rbh.AppId];
if (rbappmc.CacheExpiredDatetime >= DateTime.Now)
{
am_action = false;
}
}
if (am_action)
{
am = new AppMaster(rbh.AppId, rbEncPassPhrase, sqlConnectionString, rbCacheExpiredTimeSec);
rbappmc = am.GetAppMaster();
if (rbappmc != null)
{
lock (thisLock)
{
rbAppMasterCacheDic[rbh.AppId] = rbappmc;
}
}
else
{
throw new ApplicationException("Error ** GetAppMaster() returns Null Object");
}
}
return(rbappmc);
}
/// <summary>
/// Call Apps (Azure Functions or Custom App) with HTTP
/// </summary>
/// <param name="rbh"></param>
/// <param name="rbBodyString"></param>
/// <param name="partitionId"></param>
/// <returns>Task<JArray></returns>
private async Task <JArray> CallAppsWithHttp(RbHeader rbh, string rbBodyString, string partitionId)
{
// Get App Master Info
RbAppMasterCache rbappmc = GetAppMasterInfo(rbh);
// Get App Routing Info
RbAppRouterCache rbapprc = GetAppRoutingInfo(rbh);
JArray ja_messages = new JArray();
RbMessage message = new RbMessage();
message.RbHeader = rbh;
message.RbBody = (JObject)JsonConvert.DeserializeObject(rbBodyString);
var strMessage = (string)JsonConvert.SerializeObject(message);
// HTTP Client
var client = new HttpClient();
// Create request body
byte[] byteData = Encoding.UTF8.GetBytes(strMessage);
var content = new ByteArrayContent(byteData);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
// Call REST API
var response = await client.PostAsync(rbapprc.HttpUri, content);
if (response.IsSuccessStatusCode)
{
var responseText = await response.Content.ReadAsStringAsync();
try
{
message.RbBody = (JObject)JsonConvert.DeserializeObject(responseText);
}
catch
{
var badResponse = new BadResponse();
badResponse.StatusCode = "Parse Error";
badResponse.ReasonPhrase = "HTTP response is not regular JSON format";
message.RbBody = badResponse;
}
}
else
{
var badResponse = new BadResponse();
badResponse.StatusCode = response.StatusCode.ToString();
badResponse.ReasonPhrase = response.ReasonPhrase.ToString();
message.RbBody = badResponse;
}
message.RbHeader = rbh;
string json_message = JsonConvert.SerializeObject(message);
JObject jo = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo);
return(ja_messages);
}
private RbMessage GetRbMessage(RbHeader _rbh, AppBody _appbody)
{
RbMessage message = new RbMessage();
_rbh.MessageId = "Res" + _rbh.MessageId;
message.RbHeader = _rbh;
message.RbBody = _appbody;
return(message);
}
private JArray MakeProcessMessages(RbHeader rbh, object body)
{
var processMessages = new JArray();
var message = new RbMessage
{
RbHeader = rbh,
RbBody = body
};
var json_message = JsonConvert.SerializeObject(message);
var jo = JsonConvert.DeserializeObject <JObject>(json_message);
processMessages.Add(jo);
return(processMessages);
}
private JArrayString BuildMessages(JArray jArrayMessages, RbHeader rbh, object rbBody)
{
RbMessage rbMessage = new RbMessage();
rbMessage.RbHeader = rbh;
rbMessage.RbBody = rbBody;
var jsonRbMessage = JsonConvert.SerializeObject(rbMessage);
var joRbMessage = (JObject)JsonConvert.DeserializeObject(jsonRbMessage);
jArrayMessages.Add(joRbMessage);
JArrayString jArrayString = new JArrayString(jArrayMessages);
return(jArrayString);
}
/// <summary>
/// Call Apps Async (Azure Functions) with Queue Storage
/// </summary>
/// <param name="rbh"></param>
/// <param name="rbBodyString"></param>
/// <param name="partitionId"></param>
/// <returns>Task</returns>
private async Task CallAppsWithQueue(RbHeader rbh, string rbBodyString, string partitionId)
{
// Get App Master Info
RbAppMasterCache rbappmc = GetAppMasterInfo(rbh);
// Get App Routing Info
RbAppRouterCache rbapprc = GetAppRoutingInfo(rbh);
RbMessage message = new RbMessage();
message.RbHeader = rbh;
message.RbBody = (JObject)JsonConvert.DeserializeObject(rbBodyString);
string queueName = rbapprc.AppId + "-" + rbapprc.AppProcessingId;
var sender = new RbAppMessageToQueue(message, rbappmc.QueueStorageAccount, rbappmc.QueueStorageKey, queueName);
await sender.SendAsync();
}
JArray ProcessControlMessage(RbHeader rbh)
{
JArray ja_messages = new JArray();
try
{
RbAppMasterCache rbappmc = GetAppMasterInfo(rbh);
RbMessage message = new RbMessage();
message.RbHeader = rbh;
message.RbBody = JsonConvert.DeserializeObject <JObject>(rbappmc.AppInfoDevice);
string json_message = JsonConvert.SerializeObject(message);
JObject jo = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo);
}
catch (Exception ex)
{
RbTraceLog.WriteError("E004", ex.ToString());
ae = new ApplicationException("Error ** <<CONTROL Message Processing>> Exception occured during JSON processing on RBFX.AppMaster[AppInfoDevice]");
throw ae;
}
return(ja_messages);
}
private void callAppButton_Click(object sender, EventArgs e)
{
if (textBoxDllFilePath.Text == string.Empty)
{
MessageBox.Show("** Error ** DLL File Local Path must be set !!", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
if (textBoxDeviceId.Text == string.Empty)
{
MessageBox.Show("** Error ** Device ID must be set !!", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
if (checkBoxSkipAppRouter.Checked && textBoxClassName.Text == string.Empty)
{
MessageBox.Show("** Error ** Class Name must be set !!", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
// Save the TextBox content
saveTextBoxContent();
JObject jo_message = JsonConvert.DeserializeObject <JObject>(textBoxInput.Text);
// Check RbHeader in detail
RbHeaderBuilder hdBuilder = new RbHeaderBuilder(jo_message, textBoxDeviceId.Text);
RbHeader rbh = hdBuilder.ValidateJsonSchema();
// RbBody
JObject jo_temp = (JObject)jo_message[RbFormatType.RbBody];
string rbBodyString = JsonConvert.SerializeObject(jo_temp);
// App Master Cache (RBFX.AppMaster)
AppMaster am = new AppMaster(rbh.AppId, activeEncPassPhrase, activeSqlConnectionString, constTimeOutSec);
RbAppMasterCache rbappmc = am.GetAppMaster();
// App Router Cache (RBFX.AppRouting)
RbAppRouterCache rbapprc;
if (checkBoxSkipAppRouter.Checked)
{
rbapprc = new RbAppRouterCache();
rbapprc.AppId = rbh.AppId;
rbapprc.AppProcessingId = rbh.AppProcessingId;
rbapprc.ClassName = textBoxClassName.Text;
rbapprc.FileName = textBoxDllFilePath.Text;
}
else
{
AppRouter ar = new AppRouter(rbh.AppId, rbh.AppProcessingId, activeSqlConnectionString, constTimeOutSec);
rbapprc = ar.GetAppRouting();
}
// Load DLL
//Assembly assembly = null;
AppDomain appDomain = null;
IAppRouterDll routedAppDll = null;
try
{
//assembly = System.Reflection.Assembly.LoadFrom(textBoxDllFilePath.Text);
//routedAppDll = assembly.CreateInstance(rbapprc.ClassName) as IAppRouterDll;
string pid = Thread.CurrentThread.ManagedThreadId.ToString();
string appDomainName = "AppDomain_P" + pid;
string cachedDirectory = Path.GetDirectoryName(textBoxDllFilePath.Text);
string cachedFileName = Path.GetFileName(textBoxDllFilePath.Text);
string cachedFileNameWithoutExt = Path.GetFileNameWithoutExtension(textBoxDllFilePath.Text);
appDomain = createAppDomain(appDomainName, cachedDirectory);
routedAppDll = appDomain.CreateInstanceAndUnwrap(cachedFileNameWithoutExt, rbapprc.ClassName) as IAppRouterDll;
}
catch (Exception ex)
{
MessageBox.Show("** Application (DLL) Load Error ** Check File Path or Class Name \n" + ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
// Process Message
try
{
rbh.ProcessingStack = activeFileName;
JArrayString ja_messagesString = routedAppDll.ProcessMessage(rbappmc, rbapprc, rbh, rbBodyString);
JArray ja_messages = ja_messagesString.ConvertToJArray();
textBoxOutput.Text = JsonConvert.SerializeObject(ja_messages);
AppDomain.Unload(appDomain);
}
catch (Exception ex)
{
MessageBox.Show("** Error occured in Application (DLL) **\n" + ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
AppDomain.Unload(appDomain);
return;
}
}
private byte[] GetFaceImageData(JObject _appInfo, JObject _rbBody, string _personGroupId, RbHeader _rbh)
{
string base64Image = (_rbBody["Face"] ?? "").ToString();
if (base64Image == "" && _personGroupId == "")
{
return(null);
}
return(System.Convert.FromBase64String(base64Image));
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
var results = new JArray();
var appInfo = JsonConvert.DeserializeObject <JObject>(rbappmc.AppInfo);
var connectionString = (appInfo["SqlConnectionString"] ?? "").ToString();
PersonInfoModel.Connection(connectionString);
PersonFaceInfoModel.Connection(connectionString);
var rbBody = JsonConvert.DeserializeObject <JObject>(rbBodyString);
RbMessage rbMessage = null;
if (rbh.MessageId == "GetFaceInfo")
{
rbMessage = GetFaceInfo(rbh, appInfo, rbBody);
}
else if (rbh.MessageId == "RegisterFace")
{
rbMessage = ResisterPersonFace(rbh, appInfo, rbBody);
}
else if (rbh.MessageId == "DeleteFace")
{
rbMessage = DeleteFace(rbh, appInfo, rbBody);
}
results.Add(JsonConvert.DeserializeObject <JObject>(JsonConvert.SerializeObject(rbMessage)));
return(new JArrayString(results));
}
private RbMessage GetFaceInfo(RbHeader _rbh, JObject _appInfo, JObject _rbBody)
{
var storageAccount = (_appInfo["StorageAccount"] ?? "").ToString();
var storageKey = (_appInfo["StorageKey"] ?? "").ToString();
var storageContainer = (_appInfo["StorageContainerTemporaryThumbnail"] ?? "").ToString();
var faceApiKey = (_appInfo["FaceApiKey"] ?? "").ToString();
var appbody = new AppBody();
appbody.PersonGroupId = (_rbBody["PersonGroupId"] ?? "").ToString();
var buffer = GetFaceImageData(_appInfo, _rbBody, appbody.PersonGroupId, _rbh);
if (buffer == null)
{
return(new RbMessage());
}
var facialHairConfidence = (double)_appInfo["FacialHairConfidence"];
var fd = new Detection(FaceApiHttpClient, faceApiKey, facialHairConfidence);
var fdResult = fd.DetectFace(buffer);
appbody.PersonInfos = fdResult.PersonInfos;
if (fdResult.apiResult.IsSuccessStatusCode)
{
var faceDetectJson = JsonConvert.DeserializeObject <JArray>(fdResult.apiResult.Result);
var faceRectangle = faceDetectJson[0]["faceRectangle"];
appbody.EmotionBodies = new List <HwsFaceCores.Emotion>();
foreach (var p in appbody.PersonInfos)
{
appbody.EmotionBodies.Add(p.FaceAttributes.Emotion);
}
var fiResult = new HwsFaceCores.AppResult();
var fi = new Identification(FaceApiHttpClient, faceApiKey, facialHairConfidence);
fiResult = fi.IdentifyFace(buffer, appbody.PersonInfos, appbody.PersonGroupId);
appbody.PersonInfos = fiResult.PersonInfos;
if (fiResult.apiResult.IsSuccessStatusCode)
{
var personInfoQueryResult = CreateOrUpdatePersonInfo(appbody);
if (personInfoQueryResult.apiResult.IsSuccessStatusCode)
{
appbody = personInfoQueryResult.appBody;
foreach (var info in appbody.PersonInfos)
{
CreatePersonFaceInfo(appbody, info);
var faceFileName = info.PersonId;
}
}
else
{
_rbh.ProcessingStack += " " + personInfoQueryResult.apiResult.Message;
}
}
else
{
_rbh.ProcessingStack += " " + fiResult.apiResult.Message;
}
}
else
{
RbMessage noFaceDetectError = new RbMessage();
noFaceDetectError.RbBody = JsonConvert.DeserializeObject <JObject>("{\"Code\": 404, \"Message\": \"画像に人の顔が含まれていません.\", \"PersonInfos\": [], \"EmotionBodies\": []}");
_rbh.MessageId = "Res" + _rbh.MessageId;
noFaceDetectError.RbHeader = _rbh;
return(noFaceDetectError);
}
return(GetRbMessage(_rbh, appbody));
}
private async Task ProcessMessage(EventData eventData, long partitionKey, RbTraceLog rbTraceLog)
{
// Receive a message and system properties
string iothub_deviceId = (string)eventData.SystemProperties["iothub-connection-device-id"];
DateTime iothub_enqueuedTimeUtc = (DateTime)eventData.SystemProperties.EnqueuedTimeUtc;
string text_message = Encoding.UTF8.GetString(eventData.Body.Array);
// RbTrace (Table Storage)
if (rbTraceLevel == RbTraceType.Detail)
{
rbTraceLog.WriteLog($"Received a message. Partition({partitionKey}), DeviceId({iothub_deviceId}), Message:{text_message}");
}
// Loop of retry for reconfiguration on SQL Database
int loopCounter = 0;
while (true)
{
JObject jo_message = null;
// Routing switch
bool devRouting = false;
bool appRouting = false;
try
{
// Check RbHeader
if (text_message.IndexOf(RbFormatType.RbHeader) < 0)
{
rbTraceLog.WriteLog(RbExceptionMessage.RbHeaderNotFound
+ $" Partition({partitionKey}), DeviceId({iothub_deviceId}), Message:{text_message}");
return;
}
// Check RbHeader simplly
jo_message = JsonConvert.DeserializeObject <JObject>(text_message);
var jo_rbh = (JObject)jo_message[RbFormatType.RbHeader];
var v_rbhRoutingType = jo_rbh[RbHeaderElement.RoutingType];
if (v_rbhRoutingType == null)
{
rbTraceLog.WriteError("W001", "** Message skipped because RoutingType is null **", jo_message);
return;
}
string s_rbhRoutingType = (string)v_rbhRoutingType;
if (s_rbhRoutingType == RbRoutingType.LOG || s_rbhRoutingType == string.Empty)
{
// RoutingType == LOG -> only using IoT Hub with Stream Analytics
return;
}
// Check RbHeader in detail
RbHeaderBuilder hdBuilder = new RbHeaderBuilder(jo_message, iothub_deviceId);
RbHeader rbh = null;
try
{
rbh = hdBuilder.ValidateJsonSchema();
}
catch (Exception ex)
{
rbTraceLog.WriteError("W002", "** Message skipped because of bad RbHeader **", ex);
return;
}
// Check StorageQueueSendEnabled property in RbHeader
prevStorageQueueSendEnabled = storageQueueSendEnabled;
string messageStorageQueueSendEnabled = null;
if (storageQueueSendEnabled != "true")
{
try
{
messageStorageQueueSendEnabled = (string)jo_rbh[typeStorageQueueSendEnabled];
}
catch
{
messageStorageQueueSendEnabled = null;
}
if (messageStorageQueueSendEnabled == "true")
{
storageQueueSendEnabled = messageStorageQueueSendEnabled;
}
}
// Check RoutingType (CALL, D2D, CONTROL)
if (rbh.RoutingType == RbRoutingType.CALL || rbh.RoutingType == RbRoutingType.CALL_ASYNC)
{
appRouting = true;
}
else if (rbh.RoutingType == RbRoutingType.D2D)
{
devRouting = true;
if (rbh.AppProcessingId != string.Empty)
{
appRouting = true;
}
}
else if (rbh.RoutingType == RbRoutingType.CONTROL)
{
devRouting = false;
appRouting = false;
}
else
{
rbTraceLog.WriteError("W003", "** Message skipped because of bad RoutingType **", jo_message);
return;
}
// Device Router builds RbHeader
DeviceRouter dr = null;
if (devRouting)
{
dr = new DeviceRouter(rbh, sqlConnectionString);
rbh = dr.GetDeviceRouting();
string new_header = JsonConvert.SerializeObject(rbh);
jo_message[RbFormatType.RbHeader] = JsonConvert.DeserializeObject <JObject>(new_header);
}
else
{
rbh.TargetDeviceId = rbh.SourceDeviceId;
rbh.TargetType = RbTargetType.Device;
}
// Application Routing
JArray ja_messages = null;
if (appRouting)
{
// Application Call Logic
JObject jo_temp;
string rbBodyString;
try
{
jo_temp = (JObject)jo_message[RbFormatType.RbBody];
rbBodyString = JsonConvert.SerializeObject(jo_temp);
}
catch (Exception ex)
{
rbTraceLog.WriteError("E001", $"** RbBody is not regular JSON format ** {ex.ToString()}", jo_message);
return;
}
try
{
if (rbh.RoutingType == RbRoutingType.CALL_ASYNC)
{
await CallAppsWithQueue(rbh, rbBodyString, partitionKey.ToString());
}
else
{
ja_messages = await CallAppsWithHttp(rbh, rbBodyString, partitionKey.ToString());
}
}
catch (Exception ex)
{
rbTraceLog.WriteError("E002", $"** Error occured in CallApps ** {ex.ToString()}", jo_message);
return;
}
}
else
{
ja_messages = new JArray();
ja_messages.Add(jo_message);
}
// Send C2D Message
if (rbh.RoutingType == RbRoutingType.CALL ||
rbh.RoutingType == RbRoutingType.D2D ||
rbh.RoutingType == RbRoutingType.CONTROL)
{
if (storageQueueSendEnabled == "true")
{
// Send C2D message to Queue storage
RbC2dMessageToQueue c2dsender = null;
c2dsender = new RbC2dMessageToQueue(ja_messages, storageQueueConnString, sqlConnectionString);
await c2dsender.SendToDeviceAsync();
}
else
{
// Send C2D message to IoT Hub
RbC2dMessageSender c2dsender = null;
c2dsender = new RbC2dMessageSender(ja_messages, iotHubConnectionString, sqlConnectionString);
await c2dsender.SendToDeviceAsync();
}
// StorageQueueSendEnabled property in RbHeader
storageQueueSendEnabled = prevStorageQueueSendEnabled;
}
// Get out of retry loop because of normal completion
break;
}
catch (Exception ex)
{
rbTraceLog.WriteError("E003", $"** Critical error occured ** {ex.ToString()}", jo_message);
bool continueLoop = false;
if (ex != null && ex is SqlException)
{
foreach (SqlError error in (ex as SqlException).Errors)
{
if (sqlErrorListForRetry.Contains(error.Number))
{
continueLoop = true;
break; // Exit foreach loop
}
}
if (continueLoop)
{
++loopCounter;
rbTraceLog.WriteLog($"Transaction retry has started. Count({loopCounter})");
if (loopCounter > maxLoopCounter)
{
break; // Get out of retry loop because counter reached max number
}
else
{
Thread.Sleep(sleepInterval);
}
}
else
{
break; // Get out of retry loop because of another sql error
}
}
else
{
throw;
}
}
}
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
var appInfo = JsonConvert.DeserializeObject <JObject>(rbappmc.AppInfo);
appInfo["DeviceId"] = rbh.SourceDeviceId;
var appParams = JsonConvert.DeserializeObject <JObject>(rbBodyString);
JArrayString message = null;
if (rbh.MessageId == "Speech")
{
var speechAppBody = new SpeechAppBody();
var personId = (appParams["PersonId"] ?? "").ToString();
var talk = (appParams["talk"] ?? "").ToString();
var luisService = new LuisService(rbh.SourceDeviceId, appInfo);
speechAppBody.Behavior = luisService.CreateRobotBehavior(talk);
speechAppBody.Behavior.NaturalTalkText = TextOverflow(speechAppBody.Behavior.NaturalTalkText);
if (personId != "")
{
var actionClient = new HwsRobotBehaviorApi.Person.Action.Client(appInfo["SqlConnectionString"].ToString());
foreach (var entity in speechAppBody.Behavior.LuisEntities)
{
actionClient.CreateTalkLog(rbh.SourceDeviceId, personId, speechAppBody.Behavior.NaturalTalkText, talk, entity);
}
}
message = new JArrayString(MakeProcessMessages(rbh, speechAppBody));
}
else if (rbh.MessageId == "RobotSpeech")
{
var speechAppBody = new SpeechAppBody();
var talk = (appParams["talk"] ?? "").ToString();
var luisService = new LuisService(rbh.SourceDeviceId, appInfo);
speechAppBody.Behavior = luisService.CreateRobotBehaviorDirectSpeech(talk);
speechAppBody.Behavior.NaturalTalkText = TextOverflow(speechAppBody.Behavior.NaturalTalkText);
message = new JArrayString(MakeProcessMessages(rbh, speechAppBody));
}
else if (rbh.MessageId == "GetRobotAction")
{
var speechAppBody = RobotAction(rbh, appInfo, appParams);
message = new JArrayString(MakeProcessMessages(rbh, speechAppBody));
}
return(message);
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
string sqlConnString = string.Empty;
string storageAccount = string.Empty;
string storageKey = string.Empty;
string storageContainer = string.Empty;
string faceApiEndpoint = string.Empty;
string faceApiKey = string.Empty;
bool success = true;
JArray ja_messages = new JArray();
RbMessage message = new RbMessage();
// RbAppLog
var appName = Path.GetFileNameWithoutExtension(this.GetType().Assembly.Location);
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
rbappmc.StorageAccount, rbappmc.StorageKey);
RbAppLog RbAppLog = new RbAppLog();
RbAppLog.Initialize(storageConnectionString, "RbAppLog", appName);
// Get appinfo
JObject jo_appInfo = (JObject)JsonConvert.DeserializeObject(rbappmc.AppInfo);
var p1 = jo_appInfo["SqlConnString"];
if (p1 != null)
{
sqlConnString = (string)p1;
}
var p2 = jo_appInfo["StorageAccount"];
if (p2 != null)
{
storageAccount = (string)p2;
}
var p3 = jo_appInfo["StorageKey"];
if (p3 != null)
{
storageKey = (string)p3;
}
var p4 = jo_appInfo["FaceStorageContainer"];
if (p4 != null)
{
storageContainer = (string)p4;
}
var p5 = jo_appInfo["FaceApiEndpoint"];
if (p5 != null)
{
faceApiEndpoint = (string)p5;
}
var p6 = jo_appInfo["FaceApiKey"];
if (p6 != null)
{
faceApiKey = (string)p6;
}
JObject jo_input = (JObject)JsonConvert.DeserializeObject(rbBodyString);
AppBodyWhenError appBodyWhenError = new AppBodyWhenError();
if (rbh.MessageId == "init")
{
AppBodyInit appBody = new AppBodyInit();
appBody.storageAccount = storageAccount;
appBody.storageContainer = storageContainer;
appBody.storageKey = storageKey;
message.RbBody = appBody;
}
else if (rbh.MessageId == "getFaceInfo")
{
AppBodyFaceInfo appBody = new AppBodyFaceInfo();
appBody.visitor = (string)jo_input["visitor"];
appBody.groupId = (string)jo_input["groupId"];
appBody.locationId = (string)jo_input["locationId"];
if (appBody.locationId == null || appBody.locationId == string.Empty)
{
appBody.locationId = "all";
}
// Get image data stream
string fileName = (string)jo_input["blobFileName"];
string deleteFile = (string)jo_input["deleteFile"];
BlobData blobData = new BlobData(storageAccount, storageKey, storageContainer);
try
{
byte[] buffer = blobData.GetStream(fileName);
// Delete File
if (deleteFile == "true")
{
blobData.Delete(fileName);
}
// Get Confidence threshold
double faceConfidence = 0;
try
{
faceConfidence = (double)jo_appInfo["FaceConfidence"];
}
catch
{
faceConfidence = 0.5;
}
double smileConfidence = 0;
try
{
smileConfidence = (double)jo_appInfo["SmileConfidence"];
}
catch
{
smileConfidence = 0.5;
}
double facialHairConfidence = 0;
try
{
facialHairConfidence = (double)jo_appInfo["FacialHairConfidence"];
}
catch
{
facialHairConfidence = 0.5;
}
// Call Face API (Detection)
FaceDetection fd = new FaceDetection(faceApiEndpoint, faceApiKey, smileConfidence, facialHairConfidence);
AppResult appResult1 = fd.DetectFace(buffer, appBody);
appBody = appResult1.appBody;
if (appResult1.apiResult.IsSuccessStatusCode)
{
// Call Face API (Identification)
FaceIdentification fi = new FaceIdentification(faceApiEndpoint, faceApiKey, faceConfidence);
AppResult appResult2 = fi.IdentifyFace(appBody);
appBody = appResult2.appBody;
if (appResult2.apiResult.IsSuccessStatusCode)
{
if (appBody.visitor_id != string.Empty)
{
PersonDbData personDbData = new PersonDbData(sqlConnString, rbh.AppId);
AppResult appResult3 = personDbData.GetInfo(appBody);
if (appResult3.apiResult.IsSuccessStatusCode)
{
appBody = appResult3.appBody;
}
else
{
success = false;
appBodyWhenError.error_message = appResult3.apiResult.Message;
RbAppLog.WriteError("E001", appResult3.apiResult.Message);
}
}
}
else
{
// Set success "true" even if identification doesn't succeed
rbh.ProcessingStack = "** Notice ** Face identification missed.";
RbAppLog.WriteError("E002", appResult2.apiResult.Message);
}
}
else
{
success = false;
appBodyWhenError.error_message = appResult1.apiResult.Message;
RbAppLog.WriteError("E003", appResult1.apiResult.Message);
}
if (success)
{
appBody.success = "true";
message.RbBody = appBody;
}
else
{
appBodyWhenError.success = "false";
message.RbBody = appBodyWhenError;
}
}
catch (Exception ex)
{
appBodyWhenError.error_message = ex.Message;
RbAppLog.WriteError("E004", ex.ToString());
appBodyWhenError.success = "false";
message.RbBody = appBodyWhenError;
}
}
else if (rbh.MessageId == "registerFace")
{
AppBodyRegResult appBody = new AppBodyRegResult();
appBody.visitor = (string)jo_input["visitor"];
appBody.groupId = (string)jo_input["groupId"];
appBody.locationId = (string)jo_input["locationId"];
if (appBody.locationId == null || appBody.locationId == string.Empty)
{
appBody.locationId = "all";
}
appBody.visitor_name = (string)jo_input["visitor_name"];
appBody.visitor_name_kana = (string)jo_input["visitor_name_kana"];
// Check Person Group existence
ApiResult apiResult1 = new ApiResult();
apiResult1.IsSuccessStatusCode = true;
PersonGroup personGroup = new PersonGroup(faceApiEndpoint, faceApiKey, appBody.groupId);
if (!personGroup.GetGroupExistence())
{
// Create Person Group
apiResult1 = personGroup.CreatePersonGroup();
}
if (apiResult1.IsSuccessStatusCode)
{
// Create Person
Person person = new Person(faceApiEndpoint, faceApiKey, appBody.groupId);
ApiResult apiResult2 = person.CreatePerson(appBody.visitor_name);
if (apiResult2.IsSuccessStatusCode)
{
// Extract PersonId
JObject joRsult2 = (JObject)JsonConvert.DeserializeObject(apiResult2.Result);
appBody.visitor_id = (string)joRsult2["personId"];
// Get image data stream
string fileName = (string)jo_input["blobFileName"];
string deleteFile = (string)jo_input["deleteFile"];
BlobData blobData = new BlobData(storageAccount, storageKey, storageContainer);
try
{
byte[] buffer = blobData.GetStream(fileName);
// Delete File
if (deleteFile == "true")
{
blobData.Delete(fileName);
}
// Add Face to the Person
ApiResult apiResult3 = person.AddPersonFace(appBody.visitor_id, buffer);
if (apiResult3.IsSuccessStatusCode)
{
appBody.success = "true";
personGroup.TrainPersonGroup();
AppBodyFaceInfo appBodyFaceInfo = new AppBodyFaceInfo();
appBodyFaceInfo.visitor = appBody.visitor;
appBodyFaceInfo.groupId = appBody.groupId;
appBodyFaceInfo.locationId = appBody.locationId;
appBodyFaceInfo.visitor_id = appBody.visitor_id;
appBodyFaceInfo.visitor_name = appBody.visitor_name;
appBodyFaceInfo.visitor_name_kana = appBody.visitor_name_kana;
appBodyFaceInfo.visit_count = "1";
PersonDbData personDbData = new PersonDbData(sqlConnString, rbh.AppId);
personDbData.InsertInfo(appBodyFaceInfo);
}
else
{
success = false;
appBodyWhenError.error_message = apiResult3.Message;
RbAppLog.WriteError("E005", apiResult3.Message);
}
}
catch (Exception ex)
{
if (ex.Message.Length > 100)
{
appBodyWhenError.error_message = ex.Message.Substring(0, 100);
}
else
{
appBodyWhenError.error_message = ex.Message;
}
appBodyWhenError.success = "false";
message.RbBody = appBodyWhenError;
RbAppLog.WriteError("E006", ex.ToString());
}
}
else
{
success = false;
appBodyWhenError.error_message = apiResult2.Message;
RbAppLog.WriteError("E007", apiResult2.Message);
}
}
else
{
success = false;
appBodyWhenError.error_message = apiResult1.Message;
RbAppLog.WriteError("E008", apiResult1.Message);
}
if (success)
{
appBody.success = "true";
message.RbBody = appBody;
}
else
{
appBodyWhenError.success = "false";
message.RbBody = appBodyWhenError;
}
}
message.RbHeader = rbh;
string json_message = JsonConvert.SerializeObject(message);
JObject jo = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo);
JArrayString jaString = new JArrayString(ja_messages);
return(jaString);
}
async Task IEventProcessor.ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
try
{
#region *** Handling unexpected exceptions ***
DateTime startDateTimeUtc = DateTime.UtcNow;
int messagecnt = 0;
bool sqlex_on = false;
foreach (EventData eventData in messages)
{
try
{
// Go forward read pointer
await context.CheckpointAsync();
}
catch (Exception ex)
{
// Handling the exception of Microsoft.ServiceBus.Messaging.LeaseLostException
// This exception is usually able to occur.
ApplicationException ae = new ApplicationException(checkpointExMessage, ex);
throw ae;
}
++messagecnt;
int retryCount = 0;
bool devRouting = false;
bool appRouting = false;
bool rbHeaderNotFound = false;
string sqlConnString = rbSqlConnectionString;
string iothub_deviceId = (string)eventData.SystemProperties["iothub-connection-device-id"];
DateTime iothub_enqueuedTimeUtc = (DateTime)eventData.SystemProperties["EnqueuedTimeUtc"];
string text_message = string.Empty;
// Retry loop logic for SQLDB reconfiguration exception
while (true)
{
text_message = Encoding.UTF8.GetString(eventData.GetBytes());
string text_message_100 = text_message.Substring(0, 100);
if (text_message_100.IndexOf(RbFormatType.RbHeader) < 0)
{
rbHeaderNotFound = true;
RbTraceLog.WriteLog(string.Format(RbExceptionMessage.RbHeaderNotFound + " Partition:{0}, Message:{1}, DeviceId:{2}",
context.Lease.PartitionId, text_message_100, iothub_deviceId));
}
if (rbTraceLevel == RbTraceType.Detail)
{
DateTime dt = iothub_enqueuedTimeUtc;
TimeSpan ts = DateTime.UtcNow - dt;
RbTraceLog.WriteLog(string.Format("RoboticsEventProcessor Message received. Delayed time:{0}, Partition:{1}, DeviceId:{2}, Message:{3}",
ts.ToString(), context.Lease.PartitionId, iothub_deviceId, text_message));
if (ts > new TimeSpan(0, 0, 5))
{
RbTraceLog.WriteLog(string.Format("** Delayed time is over 5 seconds !! ** DelayedTime:{0}, Partition:{1}, DeviceId:{2}, Message:{3}",
ts.ToString(), context.Lease.PartitionId, iothub_deviceId, text_message));
}
}
JObject jo_message = null;
if (!rbHeaderNotFound) // Skip invalid data
{
try
{
jo_message = JsonConvert.DeserializeObject <JObject>(text_message);
// Check RbHeader simplly
var jo_rbh = (JObject)jo_message[RbFormatType.RbHeader];
if (jo_rbh != null)
{
string jo_rbh_RoutingType = (string)jo_rbh[RbHeaderElement.RoutingType];
// Check RoutingType (LOG, null)
if (jo_rbh_RoutingType == null)
{
RbTraceLog.WriteError("W001", "** Message skipped because RoutingType is null **", jo_message);
goto LoopExitLabel;
}
else if (jo_rbh_RoutingType == RbRoutingType.LOG)
{
// RoutingType == LOG -> only using IoT Hub with Stream Analytics
goto LoopExitLabel;
}
}
// Check RbHeader in detail
RbHeaderBuilder hdBuilder = new RbHeaderBuilder(jo_message, iothub_deviceId);
RbHeader rbh = hdBuilder.ValidateJsonSchema();
// Check RoutingType (CALL, D2D, CONTROL)
if (rbh.RoutingType == RbRoutingType.CALL)
{
appRouting = true;
}
else if (rbh.RoutingType == RbRoutingType.D2D)
{
devRouting = true;
if (rbh.AppProcessingId != string.Empty)
{
appRouting = true;
}
}
else if (rbh.RoutingType == RbRoutingType.CONTROL)
{
devRouting = false;
appRouting = false;
}
else
{
RbTraceLog.WriteError("W002", "** Message skipped because of bad RoutingType **", jo_message);
goto LoopExitLabel;
}
// Device Router builds RbHeader
DeviceRouter dr = null;
if (devRouting)
{
dr = new DeviceRouter(rbh, sqlConnString);
rbh = dr.GetDeviceRouting();
string new_header = JsonConvert.SerializeObject(rbh);
jo_message[RbFormatType.RbHeader] = JsonConvert.DeserializeObject <JObject>(new_header);
}
else
{
rbh.TargetDeviceId = rbh.SourceDeviceId;
rbh.TargetType = RbTargetType.Device;
}
// Application Routing
JArray ja_messages = null;
if (appRouting)
{
// Application Call Logic
JObject jo_temp = (JObject)jo_message[RbFormatType.RbBody];
string rbBodyString = JsonConvert.SerializeObject(jo_temp);
ja_messages = CallApps(rbh, rbBodyString, context.Lease.PartitionId);
}
else if (rbh.RoutingType != RbRoutingType.CONTROL)
{
ja_messages = new JArray();
ja_messages.Add(jo_message);
}
// RoutingType="CONTROL" and AppProcessingId="ReqAppInfo"
if (rbh.RoutingType == RbRoutingType.CONTROL)
{
if (rbh.AppProcessingId == null)
{
RbTraceLog.WriteError("W003", "** Message skipped because AppProcessingId is null when CONTROL RoutingType **", jo_message);
goto LoopExitLabel;
}
else if (rbh.AppProcessingId == RbControlType.ReqAppInfo)
{
ja_messages = ProcessControlMessage(rbh);
}
else
{
RbTraceLog.WriteError("W004", "** Message skipped because of bad AppProcessingId when CONTROL RoutingType **", jo_message);
goto LoopExitLabel;
}
}
// Send C2D Message
if (rbh.RoutingType == RbRoutingType.CALL ||
rbh.RoutingType == RbRoutingType.D2D ||
rbh.RoutingType == RbRoutingType.CONTROL)
{
if (rbStorageQueueSendEnabled)
{
// Send C2D message to Queue storage
RbC2dMessageToQueue c2dsender = null;
c2dsender = new RbC2dMessageToQueue(ja_messages, rbStorageQueueConnString, sqlConnString);
c2dsender.SendToDevice();
}
else
{
// Send C2D message to IoT Hub
RbC2dMessageSender c2dsender = null;
c2dsender = new RbC2dMessageSender(ja_messages, rbIotHubConnString, sqlConnString);
c2dsender.SendToDevice();
}
}
// C2D Message Logging to Event Hub
if (rbC2dLogEnabled)
{
RbEventHubs rbEventHubs = new RbEventHubs(rbC2dLogEventHubConnString, rbC2dLogEventHubName);
foreach (JObject jo in ja_messages)
{
string str_message = JsonConvert.SerializeObject(jo);
rbEventHubs.SendMessage(str_message, iothub_deviceId);
}
}
}
catch (Exception ex)
{
sqlex_on = false;
if (ex != null && ex is SqlException) // "is" matches extended type as well
{
foreach (SqlError error in (ex as SqlException).Errors)
{
if (sqlErrorListForRetry.Contains(error.Number))
{
sqlex_on = true;
break; // Exit foreach loop
}
}
if (sqlex_on)
{
++retryCount;
if (retryCount > maxRetryCount)
{
sqlex_on = false;
RbTraceLog.WriteError("E001", ex.ToString(), jo_message);
}
else
{
RbTraceLog.WriteLog($"Transaction retry has started. Count({retryCount})");
Thread.Sleep(sleepTime);
}
}
else
{
RbTraceLog.WriteError("E001", ex.ToString(), jo_message);
}
}
else
{
RbTraceLog.WriteError("E001", ex.ToString(), jo_message);
}
}
}
if (!sqlex_on)
{
goto LoopExitLabel;
}
}
// Label - Loop exit
LoopExitLabel :;
if (rbTraceLevel == RbTraceType.Detail)
{
TimeSpan ts = DateTime.UtcNow - startDateTimeUtc;
RbTraceLog.WriteLog(string.Format("RoboticsEventProcessor Message processed. Duration:{0}, Partition:{1}, DeviceId{2}, Message:{3}",
ts.ToString(), context.Lease.PartitionId, iothub_deviceId, text_message));
}
}
#endregion *** Handling unexpected exceptions ***
}
catch (Exception ex)
{
if (ex.Message == checkpointExMessage)
{
RbTraceLog.WriteLog("** Retrying message processing because of CheckPointAsync error ** Info => "
+ ex.InnerException.ToString());
}
else
{
RbTraceLog.WriteError("E999", ex.ToString());
}
}
}
JArray CallApps(RbHeader rbh, string rbBodyString, string partitionId)
{
// Get App Master Info
RbAppMasterCache rbappmc = GetAppMasterInfo(rbh);
// Get App Routing Info
RbAppRouterCache rbapprc = GetAppRoutingInfo(rbh);
JArrayString ja_messagesString = null;
JArray ja_messages = null;
string dllFilePath = string.Empty;
IAppRouterDll routedAppDll = null;
Assembly assembly = null;
// Load DLL from BLOB
string baseDirectory = string.Empty;
string privateDllDirectory = string.Empty;
string cachedFileName = string.Empty;
string cachedFileNameWithoutExt = string.Empty;
if (rbapprc.DevMode == "True")
{
string devdir = rbapprc.DevLocalDir;
int pos = devdir.Length - 1;
if (devdir.Substring(pos, 1) == @"\")
{
dllFilePath = rbapprc.DevLocalDir + rbapprc.FileName;
}
else
{
dllFilePath = rbapprc.DevLocalDir + @"\" + rbapprc.FileName;
}
baseDirectory = Path.GetDirectoryName(dllFilePath);
privateDllDirectory = baseDirectory;
cachedFileName = Path.GetFileName(dllFilePath);
cachedFileNameWithoutExt = Path.GetFileNameWithoutExtension(dllFilePath);
}
else
{
CachedDllFileInfo cachedDllFileInfo = null;
lock (thisLock2)
{
cachedDllFileInfo = CopyBlobToLocalDir(rbappmc, rbapprc, partitionId);
}
baseDirectory = cachedDllFileInfo.BaseDirectory;
privateDllDirectory = cachedDllFileInfo.PrivateDllDirectory;
cachedFileName = Path.GetFileName(cachedDllFileInfo.PrivateDllFilePath);
cachedFileNameWithoutExt = Path.GetFileNameWithoutExtension(cachedDllFileInfo.PrivateDllFilePath);
}
////Static load without AppDomain
//assembly = System.Reflection.Assembly.LoadFrom(dllFilePath);
//routedAppDll = assembly.CreateInstance(rbapprc.ClassName) as IAppRouterDll;
//Dynamic load using AppDomain
try
{
string appDomainName = appDomanNameBase + partitionId;
AppDomain appDomain = null;
if (appDomainList.ContainsKey(partitionId))
{
appDomain = appDomainList[partitionId];
}
if (appDomain == null)
{
appDomain = CreateAppDomain(appDomainName, baseDirectory, privateDllDirectory);
lock (thisLock2)
{
appDomainList[partitionId] = appDomain;
}
}
routedAppDll = appDomain.CreateInstanceAndUnwrap(cachedFileNameWithoutExt, rbapprc.ClassName) as IAppRouterDll;
}
catch (Exception ex)
{
RbTraceLog.WriteError("E003", ex.ToString());
ae = new ApplicationException("Error ** Exception occured during creating AppDomain & Instance(App DLL)");
throw ae;
}
// ProcessMessage
try
{
rbh.ProcessingStack = rbapprc.FileName;
ja_messagesString = routedAppDll.ProcessMessage(rbappmc, rbapprc, rbh, rbBodyString);
ja_messages = ja_messagesString.ConvertToJArray();
}
catch (Exception ex)
{
RbTraceLog.WriteError("E002", ex.ToString());
ae = new ApplicationException("Error ** Exception occured in routed App DLL");
throw ae;
}
return(ja_messages);
}
private RbMessage ResisterPersonFace(RbHeader _rbh, JObject _appInfo, JObject _rbBody)
{
var faceApiKey = (_appInfo["FaceApiKey"] ?? "").ToString();
var facialHairConfidence = (double)_appInfo["FacialHairConfidence"];
var appbody = new AppBody();
appbody.PersonName = (_rbBody["PersonName"] ?? "").ToString();
appbody.PersonNameYomi = (_rbBody["PersonNameYomi"] ?? "").ToString();
appbody.PersonGroupId = (_rbBody["PersonGroupId"] ?? "").ToString();
appbody.PersonId = (_rbBody["PersonId"] ?? "").ToString();
var buffer = GetFaceImageData(_appInfo, _rbBody, appbody.PersonGroupId, _rbh);
var faceDetection = new Detection(FaceApiHttpClient, faceApiKey, 0.1);
var faceDetectResult = faceDetection.DetectFace(buffer);
if (faceDetectResult.apiResult.IsSuccessStatusCode)
{
appbody.PersonInfos = faceDetectResult.PersonInfos;
var faceDetectJson = JsonConvert.DeserializeObject <JArray>(faceDetectResult.apiResult.Result);
var faceRectangle = faceDetectJson[0]["faceRectangle"];
if (appbody.PersonId == "")
{
var fiResult = new HwsFaceCores.AppResult();
var fi = new Identification(FaceApiHttpClient, faceApiKey, facialHairConfidence);
fiResult = fi.IdentifyFace(buffer, appbody.PersonInfos, appbody.PersonGroupId);
appbody.PersonInfos = fiResult.PersonInfos;
if (fiResult.apiResult.IsSuccessStatusCode)
{
if (appbody.PersonInfos.Count > 0)
{
RbMessage faceDuplicatedError = new RbMessage();
faceDuplicatedError.RbBody = JsonConvert.DeserializeObject <JObject>("{\"Code\": 406, \"Message\": \"顔情報は登録済みです.\", \"PersonInfos\": [], \"EmotionBodies\": []}");
_rbh.MessageId = "Res" + _rbh.MessageId;
faceDuplicatedError.RbHeader = _rbh;
return(faceDuplicatedError);
}
}
var facePerson = new HwsMicrosoftFaceApi.Person.Client(faceApiKey);
var personCreateResult = facePerson.Create(appbody.PersonGroupId, appbody.PersonName);
if (personCreateResult.apiResult.IsSuccessStatusCode)
{
var result = (JObject)JsonConvert.DeserializeObject(personCreateResult.apiResult.Result);
appbody.PersonId = (result["personId"] ?? "").ToString();
appbody = ResisterFace(_appInfo, appbody, buffer, faceRectangle);
}
}
else
{
appbody = ResisterFace(_appInfo, appbody, buffer, faceRectangle);
}
}
else
{
RbMessage noFaceDetectError = new RbMessage();
noFaceDetectError.RbBody = JsonConvert.DeserializeObject <JObject>("{\"Code\": 404, \"Message\": \"画像に人の顔が含まれていません.\", \"PersonInfos\": [], \"EmotionBodies\": []}");
_rbh.MessageId = "Res" + _rbh.MessageId;
noFaceDetectError.RbHeader = _rbh;
return(noFaceDetectError);
}
return(GetRbMessage(_rbh, appbody));
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
// RbAppLog
var appName = Path.GetFileNameWithoutExtension(this.GetType().Assembly.Location);
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
rbappmc.StorageAccount, rbappmc.StorageKey);
RbAppLog RbAppLog = new RbAppLog();
RbAppLog.Initialize(storageConnectionString, "RbAppLog", appName);
// Request Message
var requestBody = new ReqBody();
var jo_reqBody = (JObject)JsonConvert.DeserializeObject(rbBodyString);
requestBody.visitor = (string)jo_reqBody["visitor"];
requestBody.visitor_id = (string)jo_reqBody["visitor_id"];
// Response Message
JArray jArrayMessages = new JArray();
// rbh.MessageId == "finish"
if (rbh.MessageId == typeFinish)
{
var resBody = new ResBody();
resBody.visitor = requestBody.visitor;
resBody.visitor_id = requestBody.visitor_id;
resBody.type = typeFinish;
return(BuildMessages(jArrayMessages, rbh, resBody));
}
// rbh.MessageId == "init" or "talkXxxx"
requestBody.talkByMe = (string)jo_reqBody["talkByMe"];
// AppMaster
JObject jo_appInfo = (JObject)JsonConvert.DeserializeObject(rbappmc.AppInfo);
sqlConnString = (string)jo_appInfo["SqlConnString"];
try
{
rinnaApiEndpoint = (string)jo_appInfo["RinnaApiEndpoint"];
rinnaKey = (string)jo_appInfo["RinnaKey"];
rinnaId = (string)jo_appInfo["RinnaId"];
}
catch
{
rinnaApiEndpoint = string.Empty;
rinnaKey = string.Empty;
rinnaId = string.Empty;
}
luisApiEndpoint = (string)jo_appInfo["LuisApiEndpoint"];
luisPgmApiEndpoint = (string)jo_appInfo["LuisPgmApiEndpoint"];
luisKey = (string)jo_appInfo["LuisKey"];
luisAppId = (string)jo_appInfo["LuisAppId"];
// Edit Response Message
var responseBody = new ResBodyTalk();
responseBody.visitor = requestBody.visitor;
responseBody.visitor_id = requestBody.visitor_id;
responseBody.talkByAi = new List <TalkMessage>();
IntentState intentState = new IntentState();
string intent = string.Empty;
// Call LUIS API
if (rbh.MessageId == typeInit)
{
try
{
intentState = GetIntent(requestBody.talkByMe);
}
catch (Exception ex)
{
ResBodyWhenError resBodyWhenError = new ResBodyWhenError();
resBodyWhenError.success = "false";
resBodyWhenError.error_message = ex.Message;
RbAppLog.WriteError("E001", ex.ToString());
return(BuildMessages(jArrayMessages, rbh, resBodyWhenError));
}
intentState.processState = stateBegin;
intent = (string)intentState.topScoringIntent["intent"];
// set Conversation State
ConversationState convState = new ConversationState(sqlConnString, rbh.SourceDeviceId, rbh.AppId);
string conversationStateString = JsonConvert.SerializeObject(intentState);
ApiResult apiResult = convState.SetState(conversationStateString);
if (!apiResult.IsSuccessStatusCode)
{
ResBodyWhenError resBodyWhenError = new ResBodyWhenError();
resBodyWhenError.success = "false";
resBodyWhenError.error_message = "** Error ** Conversation State Save failed !! : " + apiResult.Message;
RbAppLog.WriteError("E002", apiResult.Message);
return(BuildMessages(jArrayMessages, rbh, resBodyWhenError));
}
}
else
{
// get Conversation State
ConversationState convState = new ConversationState(sqlConnString, rbh.SourceDeviceId, rbh.AppId);
AppResult appResult = convState.GetState();
if (appResult.apiResult.IsSuccessStatusCode)
{
var joIntentState = (JObject)JsonConvert.DeserializeObject(appResult.conversationStateString);
intentState.processState = (string)joIntentState["processState"];
intentState.query = (string)joIntentState["query"];
intentState.topScoringIntent = (JObject)joIntentState["topScoringIntent"];
try
{
intentState.actionList = (JArray)joIntentState["actionList"];
}
catch
{
intentState.actionList = null;
}
}
else
{
ResBodyWhenError resBodyWhenError = new ResBodyWhenError();
resBodyWhenError.success = "false";
resBodyWhenError.error_message = "** Error ** Conversation State Get failed !! : " + appResult.apiResult.Message;
RbAppLog.WriteError("E003", appResult.apiResult.Message);
return(BuildMessages(jArrayMessages, rbh, resBodyWhenError));
}
}
// Continue talk session
bool boolFinishTalk = false;
if (rbh.MessageId.Substring(0, 4) == "talk")
{
intent = rbh.MessageId;
// Check intent of finishing talk
boolFinishTalk = CheckIntentOfFinishTalk(requestBody.talkByMe);
}
// Finish Talk
if (boolFinishTalk)
{
responseBody.visitor = requestBody.visitor;
responseBody.visitor_id = requestBody.visitor_id;
responseBody.type = typeFinishTalk;
var talkMessage = new TalkMessage();
talkMessage.SayText = "会話を終了します。再度、ご用件をお話しください。";
responseBody.talkByAi.Add(talkMessage);
return(BuildMessages(jArrayMessages, rbh, responseBody));
}
// Call Rinna API
if (intent == intentTalkRinna || intent == intentNone)
{
if (rinnaApiEndpoint != string.Empty)
{
ApiResult apiResult = TalkRinna(rinnaKey, rinnaId, requestBody.talkByMe);
if (apiResult.IsSuccessStatusCode)
{
responseBody.success = "true";
responseBody.type = typeTalkRinna;
JObject jo_result = (JObject)JsonConvert.DeserializeObject(apiResult.Result);
var ja_responses = (JArray)jo_result["Responses"];
foreach (var jo_content in ja_responses)
{
var talkMessage = new TalkMessage();
string text = (string)jo_content["Content"]["Text"];
// Replace Emoji
talkMessage.SayText = ReplaceTalkMessage(text);
responseBody.talkByAi.Add(talkMessage);
}
}
else
{
ResBodyWhenError resBodyWhenError = new ResBodyWhenError();
resBodyWhenError.success = "false";
resBodyWhenError.error_message = "** Error ** Rinna API failed !! : " + apiResult.Message;
RbAppLog.WriteError("E004", apiResult.Message);
return(BuildMessages(jArrayMessages, rbh, resBodyWhenError));
}
}
else
{
responseBody.success = "true";
responseBody.type = typeFinishTalk;
var talkMessage = new TalkMessage();
talkMessage.SayText = ReplaceTalkMessage("上手く聞き取ることが出来ませんでした。再度、ご用件をお話しください。");
responseBody.talkByAi.Add(talkMessage);
}
}
// Various Talk Service defined by LUIS web service
else if (intent.Length >= 4 && intent.Substring(0, 4) == "talk")
{
var talkMessage = new TalkMessage();
string sayText = string.Empty;
if (intentState.processState == stateBegin)
{
sayText = GetNextTalkMessage(ref intentState);
}
else
{
sayText = GetNextTalkMessageWithUpdate(ref intentState, requestBody.talkByMe);
}
if (sayText == string.Empty)
{
responseBody.type = typeFinishTalk;
sayText = CompleteTalkMessage(ref intentState);
}
else
{
responseBody.type = intent;
}
responseBody.success = "true";
talkMessage.SayText = sayText;
responseBody.talkByAi.Add(talkMessage);
// set Conversation State
ConversationState convState = new ConversationState(sqlConnString, rbh.SourceDeviceId, rbh.AppId);
string conversationStateString = JsonConvert.SerializeObject(intentState);
ApiResult apiResult = convState.SetState(conversationStateString);
if (!apiResult.IsSuccessStatusCode)
{
ResBodyWhenError resBodyWhenError = new ResBodyWhenError();
resBodyWhenError.success = "false";
resBodyWhenError.error_message = "** Error ** Conversation State Save(2) failed !! : " + apiResult.Message;
RbAppLog.WriteError("E005", apiResult.Message);
return(BuildMessages(jArrayMessages, rbh, resBodyWhenError));
}
}
else
{
responseBody.success = "true";
responseBody.type = intent;
}
return(BuildMessages(jArrayMessages, rbh, responseBody));
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
// Prepare variables for storage account.
string storageAccount = string.Empty;
string storageKey = string.Empty;
string storageContainer = string.Empty;
string translatorAccountKey = string.Empty;
string languageCode = "en"; //set english as the default
string visionApiKey = string.Empty;
string visionApiEndpoint = string.Empty;
string facesOption = string.Empty;
JArray ja_messages = new JArray();
RbMessage message = new RbMessage();
// RbAppLog
var appName = Path.GetFileNameWithoutExtension(this.GetType().Assembly.Location);
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
rbappmc.StorageAccount, rbappmc.StorageKey);
RbAppLog RbAppLog = new RbAppLog();
RbAppLog.Initialize(storageConnectionString, "RbAppLog", appName);
// Set cognitive service account key
JObject jo_appInfo = (JObject)JsonConvert.DeserializeObject(rbappmc.AppInfo);
JObject jo_input = (JObject)JsonConvert.DeserializeObject(rbBodyString);
var p1 = jo_appInfo["StorageAccount"];
if (p1 != null)
{
storageAccount = (string)p1;
}
var p2 = jo_appInfo["StorageKey"];
if (p2 != null)
{
storageKey = (string)p2;
}
var p3 = jo_appInfo["VisionStorageContainer"];
if (p3 != null)
{
storageContainer = (string)p3;
}
var p4 = jo_appInfo["VisionTranslatorToLang"];
if (p4 != null)
{
languageCode = (string)p4;
}
var p5 = jo_appInfo["VisionTranslatorApiKey"];
if (p5 != null)
{
translatorAccountKey = (string)p5;
}
var p6 = jo_appInfo["VisionApiEndpoint"];
if (p6 != null)
{
visionApiEndpoint = (string)p6;
}
var p7 = jo_appInfo["VisionApiKey"];
if (p7 != null)
{
visionApiKey = (string)p7;
}
if (rbh.MessageId == "init")
{
InitBody initBody = new InitBody();
initBody.storageAccount = storageAccount;
initBody.storageContainer = storageContainer;
initBody.storageKey = storageKey;
message.RbBody = initBody;
}
else if (rbh.MessageId == "analyze")
{
// Prepare response body
AnalyzeBody analyzeBody = new AnalyzeBody();
if ((string)jo_input["visitor"] != null)
{
if ((string)jo_input["visitor"] != "")
{
analyzeBody.visitor = (string)jo_input["visitor"];
}
}
if ((string)jo_input["visitor_id"] != null)
{
if ((string)jo_input["visitor_id"] != "")
{
analyzeBody.visitor = (string)jo_input["visitor_id"];
}
}
// Set http client
var client = new HttpClient();
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", visionApiKey);
var visionDescribeUrl = $"{visionApiEndpoint}/analyze?visualFeatures=Tags,Description,Faces,Adult";
try
{
// Prepare target file data
BlobData blobData = new BlobData(storageAccount, storageKey, storageContainer);
string fileName = (string)jo_input["blobFileName"];
byte[] buffer = blobData.GetStream(fileName);
var content = new ByteArrayContent(buffer);
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
// Get result which describe the image.
DateTime dt1 = DateTime.Now;
var visionDescribeResult = client.PostAsync(visionDescribeUrl, content);
visionDescribeResult.Wait();
DateTime dt2 = DateTime.Now;
TimeSpan ts = dt2 - dt1;
if (visionDescribeResult.Result.IsSuccessStatusCode)
{
// If success, convert to the response body form and translate messages.
analyzeBody.success = "true";
var responseBody = visionDescribeResult.Result.Content.ReadAsStringAsync();
var resultBody = JObject.Parse(responseBody.Result.ToString());
// combine text and tags to make one sentence
string descriptionText = (string)resultBody["description"]["captions"][0]["text"];
string baseTextTags = joinTags((JArray)resultBody["tags"]);
// translate text.
DateTime dt3 = DateTime.Now;
string translatedDescription = translateText(descriptionText, translatorAccountKey, languageCode);
string translatedTags = translateText(baseTextTags, translatorAccountKey, languageCode);
DateTime dt4 = DateTime.Now;
TimeSpan ts2 = dt4 - dt3;
// reform the translated result.
var resultTextTags = convertTextTags(translatedTags, (JArray)resultBody["tags"]);
// set results
analyzeBody.Description = (string)resultBody["description"]["captions"][0]["text"];
analyzeBody.IsAdultContent = ((string)resultBody["adult"]["isAdultContent"]).ToLower();
analyzeBody.IsRacyContent = ((string)resultBody["adult"]["isRacyContent"]).ToLower();
analyzeBody.Faces = convertFaces((JArray)resultBody["faces"]);
analyzeBody.Description_jp = translatedDescription;
analyzeBody.Tags = resultTextTags;
message.RbBody = analyzeBody;
}
else
{
// If failure, convert to the response body form and translate messages.
AppBodyWhenError appBodyWhenError = new AppBodyWhenError();
appBodyWhenError.success = "false";
appBodyWhenError.error_message = visionDescribeResult.Result.ToString();
RbAppLog.WriteError("E001", appBodyWhenError.error_message);
message.RbBody = appBodyWhenError;
}
// if deleteFile value is true, delete data from blob container.
if ((string)jo_input["deleteFile"] == "true")
{
blobData.Delete(fileName);
}
}
// catch (ApplicationException ex)
catch (Exception ex)
{
AppBodyWhenError appBodyWhenError = new AppBodyWhenError();
appBodyWhenError.success = "false";
appBodyWhenError.error_message = ex.Message;
RbAppLog.WriteError("E002", ex.ToString());
message.RbBody = appBodyWhenError;
}
}
message.RbHeader = rbh;
string json_message = JsonConvert.SerializeObject(message);
JObject jo = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo);
JArrayString jaString = new JArrayString(ja_messages);
return(jaString);
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
string translatorAccountKey = string.Empty;
// RbAppLog
var appName = Path.GetFileNameWithoutExtension(this.GetType().Assembly.Location);
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
rbappmc.StorageAccount, rbappmc.StorageKey);
RbAppLog RbAppLog = new RbAppLog();
RbAppLog.Initialize(storageConnectionString, "RbAppLog", appName);
// Request Message
var requestBody = new ReqBody();
var jo_reqBody = (JObject)JsonConvert.DeserializeObject(rbBodyString);
try
{
requestBody.visitor = (string)jo_reqBody["visitor"];
requestBody.visitor_id = (string)jo_reqBody["visitor_id"];
}
catch
{
requestBody.visitor = string.Empty;
requestBody.visitor_id = string.Empty;
}
requestBody.text = (string)jo_reqBody["text"];
requestBody.tolang = (string)jo_reqBody["tolang"];
// Response Message
JArray ja_messages = new JArray();
RbMessage message = new RbMessage();
string json_message = string.Empty;
var responseBody = new ResBody();
responseBody.translated_text = string.Empty;
var responseBodyWhenError = new ResBodyWhenError();
if (requestBody.tolang == null || requestBody.tolang == string.Empty) //in case no language is selected.
{
requestBody.tolang = languageCode;
}
// Set cognitive service account key
JObject jo_appInfo = (JObject)JsonConvert.DeserializeObject(rbappmc.AppInfo);
var p1 = jo_appInfo["TranslatorApiKey"];
if (p1 != null)
{
if ((string)p1 != "")
{
translatorAccountKey = (string)p1;
}
}
if (translatorAccountKey == string.Empty)
{
// Send Error to device
responseBodyWhenError.error_message = "[TranslatorApiKey] not found in RBFX.AppMaster !!";
responseBodyWhenError.success = "false";
RbAppLog.WriteError("E001", responseBodyWhenError.error_message);
message.RbHeader = rbh;
message.RbBody = responseBodyWhenError;
json_message = JsonConvert.SerializeObject(message);
JObject jo1 = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo1);
return(new JArrayString(ja_messages));
}
ApiResult apiResult = TranslateText(requestBody.text, translatorAccountKey, requestBody.tolang);
// Respond to device
message = new RbMessage();
message.RbHeader = rbh;
if (apiResult.IsSuccessStatusCode)
{
responseBody.success = "true";
responseBody.visitor = requestBody.visitor;
responseBody.visitor_id = requestBody.visitor_id;
responseBody.translated_text = apiResult.Result;
message.RbBody = responseBody;
}
else
{
responseBodyWhenError.success = "false";
responseBodyWhenError.error_message = apiResult.Message;
message.RbBody = responseBodyWhenError;
RbAppLog.WriteError("E002", apiResult.Message);
}
json_message = JsonConvert.SerializeObject(message);
JObject jo = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo);
JArrayString jaString = new JArrayString(ja_messages);
return(jaString);
}
public JArrayString ProcessMessage(RbAppMasterCache rbappmc, RbAppRouterCache rbapprc, RbHeader rbh, string rbBodyString)
{
JArray ja_messages = new JArray();
AppBody appbody = new AppBody();
appbody.Hello = "Hello World !!!!!!";
RbMessage message = new RbMessage();
message.RbHeader = rbh;
message.RbBody = appbody;
string json_message = JsonConvert.SerializeObject(message);
JObject jo = (JObject)JsonConvert.DeserializeObject(json_message);
ja_messages.Add(jo);
JArrayString jaString = new JArrayString(ja_messages);
return(jaString);
}
