private AssistRequest CreateNewRequest()
{
// initial request is the config this then gets followed by all out audio
var converseRequest = new AssistRequest();
var audioIn = new AudioInConfig()
{
Encoding = AudioInConfig.Types.Encoding.Linear16,
SampleRateHertz = Const.SampleRateHz
};
var audioOut = new AudioOutConfig()
{
Encoding = AudioOutConfig.Types.Encoding.Linear16,
SampleRateHertz = Const.SampleRateHz,
VolumePercentage = 75
};
DialogStateIn state = new DialogStateIn()
{
ConversationState = ByteString.Empty, LanguageCode = "en-US"
};
DeviceConfig device = new DeviceConfig()
{
DeviceModelId = "assistanttest-187121-myTest", DeviceId = "mylaptop"
};
converseRequest.Config = new AssistConfig()
{
AudioInConfig = audioIn, AudioOutConfig = audioOut, DialogStateIn = state, DeviceConfig = device
};
return(converseRequest);
}
public static void StartRecording(bool htmlResponse)
{
//_audioCapture = new AudioCapture(16000, AudioChannel.Mono, AudioSampleType.S16Le);
IsRecording = true;
var ar = new AssistRequest
{
Config = new AssistConfig
{
AudioInConfig = new AudioInConfig
{
SampleRateHertz = 16000,
Encoding = AudioInConfig.Types.Encoding.Linear16
},
ScreenOutConfig = new ScreenOutConfig
{
ScreenMode = htmlResponse
? ScreenOutConfig.Types.ScreenMode.Playing
: ScreenOutConfig.Types.ScreenMode.Unspecified
}
}
};
SapService.SendData(ar.ToByteArray());
_audioCapture.Prepare();
Record();
}
public void Stop()
{
var ar = new AssistRequest
{
Config = new AssistConfig
{
AudioInConfig = new AudioInConfig
{
SampleRateHertz = 16000,
Encoding = AudioInConfig.Types.Encoding.Flac
}
}
};
SapService.SendData(ar.ToByteArray());
IsPlaying = false;
IsPrepared = false;
if (_player.State != PlayerState.Playing)
{
return;
}
_player.Stop();
_source.Cancel();
BufferFileStream.Close();
//if (File.Exists(BufferFilePath)) File.Delete(BufferFilePath);
_player.PlaybackCompleted -= Player_PlaybackCompleted;
OnPlaybackStopped(EventArgs.Empty);
MainPage.SetButtonImage("listen_blue.png");
MainPage.SetActionButtonIsEnabled(true);
}
private async Task WriteAudioIn(byte[] buffer)
{
OnDebug?.Invoke("Write Audio " + buffer.Length, true);
var request = new AssistRequest()
{
AudioIn = ByteString.CopyFrom(buffer)
};
await _requestStream.WriteAsync(request);
}
public async Task AssistCommand(CommandContext ctx, [RemainingText] string input)
{
var secrets = new ClientSecrets
{
ClientId = ClientId,
ClientSecret = ClientSecret
};
// check if already authorized, else authorize
if (UserCredentials == null)
{
UserCredentials = await Authorize(secrets);
}
// create the request
var request = new AssistRequest();
request.Config = CreateConfig("en-US");
request.Config.TextQuery = input;
var channel = GrpcChannel.ForAddress(Endpoint, new GrpcChannelOptions
{
Credentials = UserCredentials.ToChannelCredentials()
});
var client = new EmbeddedAssistant.EmbeddedAssistantClient(channel);
var assist = client.Assist();
// send the gRPC request to google assistant
await assist.RequestStream.WriteAsync(request);
// first element is the response without audio
await assist.ResponseStream.MoveNext(CancellationToken.None);
var response = assist.ResponseStream.Current;
// read audio stream
var audioStream = new MemoryStream();
while (await assist.ResponseStream.MoveNext(CancellationToken.None))
{
var current = assist.ResponseStream.Current;
current.AudioOut.AudioData.WriteTo(audioStream);
}
audioStream.Position = 0;
await ctx.RespondWithFileAsync(audioStream, file_name : "response.ogg", content : response.DialogStateOut?.SupplementalDisplayText);
audioStream.Dispose();
}
private AssistRequest GetConfigRequest(string textQuery)
{
var audioInConfig = new AudioInConfig()
{
Encoding = AudioInConfig.Types.Encoding.Linear16,
SampleRateHertz = assistantConf.audioSampleRate
};
var audioOutConfig = new AudioOutConfig()
{
Encoding = AudioOutConfig.Types.Encoding.Linear16,
SampleRateHertz = assistantConf.audioSampleRate,
VolumePercentage = assistantConf.volumePercent,
};
var dialogStateInConfig = new DialogStateIn()
{
// We set the us local as default
LanguageCode = assistantConf.languageCode,
ConversationState = currentConversationState
};
var deviceConfig = new DeviceConfig()
{
DeviceModelId = deviceModel.deviceModelId,
DeviceId = device.id
};
var assistConfig = new AssistConfig()
{
AudioInConfig = audioInConfig,
AudioOutConfig = audioOutConfig,
DeviceConfig = deviceConfig,
DialogStateIn = dialogStateInConfig,
TextQuery = textQuery
};
AssistRequest assistRequest = new AssistRequest()
{
Config = assistConfig
};
return(assistRequest);
}
private static void Record()
{
_source = new CancellationTokenSource();
_token = _source.Token;
Task.Run(() =>
{
while (!_token.IsCancellationRequested)
{
var ar2 = new AssistRequest
{
AudioIn = ByteString.CopyFrom(_audioCapture.Read(_bufferSize))
};
SapService.SendData(ar2.ToByteArray());
}
_audioCapture.Flush();
_audioCapture.Unprepare();
}, _token);
}
private AssistRequest CreateNewRequest()
{
// initial request is the config this then gets followed by all out audio
var converseRequest = new AssistRequest();
var audioIn = new AudioInConfig()
{
Encoding = AudioInConfig.Types.Encoding.Linear16,
SampleRateHertz = Const.SampleRateHz
};
var audioOut = new AudioOutConfig()
{
Encoding = AudioOutConfig.Types.Encoding.Linear16,
SampleRateHertz = Const.SampleRateHz,
VolumePercentage = 75
};
var screenOut = new ScreenOutConfig()
{
ScreenMode = ScreenMode.Playing
};
DialogStateIn state = new DialogStateIn()
{
ConversationState = ByteString.Empty, LanguageCode = this.settings.LanguageCode
};
DeviceConfig device = new DeviceConfig()
{
DeviceModelId = this.settings.DeviceModelId, DeviceId = this.settings.DeviceId
};
converseRequest.Config = new AssistConfig()
{
AudioInConfig = audioIn, AudioOutConfig = audioOut, DialogStateIn = state, DeviceConfig = device, ScreenOutConfig = screenOut
};
return(converseRequest);
}
public async Task Assist(CommandContext ctx, int @in = 48000, int @out = 16000, int inChan = 1, int outChan = 1, string raw = null)
{
if (!ctx.Services.GetService <IProvideAudioState>().SpeechFromUser.ContainsKey(ctx.User.Id))
{
return;
}
var buff = ctx.Services.GetService <IProvideAudioState>().SpeechFromUser[ctx.User.Id].ToArray();
AssistantConfig.AudioInConfig = new AudioInConfig()
{
Encoding = AudioInConfig.Types.Encoding.Linear16,
SampleRateHertz = 16000
};
if (raw != "raw")
{
buff = AudioConverter.Resample(buff, @in, @out, inChan, outChan);
}
else
{
AssistantConfig.AudioInConfig.SampleRateHertz = AudioFormat.Default.SampleRate;
}
var token = new GoogleOAuth(KarvisConfiguration).GetTokenForUser(ctx.User.Id);
if (string.IsNullOrWhiteSpace(token))
{
await ctx.RespondAsync("Sorry, you have not signed up.");
return;
}
if (!UserEmbeddedAssistantClients.ContainsKey(ctx.User.Id))
{
var channelCredentials = ChannelCredentials.Create(new SslCredentials(), GoogleGrpcCredentials.FromAccessToken(token));
UserEmbeddedAssistantClients[ctx.User.Id] = new EmbeddedAssistant.EmbeddedAssistantClient(new Channel("embeddedassistant.googleapis.com", 443, channelCredentials));
}
using (var call = UserEmbeddedAssistantClients[ctx.User.Id].Assist())
{
try
{
var configRequest = new AssistRequest()
{
AudioIn = ByteString.Empty,
Config = AssistantConfig
};
ctx.LogInfo($"GoogleAssistant: Sending config message: Audio IsEmpty: {configRequest.AudioIn.IsEmpty}, Request Size: {configRequest.CalculateSize()}");
await call.RequestStream.WriteAsync(configRequest);
await SendAudioInChunks(call, ctx, buff);
await call.RequestStream.CompleteAsync();
ctx.LogInfo($"GoogleAssistant: Completing request and awaiting response.");
var audioOut = new List <byte>();
await call.ResponseStream.ForEachAsync((response) => ProcessVoiceAssistResponse(response, ctx, ref audioOut));
if (audioOut.Any())
{
var voiceConnection = await ctx.EnsureVoiceConnection();
if (voiceConnection == null || (voiceConnection.Channel != ctx.Member?.VoiceState?.Channel))
{
throw new InvalidOperationException($"I'm not connected to your voice channel, so I can't speak.");
}
var audio = AudioConverter.Resample(audioOut.ToArray(),
16000, 48000,
1, 2);
voiceConnection.SendSpeaking();
voiceConnection.GetTransmitStream().Write(audio, 0, audio.Length);
voiceConnection.GetTransmitStream().Flush();
voiceConnection.SendSpeaking(false);
}
try
{
var status = call.GetStatus();
ctx.LogInfo($"GoogleAssistant: Final Status: {status.StatusCode}, Detail: {status.Detail}.");
}
catch (InvalidOperationException ex)
{
}
}
catch (RpcException ex)
{
ctx.LogError($"GoogleAssistant: Exception: {ex.StatusCode}, {ex.Status.Detail}, {ex.Message}.");
if (ex.StatusCode == StatusCode.Unauthenticated)
{
if (!UserGoogleAuthRetries.ContainsKey(ctx.User.Id))
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
if (UserGoogleAuthRetries[ctx.User.Id] < 3)
{
UserGoogleAuthRetries[ctx.User.Id]++;
await ReAuth(ctx);
await Assist(ctx, @in, @out, inChan, outChan, raw);
}
else
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
}
}
catch (Exception ex)
{
await ctx.RespondAsync($"Sorry, {ctx.User.Username}, I can't google. \n\n``{ex.Message}``");
}
}
}
public async Task TextAssist(CommandContext ctx, [RemainingText] string query)
{
if (ctx.User?.Id == null)
{
return;
}
var token = new GoogleOAuth(KarvisConfiguration).GetTokenForUser(ctx.User.Id);
if (string.IsNullOrWhiteSpace(token))
{
await ctx.RespondAsync("Sorry, you have not signed up.");
return;
}
if (!UserEmbeddedAssistantClients.ContainsKey(ctx.User.Id))
{
var channelCredentials = ChannelCredentials.Create(new SslCredentials(), GoogleGrpcCredentials.FromAccessToken(token));
UserEmbeddedAssistantClients[ctx.User.Id] = new EmbeddedAssistant.EmbeddedAssistantClient(new Channel("embeddedassistant.googleapis.com", 443, channelCredentials));
}
using (var call = UserEmbeddedAssistantClients[ctx.User.Id].Assist())
{
try
{
AssistantConfig.AudioInConfig = null;
AssistantConfig.TextQuery = query;
var request = new AssistRequest()
{
AudioIn = ByteString.Empty,
Config = AssistantConfig
};
ctx.LogInfo($"GoogleAssistant: Sending config message: Audio IsEmpty: {request.AudioIn.IsEmpty}, Request Size: {request.CalculateSize()}");
await call.RequestStream.WriteAsync(request);
await call.RequestStream.CompleteAsync();
ctx.LogInfo($"GoogleAssistant: Completing request and awaiting response.");
var audioOut = new List <byte>();
await call.ResponseStream.ForEachAsync((response) => ProcessTextAssistResponse(response, ctx, ref audioOut));
try
{
var status = call.GetStatus();
ctx.LogInfo($"GoogleAssistant: Final Status: {status.StatusCode}, Detail: {status.Detail}.");
}
catch (InvalidOperationException ex)
{
}
if (audioOut.Any())
{
var voiceConnection = await ctx.EnsureVoiceConnection();
if (voiceConnection == null || (voiceConnection.Channel != ctx.Member?.VoiceState?.Channel))
{
throw new InvalidOperationException($"I'm not connected to your voice channel, so I can't speak.");
}
var audio = AudioConverter.Resample(audioOut.ToArray(),
16000, 48000,
1, 2);
voiceConnection.SendSpeaking();
voiceConnection.GetTransmitStream().Write(audio, 0, audio.Length);
voiceConnection.GetTransmitStream().Flush();
voiceConnection.SendSpeaking(false);
}
}
catch (RpcException ex)
{
ctx.LogError($"GoogleAssistant: Exception: {ex.StatusCode}, Detail: {ex.Status.Detail}, Message: {ex.Message}.");
if (ex.StatusCode == StatusCode.Unauthenticated)
{
if (!UserGoogleAuthRetries.ContainsKey(ctx.User.Id))
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
if (UserGoogleAuthRetries[ctx.User.Id] < 3)
{
UserGoogleAuthRetries[ctx.User.Id]++;
await ReAuth(ctx);
await TextAssist(ctx, query);
}
else
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
}
ctx.LogError($"GoogleAssistant: Exception: {ex.StatusCode}, Detail: {ex.Status.Detail}, Message: {ex.Message}.");
}
catch (Exception ex)
{
await ctx.RespondAsync($"Sorry, {ctx.User.Username}, I can't google. \n\n``{ex.Message}``");
}
}
}
