public async Task Assist(CommandContext ctx, int @in = 48000, int @out = 16000, int inChan = 1, int outChan = 1, string raw = null)
{
if (!ctx.Services.GetService <IProvideAudioState>().SpeechFromUser.ContainsKey(ctx.User.Id))
{
return;
}
var buff = ctx.Services.GetService <IProvideAudioState>().SpeechFromUser[ctx.User.Id].ToArray();
AssistantConfig.AudioInConfig = new AudioInConfig()
{
Encoding = AudioInConfig.Types.Encoding.Linear16,
SampleRateHertz = 16000
};
if (raw != "raw")
{
buff = AudioConverter.Resample(buff, @in, @out, inChan, outChan);
}
else
{
AssistantConfig.AudioInConfig.SampleRateHertz = AudioFormat.Default.SampleRate;
}
var token = new GoogleOAuth(KarvisConfiguration).GetTokenForUser(ctx.User.Id);
if (string.IsNullOrWhiteSpace(token))
{
await ctx.RespondAsync("Sorry, you have not signed up.");
return;
}
if (!UserEmbeddedAssistantClients.ContainsKey(ctx.User.Id))
{
var channelCredentials = ChannelCredentials.Create(new SslCredentials(), GoogleGrpcCredentials.FromAccessToken(token));
UserEmbeddedAssistantClients[ctx.User.Id] = new EmbeddedAssistant.EmbeddedAssistantClient(new Channel("embeddedassistant.googleapis.com", 443, channelCredentials));
}
using (var call = UserEmbeddedAssistantClients[ctx.User.Id].Assist())
{
try
{
var configRequest = new AssistRequest()
{
AudioIn = ByteString.Empty,
Config = AssistantConfig
};
ctx.LogInfo($"GoogleAssistant: Sending config message: Audio IsEmpty: {configRequest.AudioIn.IsEmpty}, Request Size: {configRequest.CalculateSize()}");
await call.RequestStream.WriteAsync(configRequest);
await SendAudioInChunks(call, ctx, buff);
await call.RequestStream.CompleteAsync();
ctx.LogInfo($"GoogleAssistant: Completing request and awaiting response.");
var audioOut = new List <byte>();
await call.ResponseStream.ForEachAsync((response) => ProcessVoiceAssistResponse(response, ctx, ref audioOut));
if (audioOut.Any())
{
var voiceConnection = await ctx.EnsureVoiceConnection();
if (voiceConnection == null || (voiceConnection.Channel != ctx.Member?.VoiceState?.Channel))
{
throw new InvalidOperationException($"I'm not connected to your voice channel, so I can't speak.");
}
var audio = AudioConverter.Resample(audioOut.ToArray(),
16000, 48000,
1, 2);
voiceConnection.SendSpeaking();
voiceConnection.GetTransmitStream().Write(audio, 0, audio.Length);
voiceConnection.GetTransmitStream().Flush();
voiceConnection.SendSpeaking(false);
}
try
{
var status = call.GetStatus();
ctx.LogInfo($"GoogleAssistant: Final Status: {status.StatusCode}, Detail: {status.Detail}.");
}
catch (InvalidOperationException ex)
{
}
}
catch (RpcException ex)
{
ctx.LogError($"GoogleAssistant: Exception: {ex.StatusCode}, {ex.Status.Detail}, {ex.Message}.");
if (ex.StatusCode == StatusCode.Unauthenticated)
{
if (!UserGoogleAuthRetries.ContainsKey(ctx.User.Id))
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
if (UserGoogleAuthRetries[ctx.User.Id] < 3)
{
UserGoogleAuthRetries[ctx.User.Id]++;
await ReAuth(ctx);
await Assist(ctx, @in, @out, inChan, outChan, raw);
}
else
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
}
}
catch (Exception ex)
{
await ctx.RespondAsync($"Sorry, {ctx.User.Username}, I can't google. \n\n``{ex.Message}``");
}
}
}
public async Task TextAssist(CommandContext ctx, [RemainingText] string query)
{
if (ctx.User?.Id == null)
{
return;
}
var token = new GoogleOAuth(KarvisConfiguration).GetTokenForUser(ctx.User.Id);
if (string.IsNullOrWhiteSpace(token))
{
await ctx.RespondAsync("Sorry, you have not signed up.");
return;
}
if (!UserEmbeddedAssistantClients.ContainsKey(ctx.User.Id))
{
var channelCredentials = ChannelCredentials.Create(new SslCredentials(), GoogleGrpcCredentials.FromAccessToken(token));
UserEmbeddedAssistantClients[ctx.User.Id] = new EmbeddedAssistant.EmbeddedAssistantClient(new Channel("embeddedassistant.googleapis.com", 443, channelCredentials));
}
using (var call = UserEmbeddedAssistantClients[ctx.User.Id].Assist())
{
try
{
AssistantConfig.AudioInConfig = null;
AssistantConfig.TextQuery = query;
var request = new AssistRequest()
{
AudioIn = ByteString.Empty,
Config = AssistantConfig
};
ctx.LogInfo($"GoogleAssistant: Sending config message: Audio IsEmpty: {request.AudioIn.IsEmpty}, Request Size: {request.CalculateSize()}");
await call.RequestStream.WriteAsync(request);
await call.RequestStream.CompleteAsync();
ctx.LogInfo($"GoogleAssistant: Completing request and awaiting response.");
var audioOut = new List <byte>();
await call.ResponseStream.ForEachAsync((response) => ProcessTextAssistResponse(response, ctx, ref audioOut));
try
{
var status = call.GetStatus();
ctx.LogInfo($"GoogleAssistant: Final Status: {status.StatusCode}, Detail: {status.Detail}.");
}
catch (InvalidOperationException ex)
{
}
if (audioOut.Any())
{
var voiceConnection = await ctx.EnsureVoiceConnection();
if (voiceConnection == null || (voiceConnection.Channel != ctx.Member?.VoiceState?.Channel))
{
throw new InvalidOperationException($"I'm not connected to your voice channel, so I can't speak.");
}
var audio = AudioConverter.Resample(audioOut.ToArray(),
16000, 48000,
1, 2);
voiceConnection.SendSpeaking();
voiceConnection.GetTransmitStream().Write(audio, 0, audio.Length);
voiceConnection.GetTransmitStream().Flush();
voiceConnection.SendSpeaking(false);
}
}
catch (RpcException ex)
{
ctx.LogError($"GoogleAssistant: Exception: {ex.StatusCode}, Detail: {ex.Status.Detail}, Message: {ex.Message}.");
if (ex.StatusCode == StatusCode.Unauthenticated)
{
if (!UserGoogleAuthRetries.ContainsKey(ctx.User.Id))
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
if (UserGoogleAuthRetries[ctx.User.Id] < 3)
{
UserGoogleAuthRetries[ctx.User.Id]++;
await ReAuth(ctx);
await TextAssist(ctx, query);
}
else
{
UserGoogleAuthRetries[ctx.User.Id] = 0;
}
}
ctx.LogError($"GoogleAssistant: Exception: {ex.StatusCode}, Detail: {ex.Status.Detail}, Message: {ex.Message}.");
}
catch (Exception ex)
{
await ctx.RespondAsync($"Sorry, {ctx.User.Username}, I can't google. \n\n``{ex.Message}``");
}
}
}