private async Task <bool> ImageValidatorAsync(PromptValidatorContext <IList <Attachment> > promptContext, CancellationToken cancellationToken)
{
var userData = await BotStateAccessor.GetAsync(promptContext.Context, () => new UserData(), cancellationToken);
bool result = false;
var userInput = promptContext.Recognized.Value;
if (userInput != null && userInput.Any())
{
var remoteFileUrl = userInput[0].ContentUrl;
// We make the call to the CustomVisionService
var imageResult = await CustomVisionService.AnalyzeAsync(remoteFileUrl);
if (imageResult != null)
{
userData.ImageBoundingBox = imageResult.BoundingBox;
userData.CroppedImage = await ImageUtils.GetCroppedImageAsync(remoteFileUrl, imageResult.BoundingBox.Left, imageResult.BoundingBox.Top, imageResult.BoundingBox.Width, imageResult.BoundingBox.Height);
result = imageResult.TagName.Contains("watch");
}
}
return(await Task.FromResult(result));
}
public static async Task <bool> CarPictureValidator(PromptValidatorContext <IList <Attachment> > promptContext, string carType)
{
// Add validation code here
var computerVisionService = new ComputerVisionService();
var detectResult = await computerVisionService.Detect(promptContext.Recognized.Value[0].ContentUrl);
if (!detectResult.IsCar)
{
await promptContext.Context.SendActivityAsync($"That doesn't look like a car. It looks more like {detectResult.Description}.");
return(false);
}
// Add Custom Vision validation here
var customVisionService = new CustomVisionService();
var predictedCarType = await customVisionService.Analyze(promptContext.Recognized.Value[0].ContentUrl);
var isRightCarType = string.Equals(predictedCarType, carType, StringComparison.OrdinalIgnoreCase);
if (!isRightCarType)
{
await promptContext.Context.SendActivityAsync($"That doesn't look like a {carType}.");
return(false);
}
return(true);
}
public IWantANewWatchDialog(IStatePropertyAccessor <UserData> botStateAccessor, IConfiguration configuration, BotConfigOptions botConfigOptions)
: base(nameof(IWantANewWatchDialog))
{
BotStateAccessor = botStateAccessor ?? throw new ArgumentNullException(nameof(botStateAccessor));
_botConfigOptions = botConfigOptions;
SiteImagesPath = configuration["imageHostUrl"];
TextAnalyticsService = new TextAnalyticsService(configuration["textAnalyticsKey"], configuration["region"]);
CustomVisionService = new CustomVisionService(configuration["customVisionKey"], configuration["customVisionProjectId"]);
ProductService = new ProductService(configuration);
// This array defines how the Waterfall will execute.
var waterfallSteps = new WaterfallStep[]
{
CheckWatchesStepAsync,
AskforPhotoStepAsync,
SearchWatchesStepAsync,
};
// Add named dialogs to the DialogSet. These names are saved in the dialog state.
AddDialog(new WaterfallDialog(SearchWatches, waterfallSteps));
AddDialog(new TextPrompt(ConfirmWatchFound));
AddDialog(new AttachmentPrompt(AskForImage, ImageValidatorAsync));
AddDialog(new TextPrompt(SearchSimilarWatches));
}
static async Task Main(string[] args)
{
//---------- Configuration -------------------
const string filesPath = @"D:\Mushrooms";
const string trainingApiKey = "fb972b87bc4b45e5b80c396c2f36fc1d";
Guid projectId = new Guid("7fd23595-1444-41c6-a91e-4c2ab67c96a3");
//Uri apiBase = new Uri("http://localhost:22481/umbraco/api/");
Uri apiBase = new Uri("http://setia-dev.azurewebsites.net/umbraco/api/");
const string trainingEndpoint = "https://southcentralus.api.cognitive.microsoft.com";
const string readyFolderId = "clean";
//----------------------------------------------
//init services
umbracoService = new UmbracoService();
umbracoService.ApiBase = apiBase;
customVisionService = new CustomVisionService(trainingApiKey, trainingEndpoint, projectId);
//----------------
string[] lettersFolders = default;
try
{
lettersFolders = Directory.GetDirectories(filesPath);
}
catch (Exception ex)
{
ConsoleError("Error opening path." + ex.Message);
Environment.Exit(0);
}
foreach (var letterFolder in lettersFolders)
{
var mushroomsFolders = Directory.GetDirectories(letterFolder).Where(folder => folder.Contains(readyFolderId, StringComparison.OrdinalIgnoreCase));
foreach (var mushroomFolder in mushroomsFolders)
{
try
{
await ProcessFolderAsync(mushroomFolder);
}
catch
{
continue;
}
}
}
Console.WriteLine();
ConsoleSuccess($"Import finished!");
Console.ReadLine();
}
private async Task <PredictionResult> getPrediction(string file)
{
if (cache.TryGetValue(file, out PredictionResult prediction))
{
return(prediction);
}
var predictionResult = await CustomVisionService.MakePredictionRequest(file);
cache.Set(file, predictionResult);
return(predictionResult);
}
private async Task MessageReceivedAsync
(IDialogContext context, IAwaitable <IMessageActivity> result)
{
var CustomVisionServiceInstance =
new CustomVisionService
(ConfigurationManager.AppSettings["CustomVision.ProjectId"],
ConfigurationManager.AppSettings["CustomVision.Key"]);
var messageResult = await result;
var connector = messageResult.GetConnector();
var finalResult = string.Empty;
var imageAttachment = messageResult
.Attachments
?.FirstOrDefault
(a => a.ContentType.Contains("image"));
if (imageAttachment != null)
{
using (var stream = await connector.GetImageStream(imageAttachment))
{
finalResult = await CustomVisionServiceInstance.GetTag(stream);
}
}
else if (Uri.IsWellFormedUriString(messageResult.Text, UriKind.Absolute))
{
finalResult = await CustomVisionServiceInstance.GetTag(messageResult.Text);
}
switch (finalResult)
{
case "coke":
finalResult = "可樂：20元";
break;
case "sprite":
finalResult = "雪碧：10元";
break;
case "pepsi":
finalResult = "百事可樂：50元";
break;
default:
finalResult = "找不到對應的飲料，請重新拍照";
break;
}
context.Done(finalResult);
}
public async Task MessageRecievedAsync(IDialogContext context, IAwaitable <IMessageActivity> argument)
{
var message = await argument;
if (message.Attachments != null && message.Attachments.Any())
{
Debug.WriteLine("message has attachment");
//read the images
var attachmentUrl = message.Attachments[0].ContentUrl;
var httpClient = new HttpClient();
try
{
var attachmentData = await httpClient.GetByteArrayAsync(attachmentUrl);
var resp = await CustomVisionService.GetPredictionsAsync2(attachmentData, "Insurance");
if (resp != null)
{
Debug.WriteLine($"prediction response {resp}");
//we got the image recognized from cog custom service
var webresult = await BingWebSearch.SearchWeb(resp + " insurance bangalore");
//once the result is there build the card
var adaptiveCards = BuildCard(webresult);
var reply = context.MakeMessage();
reply.Attachments = adaptiveCards;
reply.AttachmentLayout = AttachmentLayoutTypes.Carousel;
await context.PostAsync(reply);
}
else
{
await context.PostAsync("Sorry, I didn't recognize the image");
}
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
Debug.WriteLine(ex.InnerException);
Debug.WriteLine(ex.Source);
Debug.WriteLine(ex.StackTrace);
}
}
//only show the prompt once
if (!isUserUploadingPic)
{
PromptDialog.Confirm(context,
AfterConfirm,
"Do you have a picture of the car?");
}
}
public static async Task <IActionResult> SubmitTrainingMedia(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequestMessage req, ILogger log)
{
var response = new BaseResponse();
try
{
var provider = new MultipartMemoryStreamProvider();
await req.Content.ReadAsMultipartAsync(provider);
// Grab the file
var file = provider.Contents[0];
var fileData = await file.ReadAsByteArrayAsync();
// Grab the Data
var data = provider.Contents[1];
var stringData = await data.ReadAsStringAsync();
// Deserialize the JSON
var mediaEntry = JsonConvert.DeserializeObject <MediaTrainingData>(stringData);
mediaEntry.FileName = Guid.NewGuid().ToString() + ".jpg";
mediaEntry.SubmissionDate = DateTime.Now;
// Store the image
var uri = await FileStorageService.Instance.StoreImage(fileData, "visiontrainer", mediaEntry.FileName);
// Save into the DB
var dbService = new DBService();
var token = new System.Threading.CancellationToken();
var result = await dbService.WriteAsync(mediaEntry, token);
// Submit for training
await CustomVisionService.UploadTrainingImage(fileData);
response.StatusCode = (int)HttpStatusCode.OK;
return(new OkObjectResult(response));
}
catch (Exception ex)
{
response.Message = ex.Message;
response.StatusCode = (int)HttpStatusCode.InternalServerError;
return(new BadRequestObjectResult(response));
}
}
public static async Task <IActionResult> SubmitPredictionMedia(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequestMessage req, ILogger log)
{
var response = new PredictionMediaResponse();
try
{
var provider = new MultipartMemoryStreamProvider();
await req.Content.ReadAsMultipartAsync(provider);
var contents = await MultipartPostUtils.Parse(provider.Contents);
// Grab the Image
var fileContent = contents.GetValueOrDefault(HttpContentIds.Image);
if (fileContent == null)
{
throw new ArgumentException("'file' content was not present in the request");
}
// Grab the Data
var dataContent = contents.GetValueOrDefault(HttpContentIds.Data);
if (dataContent == null)
{
throw new ArgumentException("'data' content was not present in the request");
}
// Deserialize the JSON
var mediaData = JsonConvert.DeserializeObject <MediaPredictionData>(dataContent.Value);
// Submit for training
response.Data = await CustomVisionService.UploadPredictionImage(fileContent.Data, mediaData.TargetModelName);
response.StatusCode = (int)HttpStatusCode.OK;
return(new OkObjectResult(response));
}
catch (Exception ex)
{
response.Message = ex.Message;
response.StatusCode = (int)HttpStatusCode.InternalServerError;
return(new BadRequestObjectResult(response));
}
}
// method to send the photo to Custom Vision
public async void SendPhoto(byte[] image)
{
var current = CrossConnectivity.Current.IsConnected;
// check for the internet connection
if (!current)
{
await App.Current.MainPage.DisplayAlert("Connection Error", "Please connect to the internet", "OK");
}
else
{
var results = await CustomVisionService.PredictImageContentsAsync(image);
String resultInString = results.ToString();
if (resultInString.Length > 0)
{
if (Geolocation.HasMoreOptions(resultInString))
{
DialogService.ShowLoading("More Restaurants Available");
resultInString = await Geolocation.GetCloserOptionAsync(resultInString);
}
var navigationPage = new NavigationPage(new RestaurantPage(resultInString));
DialogService.HideLoading();
await App.Current.MainPage.Navigation.PushModalAsync(navigationPage, true);
var error = new NSError();
var device = captureDeviceInput.Device;
device.LockForConfiguration(out error);
device.FlashMode = AVCaptureFlashMode.Off;
device.UnlockForConfiguration();
}
else
{
DialogService.HideLoading();
await App.Current.MainPage.DisplayAlert("Restaurant Not Found", "Please rescan the Logo", "OK");
}
}
}
// GET: /<controller>/
public async Task <IActionResult> Index()
{
var response = await CustomVisionService.AnalyzeImage("wwwroot/demoImages/testImage.jpg");
return(View(response));
}
private async void TakePhotoButtonTapped(object sender, EventArgs e)
{
var current = CrossConnectivity.Current.IsConnected;
// check the internet connection to use the ResDiary API
if (!current)
{
await App.Current.MainPage.DisplayAlert("Connection Error", "Please connect to the internet", "OK");
}
else
{
try
{
var parameters = camera.GetParameters();
parameters.FlashMode = global::Android.Hardware.Camera.Parameters.FlashModeOff;
camera.SetParameters(parameters);
camera.StopPreview();
DialogService.ShowLoading("Scanning Logo");
// crop the image into the sqaure in order to make the prediction more accuracy
var image = CropImage(textureView.Bitmap);
using (var imageStream = new MemoryStream())
{
await image.CompressAsync(Bitmap.CompressFormat.Jpeg, 50, imageStream);
image.Recycle();
imageBytes = imageStream.ToArray();
}
// send the image to CustomVision in form of bytes
var results = await CustomVisionService.PredictImageContentsAsync(imageBytes);
String resultInString = results.ToString();
if (resultInString.Length > 0)
{
// if the logo appeared more than 1 result than use the Geolocation
if (Geolocation.HasMoreOptions(resultInString))
{
DialogService.ShowLoading("More Restaurants Available");
resultInString = await Geolocation.GetCloserOptionAsync(resultInString);
}
var navigationPage = new NavigationPage(new RestaurantPage(resultInString));
DialogService.HideLoading();
camera.StartPreview();
await App.Current.MainPage.Navigation.PushModalAsync(navigationPage, true);
}
else
{
DialogService.HideLoading();
camera.StartPreview();
await App.Current.MainPage.DisplayAlert("Restaurant Not Found", "Please re-scan the Logo", "OK");
}
}
catch (Exception)
{
camera.StopPreview();
camera.Release();
camera = global::Android.Hardware.Camera.Open((int)cameraType);
camera.SetPreviewTexture(surfaceTexture);
PrepareAndStartCamera();
}
}
}
public CustomVisionController(CustomVisionService customVisionService)
{
_customVisionService = customVisionService;
}
public MainViewModel()
{
_service = new CustomVisionService();
}
public HomeController(ILogger <HomeController> logger, DogAPIService service, CustomVisionService customVision)
{
_logger = logger;
_dogService = service;
_visionService = customVision;
}
