public async Task GetTaskAsyncCorrectResponse() { // ARRANGE OCRTask taskToReturn = new OCRTask() { TaskId = "00000000-0000-0000-0000-000000000000", Priority = 5, Intensity = 0, OrigFileName = "testfile.jpg", LocalTempFile = null }; // prepare what is going to be returned by httpclient string content = JsonConvert.SerializeObject(taskToReturn); var httpClient = CreateHttpClientMock(HttpStatusCode.OK, content); var rc = new RestClient(httpClient, CreateIOptionsMonitorMock(), CreateILoggerMock()); // ACT var result = await rc.GetTaskAsync(new CancellationToken()); // ASSERT result.Should().NotBeNull(); result.TaskId.Should().Be(taskToReturn.TaskId); result.Priority.Should().Be(taskToReturn.Priority); result.Intensity.Should().Be(taskToReturn.Intensity); result.OrigFileName.Should().Be(taskToReturn.OrigFileName); result.LocalTempFile.Should().Be(taskToReturn.LocalTempFile); result.InternalFileName.Should().NotBe(taskToReturn.InternalFileName); // unique guid }
private static async Task <OCRTask> GetNewImage(IClient hlidacRest, ILogger logger) { logger.LogInformation("Getting new image."); // try to get task until we get some :) while (true) { OCRTask task = await hlidacRest.GetTaskAsync(); if (task != null && !string.IsNullOrWhiteSpace(task.TaskId)) { var downloadStream = await hlidacRest.GetFileToAnalyzeAsync(task.TaskId); using (var fileStream = new FileStream(task.InternalFileName, FileMode.Create, FileAccess.Write)) { await downloadStream.CopyToAsync(fileStream); } logger.LogInformation($"Image for task[{task.TaskId}] successfully downloaded."); return(task); } else { string returnedTask = Newtonsoft.Json.JsonConvert.SerializeObject(task); logger.LogWarning($"Returned task is invalid. \n{returnedTask}"); // invalid task is probably because there were no tasks to process on server side, we need to wait some time // todo - this can be done in polly probably await Task.Delay(TimeSpan.FromSeconds(20)); } } }
public async Task RunFlow(CancellationToken cancellationToken) { try { _currentTask = await _hlidacRest.GetTaskAsync(cancellationToken); await GetImageAsync(cancellationToken); await RunTesseractAsync(cancellationToken); } catch (DelayedException ex) { if (ex is ServerHasNoTasksException) { _logger.LogDebug($"{ex.Message} Next try in {ex.DelayInSec.ToString()} seconds."); } else { _logger.LogWarning($"{ex.Message} Next try in {ex.DelayInSec.ToString()} seconds."); } try { await Task.Delay(TimeSpan.FromSeconds(ex.DelayInSec), cancellationToken); } catch { // cancellation token called } return; } catch (TaskCanceledException) { _logger.LogInformation("Application was canceled."); if (_currentTask != null && !string.IsNullOrWhiteSpace(_currentTask.TaskId)) { await _hlidacRest.CancelTaskAsync(_currentTask.TaskId); } } catch (Exception ex) { _logger.LogWarning($"{ex.Message}."); try { await Task.Delay(TimeSpan.FromSeconds(5 * 60), cancellationToken); // lets wait for a while } catch { // cancellation token was called } } finally { Cleanup(); } }
private static async Task <int> Main() { #region preconfiguration const string env_apiKey = "OCRM_APIKEY"; const string env_email = "OCRM_EMAIL"; const string env_demo = "OCRM_DEMO"; const string base_address = "base_address"; const string user_agent = "user_agent"; var confBuilder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true) .AddEnvironmentVariables(); var appConfiguration = confBuilder.Build(); // check if api key is set, otherwise, close app if (string.IsNullOrWhiteSpace(appConfiguration.GetValue <string>(env_apiKey))) { Console.WriteLine($"Environment variable{env_apiKey} is not set. Exiting app."); return(1); } // write basic configuration to stdout Console.WriteLine("Loaded configuration:"); Console.WriteLine($" {env_apiKey}={appConfiguration.GetValue<string>(env_apiKey)}"); Console.WriteLine($" {env_email}={appConfiguration.GetValue<string>(env_email)}"); Console.WriteLine($" {env_demo}={appConfiguration.GetValue<bool>(env_demo)}"); Console.WriteLine($" {base_address}={appConfiguration.GetValue<string>(base_address)}"); Console.WriteLine($" {user_agent}={appConfiguration.GetValue<string>(user_agent)}"); #endregion preconfiguration // todo: graceful shutdown https://stackoverflow.com/questions/40742192/how-to-do-gracefully-shutdown-on-dotnet-with-docker #region configuration var builder = new HostBuilder() .ConfigureAppConfiguration(configure => { configure.AddConfiguration(appConfiguration); }) .ConfigureLogging((hostContext, logging) => { logging.AddConfiguration(hostContext.Configuration.GetSection("Logging")); logging.AddConsole(); // time doesn't need timestamp to it, because it is appended by docker }) .ConfigureServices((hostContext, services) => { services.Configure <ClientOptions>(config => { config.ApiKey = hostContext.Configuration.GetValue <string>(env_apiKey); config.Email = hostContext.Configuration.GetValue <string>(env_email); if (string.IsNullOrWhiteSpace(config.Email)) { config.Email = Guid.NewGuid().ToString(); } config.Demo = hostContext.Configuration.GetValue <bool>(env_demo, false); }); services.AddHttpClient <IClient, RestClient>(config => { config.BaseAddress = new Uri(hostContext.Configuration.GetValue <string>(base_address)); config.DefaultRequestHeaders.Add("User-Agent", hostContext.Configuration.GetValue <string>(user_agent)); }) .AddPolicyHandler(Policy.TimeoutAsync <HttpResponseMessage>( TimeSpan.FromMinutes(5), // polly wait max 5 minutes for response Polly.Timeout.TimeoutStrategy.Optimistic)) .AddTransientHttpErrorPolicy(pb => pb.WaitAndRetryAsync(400, retryAttempt => TimeSpan.FromSeconds(retryAttempt / 20)) ); // total waiting time in case of repeating transient error // should be about 67 minutes, then app restarts }).UseConsoleLifetime(); var host = builder.Build(); #endregion configuration using (var serviceScope = host.Services.CreateScope()) { var services = serviceScope.ServiceProvider; var logger = services.GetRequiredService <ILogger <Program> >(); int taskCounter = 0; try { var hlidacRest = services.GetRequiredService <IClient>(); var taskQueue = new Queue <Task <OCRTask> >(3); logger.LogInformation("OCR minion initialized."); taskQueue.Enqueue(GetNewImage(hlidacRest, logger)); while (true) { OCRTask currentTask = await taskQueue.Dequeue(); logger.LogInformation($"Starting OCR process of #{++taskCounter}. task."); // run OCR and wait for its end // to run OCR asynchronously I am using this library: https://github.com/jamesmanning/RunProcessAsTask DateTime taskStart = DateTime.Now; Task <ProcessResults> tesseractTask; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // this part is here only for debugging purposes string tesseractArgs = $"{currentTask.InternalFileName} {currentTask.InternalFileName} -l ces --psm 1 --dpi 300"; tesseractTask = ProcessEx.RunAsync("tesseract.exe", tesseractArgs); } else { string tesseractArgs = $"{currentTask.InternalFileName} {currentTask.InternalFileName} -l ces --psm 1 --dpi 300".Replace("\"", "\\\""); tesseractTask = ProcessEx.RunAsync("tesseract", tesseractArgs); } // we can preload new image here, so we doesnt have to wait for it later taskQueue.Enqueue(GetNewImage(hlidacRest, logger)); var tesseractResult = await tesseractTask; DateTime taskEnd = DateTime.Now; if (tesseractResult.ExitCode == 0) { logger.LogInformation($"OCR process of #{taskCounter}. task successfully finished."); string text = await File.ReadAllTextAsync($"{currentTask.InternalFileName}.txt", Encoding.UTF8); Document document = new Document(currentTask.TaskId, taskStart, taskEnd, currentTask.OrigFileName, text, tesseractResult.RunTime.TotalSeconds.ToString()); await hlidacRest.SendResultAsync(currentTask.TaskId, document); File.Delete(currentTask.InternalFileName); File.Delete(currentTask.InternalFileName + ".txt"); } else { logger.LogWarning($"OCR process of #{taskCounter}. task unsuccessfully finished."); logger.LogWarning(string.Join('\n', tesseractResult.StandardError)); } } } catch (Exception ex) { logger.LogError(ex, "Guess what? Something went wrong and we don't know what."); // todo: send this error message to a server return(1); } } }