public void ProcessesFiles(string pdfPath)
{
var csvFile = Path.Combine(
Path.GetDirectoryName(pdfPath),
Path.GetFileNameWithoutExtension(pdfPath) + ".csv");
var expectedResult = File.ReadAllText(csvFile);
var statement = _fileProcessor.Process(pdfPath)
.ShouldHaveSingleItem();
var actualResult = _transactionSerializer
.Serialize(statement.Transactions.ToArray());
actualResult.ShouldBe(expectedResult);
}
public async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequest req,
ILogger log)
{
try
{
log.LogInformation("processing request...");
var processedFile =
await Task.Run(() => _fileProcessor.Process(req.Body));
var calculationResult = await Task.Run(() => _instructionsProcessor
.Process(processedFile.instructions, processedFile.seed));
return(new OkObjectResult(calculationResult));
}
catch (ArgumentException exception)
{
log.LogError(exception, exception.Message);
return(new BadRequestObjectResult(exception.Message));
}
catch (Exception exception)
{
log.LogError(exception, exception.Message);
return(new StatusCodeResult(500));
}
}
public DirectoryProcessingResult Process(string targetDirectory)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var files = Directory.GetFiles(targetDirectory, "*.cs", SearchOption.AllDirectories);
var result = new DirectoryProcessingResult {
TotalFileCount = files.Length
};
foreach (var path in files)
{
var fileProcessingResult = fileProcessor.Process(path);
if (fileProcessingResult.ReplacedLineCount > 0)
{
result.ReplacedFileCount++;
}
result.TotalLineCount += fileProcessingResult.TotalLineCount;
result.ReplacedLineCount += fileProcessingResult.ReplacedLineCount;
}
stopwatch.Stop();
result.Time = stopwatch.Elapsed;
return(result);
}
public IEnumerable <string> Process(string directoryName)
{
List <string> result = new List <string>();
string[] files = GetFiles(directoryName);
for (int i = 0; i < files.Length; i++)
{
if (Progress.IsCancellationPending())
{
throw new UserTerminatedException();
}
Progress.SetMessage(1, MyConvert.Format("{0}/{1}, Processing {2}", i + 1, files.Length, files[i]));
IFileProcessor processor = GetFileProcessor();
if (processor is IThreadFileProcessor)
{
(processor as IThreadFileProcessor).Progress = this.Progress;
}
result.AddRange(processor.Process(files[i]));
}
return(result);
}
private void GetFileFromList()
{
while (FileQueue.TryDequeue(out FileInfo currentFI))
{
FileInfo outpath = GetOutputPath(currentFI);
Processor.Process(currentFI, outpath, Plugins);
}
}
private async void OnCreatedFile(object sender, FileSystemEventArgs e)
{
Receipt receipt = null;
try
{
_logger.Log($"Fired OnCreatedFile...");
receipt = await _fileProcessor.Process(e.FullPath);
;
}
catch (Exception ex)
{
Console.WriteLine($"Exception {ex}:{ex.Message}");
_logger.Log($"Exception {ex}:{ex.Message}");
//Logger.Log(...);
//throw;
}
if (receipt == null)
{
Console.WriteLine("Bad file!");
// Logget.Log(...);
}
else
{
Console.WriteLine("Receipt processed ok!");
_logger.Log("Receipt processed ok!");
try
{
await _receiptSender.SendAsync(receipt);
_logger.Log($"Sent \n{receipt}");
}
catch (System.ServiceModel.CommunicationObjectFaultedException ex)
{
Console.WriteLine($"Ошибка связи с WCF-сервисом! \n{ex.GetType()}:\n{ex.Message}");
_logger.Log($"Ошибка связи с WCF-сервисом! \n{ex.GetType()}:\n{ex.Message}");
}
catch (Exception ex)
{
Console.WriteLine("Неизвестное исключение в момент связи с WCF-сервисом:");
Console.WriteLine($"{ex.GetType()}:\n{ex.Message}");
_logger.Log("Неизвестное исключение в момент связи с WCF-сервисом:");
_logger.Log($"{ex.GetType()}:\n{ex.Message}");
}
}
}
private void DoWork(object sender, DoWorkEventArgs e)
{
try
{
SendingStatus = "Getting file from AdaptIt...";
var processedFile = _fileProcessor.Process(SelectedPath.Path);
SendingStatus = "Zipping file...";
string zipPath = GetZipFile(processedFile);
SendingStatus = "Preparing email message...";
SendEmail(zipPath);
}
catch (Exception error)
{
e.Result = error;
}
}
private void SearchDirectoryInner(string dir, IFileProcessor fleProcessor, CancellationToken ct)
{
try
{
foreach (string file in Directory.GetFiles(dir))
{
//When "Just My Code" is enabled, Visual Studio in some cases will break on the line that throws the exception
//and display an error message that says "exception not handled by user code." This error is benign.
//You can press F5 to continue from it.To prevent Visual Studio from breaking on the first error, just uncheck
//the "Just My Code" checkbox under Tools, Options, Debugging, General.
ct.ThrowIfCancellationRequested();
SaveReport(fleProcessor.Process(file.Substring(_rootDirectory.Length).TrimStart(Path.DirectorySeparatorChar)));
}
foreach (string subDir in Directory.GetDirectories(dir))
{
SearchDirectoryInner(subDir, fleProcessor, ct);
}
}
catch (System.UnauthorizedAccessException excpt)
{
Console.WriteLine(excpt.Message);
}
}
// returns a wait time so that certain actions cause a longer delay before polling again
public async Task <int> Run()
{
string messageId = null; // useful to have in scope here so we can use it in exception catching
_logger.LogDebug("App.Run");
var processQueue = _configuration["rsmq:queueName"];
// Check Elastic Search status
try
{
_logger.LogDebug("checking Elastic Search status");
var response = await _elastic.ClusterHealthAsync(
new ClusterHealthRequest
{
WaitForStatus = WaitForStatus.Yellow
});
if (response.TimedOut || !response.IsValid)
{
throw new ApplicationException();
}
}
catch (Exception e)
{
_logger.LogWarning("Elastic Search server doesn't seem to be available");
_logger.LogError(e.Message);
return(_configuration.GetValue <int>("intervalMs")); // wait before re-polling
}
// Ensure the Message Queue API is up, and the queue we want exists
if (!await QueueReady(processQueue))
{
return(_configuration.GetValue <int>("intervalMs")); // wait before re-polling
}
// now continue with our work
_logger.LogDebug("Checking for messages...");
try
{
var message = await _redis.ReceiveMessage(processQueue);
if (!string.IsNullOrWhiteSpace(message.Id))
{
messageId = message.Id;
_logger.LogInformation($"Message received: {message.Id}");
// process it
// TODO if we support multiple file types
// we'll need a way to DI the correct processor
// but for now just CSV so it's ok
var ext = Path.GetExtension(message.Message);
if (Array.FindIndex(
_configuration["fileExtensions"].Split(),
x => x == ext) < 0)
{
_logger.LogInformation($"unsupported file type detected: {message.Message}");
Utils.MoveFile(PathTypes.Queued, PathTypes.BadType, message.Message, _logger);
}
else
{
// TODO really processors should work on streams
// so the ES bit is in App, and not recreated by every processor?
// but no time for that yet
await _fileProcessor.Process(message.Message);
}
}
else
{
// no message right now
return(_configuration.GetValue <int>("intervalMs")); // wait before re-polling
}
}
catch (Exception e)
{
_logger.LogError(e.Message);
}
// try and delete the message (if any) regardless of the outcome above
try
{
if (!string.IsNullOrWhiteSpace(messageId))
{
await _redis.DeleteMessage(processQueue, messageId);
}
}
catch (Exception e)
{
_logger.LogError(e.Message);
}
return(0); //re-poll immediately after actual work
}
private void Watcher_Changed(object sender, FileSystemEventArgs e)
{
Console.WriteLine("File: " + e.FullPath + " " + e.ChangeType);
_fileProcessor.Process(e.FullPath);
}
public async Task GenerateSite(GenerateSiteRequest request)
{
GlobalFunctions.Instance.Url = _siteInfo.Url;
GlobalFunctions.Instance.BaseUrl = _siteInfo.BaseUrl;
var siteGuid = _siteInfo.Url.CreateSiteGuid();
var processed = await _fileProcessor.Process(new FileFilterCriteria
{
DirectoriesToSkip = new string[] {
request.Configuration.LayoutDirectory,
request.Configuration.PartialsDirectory,
request.Configuration.DataDirectory,
request.Configuration.AssetDirectory
},
FileExtensionsToTarget = _siteInfo.SupportedFileExtensions.ToArray()
});
var pageMetadatas = processed.ToArray().ToPages(siteGuid);
var info = new AssemblyUtil().RetrieveAssemblyInfo(Assembly.GetExecutingAssembly());
var buildMetadata = new BuildData(info);
var siteMetadata = _siteMetadataFactory.EnrichSite(request.Configuration, siteGuid, pageMetadatas.ToList());
var requests = pageMetadatas
.Select(pageMetadata => new MetadataRenderRequest {
Metadata = new RenderData
{
Build = buildMetadata,
Site = siteMetadata,
Page = pageMetadata
},
Template = pageMetadata.GetValue <string>("layout")
})
.ToArray();
var renderResults = await _transformationEngine.Render(requests);
var artifacts = processed.Select((t, i) =>
{
var renderResult = renderResults[i];
return(new Artifact
{
Path = $"{t.MetaData.Uri}",
Contents = Encoding.UTF8.GetBytes(renderResult.Content)
});
}).ToList();
// TODO can we do this better?
var directoryContents =
_fileSystem.GetDirectoryContents("");
var rootFile = directoryContents.FirstOrDefault();
if (rootFile != null)
{
var root = rootFile.PhysicalPath.Replace(rootFile.Name, "");
// var root2 = Directory.GetCurrentDirectory();
var assets = _fileSystem.GetFiles(request.Configuration.AssetDirectory, true)
.Select(x => x.PhysicalPath.Replace(root, string.Empty));
artifacts.AddRange(assets.Select(asset =>
{
return(new Artifact
{
Path = $"{asset}",
Contents = _fileSystem.GetFileBytes(asset)
});
}));
await _artifactAccess.Store(new StoreArtifactsRequest
{
Artifacts = artifacts.ToArray(),
OutputLocation = new FileSystemOutputLocation()
{
Clean = false,
Path = request.Configuration.Destination
}
});
}
}
