public Task <SearchResult <ProcessFileResult> > Find(FindProcessFilesParameter parameter, CancellationToken token)
{
IQueryable <ProcessFileAggregate> result = _processFiles.AsQueryable();
if (!string.IsNullOrWhiteSpace(parameter.FileId))
{
result = result.Where(_ => _.FileId == parameter.FileId);
}
if (parameter.TakeLatest)
{
result = result.OrderByDescending(r => r.Version);
result = result.GroupBy(r => r.FileId).Select(r => r.First());
}
if (MAPPING_PROCESSFILE_TO_PROPERTYNAME.ContainsKey(parameter.OrderBy))
{
result = result.InvokeOrderBy(MAPPING_PROCESSFILE_TO_PROPERTYNAME[parameter.OrderBy], parameter.Order);
}
int totalLength = result.Count();
result = result.Skip(parameter.StartIndex).Take(parameter.Count);
return(Task.FromResult(new SearchResult <ProcessFileResult>
{
StartIndex = parameter.StartIndex,
Count = parameter.Count,
TotalLength = totalLength,
Content = result.Select(_ => ProcessFileResult.ToDto(_)).ToList()
}));
}
public async Task <SearchResult <ProcessFileResult> > Find(FindProcessFilesParameter parameter, CancellationToken token)
{
IQueryable <ProcessFileAggregate> result = _dbContext.ProcessFiles;
if (MAPPING_PROCESSFILE_TO_PROPERTYNAME.ContainsKey(parameter.OrderBy))
{
result = result.InvokeOrderBy(MAPPING_PROCESSFILE_TO_PROPERTYNAME[parameter.OrderBy], parameter.Order);
}
int totalLength = await result.CountAsync(token);
result = result.Skip(parameter.StartIndex).Take(parameter.Count);
var content = await result.ToListAsync(token);
if (parameter.TakeLatest)
{
content = content.OrderByDescending(r => r.Version).GroupBy(r => r.FileId).Select(r => r.First()).ToList();
}
return(new SearchResult <ProcessFileResult>
{
StartIndex = parameter.StartIndex,
Count = parameter.Count,
TotalLength = totalLength,
Content = content.Select(_ => ProcessFileResult.ToDto(_)).ToList()
});
}
public Task <ProcessFileResult> Get(string id, CancellationToken token)
{
var processFile = _processFiles.FirstOrDefault(_ => _.AggregateId == id);
var result = processFile == null ? null : ProcessFileResult.ToDto(processFile);
return(Task.FromResult(result));
}
/// <summary>
/// Main method executed when the console app is initially started.
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
string continueAnswer = "y";
do
{
try
{
string filePath;
Console.WriteLine("Enter the Path of the file");
filePath = Console.ReadLine();
File72ByteImportEngine engine = new File72ByteImportEngine();
var results = engine.EvaluateFileContents(filePath);
string bestPermormerTeam = ProcessFileResult.GetBestPerformer(results);
Console.WriteLine("The team which have minimum difference of for and against goal is " + bestPermormerTeam);
Console.ReadLine();
Console.WriteLine("Do you wish to evaluate another file (y/n)");
continueAnswer = Console.ReadLine();
}
catch (Exception exp)
{
Console.WriteLine(exp.Message);
Console.WriteLine("Do you wish to evaluate another file (y/n)");
continueAnswer = Console.ReadLine();
}
} while (continueAnswer.Trim().ToLower().Equals("y"));
}
public ProcessResultModel Map(ProcessFileResult processFileResult)
{
return(new ProcessResultModel
{
FileNumber = processFileResult.FileNumber,
Items = processFileResult.Items.ConvertAll(x => MapItem(x))
});
}
/* Function: FinalizeNewOrChangedFile
*
* Finalizes processing of a changed file based on the <ProcessFileResult> and whether the original file has changed or
* been deleted since processing began.
*/
protected void FinalizeNewOrChangedFile(File file, ProcessFileResult processResult)
{
bool deletedSinceProcessed, changedSinceProcessed;
lock (accessLock)
{
var originalProperties = filesBeingProcessed.GetPropertiesWhenAdded(file);
deletedSinceProcessed = file.Deleted;
changedSinceProcessed = (!deletedSinceProcessed && file.LastModified != originalProperties.LastModified);
filesBeingProcessed.Remove(file);
}
if (processResult == ProcessFileResult.Success)
{
if (changedSinceProcessed)
{
Manager.UnprocessedChanges.AddChangedFile(file);
}
else if (deletedSinceProcessed)
{
Manager.UnprocessedChanges.AddDeletedFile(file);
}
// else
// success
}
else if (processResult == ProcessFileResult.Cancelled)
{
if (deletedSinceProcessed)
{
Manager.UnprocessedChanges.AddDeletedFile(file);
}
else
{
Manager.UnprocessedChanges.AddChangedFile(file);
}
}
else if (processResult == ProcessFileResult.CantAccessFile ||
processResult == ProcessFileResult.FileDoesntExist)
{
if (changedSinceProcessed)
{
Manager.UnprocessedChanges.AddChangedFile(file);
}
else
{
Manager.UnprocessedChanges.AddDeletedFile(file);
}
}
else
{
throw new ArgumentException();
}
}
private void LogFileProcessingResults(string filePath, long fileSizeBytes, ProcessFileResult processFileResult, TimeSpan elapsed)
{
var fileSizeMb = fileSizeBytes / 1024 / 1024;
var mbPerSecond = elapsed.TotalSeconds > 0
? fileSizeMb / elapsed.TotalSeconds
: fileSizeMb;
_logger.LogInformation("Completed processing file {logFile} in {logFileElapsed} ({logFileSizeMb} MB, {logFileLines} lines, {logFileMbPerSecond:F2} MB/s)", filePath, elapsed, fileSizeMb, processFileResult.LinesProcessed, mbPerSecond);
}
public static void ProcessFileUpload(this HttpRequestMessage request, Func <ProcessFileArguments, bool> processor)
{
HttpContext.Current.Response.Buffer = false;
HttpContext.Current.Response.Expires = -1;
HttpContext.Current.Response.Cache.SetCacheability(HttpCacheability.NoCache);
HttpContext.Current.Response.Write(new string(' ', 4096));
HttpContext.Current.Response.Flush();
ProcessProgress.Current.RegisterResponser(ScriptProgressResponser.Instance);
ProcessFileResult result = new ProcessFileResult();
try
{
ProcessProgress.Current.MinStep = 0;
ProcessProgress.Current.CurrentStep = 0;
ProcessProgress.Current.MaxStep = 100;
request.Content.IsMimeMultipartContent("form-data").FalseThrow("上传请求的格式不正确");
InMemoryMultipartFormDataStreamProvider provider = InMemoryMultipartFormDataStreamProvider.GetProvider(request);
bool needContinue = true;
foreach (StreamContent fileContent in provider.Files)
{
ProcessFileArguments args = new ProcessFileArguments()
{
FormData = provider.FormData,
UploadedStream = fileContent.ReadAsStreamAsync().Result,
ProcessResult = result
};
if (processor != null)
{
needContinue = processor(args);
}
if (needContinue == false)
{
break;
}
}
}
catch (System.Exception ex)
{
result.Error = ex.Message;
}
finally
{
ScriptProgressResponser.Instance.ResponseResult("processComplete", result);
HttpContext.Current.Response.End();
}
}
public void ResponseResult(string funcName, ProcessFileResult pfResult)
{
JsonSerializerSettings settings = new JsonSerializerSettings();
settings.ContractResolver = new CamelCasePropertyNamesContractResolver();
string json = ScriptHelper.CheckScriptString(JsonConvert.SerializeObject(pfResult, settings), false);
string script = string.Format("{0}(\"{1}\")", funcName, json);
script = string.Format("<script type=\"text/javascript\">\n parent.{0}\n</script>", script);
HttpResponse response = HttpContext.Current.Response;
response.Write(script);
response.Flush();
}
public async Task <ProcessFileResult> ProcessFile(IFormFile formFile)
{
var result = new ProcessFileResult();
var options = _configuration.GetSection(SiUpinOptions.RootSection).Get <SiUpinOptions>();
var fileName = WebUtility.HtmlEncode(formFile.FileName);
if (formFile.Length > options.FileSizeLimit)
{
result.ErrorMessage = $"File {fileName} is too large.";
}
else
{
try
{
using var memoryStream = new MemoryStream();
await formFile.CopyToAsync(memoryStream);
if (memoryStream.Length > 0)
{
result.IsSuccessful = true;
result.FileContent = memoryStream.ToArray();
}
else
{
result.ErrorMessage = $"File {fileName} is empty.";
}
}
catch
{
result.ErrorMessage = $"Failed to process File: {fileName}";
}
}
return(result);
}
/* Function: FinalizeDeletedFile
*
* Finalizes processing of a deleted file based on the <ProcessFileResult> and whether the original file has changed or
* been deleted since processing began.
*/
protected void FinalizeDeletedFile(File file, ProcessFileResult processResult)
{
bool recreatedSinceProcessed = !file.Deleted;
lock (accessLock)
{
filesBeingProcessed.Remove(file);
}
if (processResult == ProcessFileResult.Success)
{
if (recreatedSinceProcessed)
{
Manager.UnprocessedChanges.AddNewFile(file);
}
// else
// success
}
else if (processResult == ProcessFileResult.Cancelled)
{
if (recreatedSinceProcessed)
{
Manager.UnprocessedChanges.AddNewFile(file);
}
else
{
Manager.UnprocessedChanges.AddDeletedFile(file);
}
}
else
{
throw new ArgumentException();
}
}
public ProcessLogSetResult ProcessLogSet()
{
_logger.LogInformation("Using temp folder `{tempDir}`", _config.TempDir);
using var logsExtractor = new TableauLogsExtractor(_config.LogSetLocation, _config.TempDir, _processingNotificationsCollector, _loggerFactory.CreateLogger <TableauLogsExtractor>());
if (!_pluginManager.IsValidPluginConfiguration(out var badPluginNames))
{
return(ProcessLogSetResult.Failed(FailureReasonMessageGenerator.BadPluginNamesSpecified(badPluginNames), ExitReason.IncorrectConfiguration));
}
var loadedPlugins = _pluginManager.CreatePlugins(_writerFactory, _processingNotificationsCollector).ToList();
var requiredLogTypes = _pluginManager.GetRequiredLogTypes().ToList();
var processedLogTypes = string.Join(", ", requiredLogTypes.OrderBy(name => name));
_logger.LogInformation("Based on requested plugins, the following log types will be processed: {processedLogTypes}", processedLogTypes);
var logProcessingStatistics = new Dictionary <LogType, ProcessLogTypeResult>();
var pluginsReceivedAnyData = new HashSet <string>();
ProcessLogTypeResult failedLogTypeResult = null;
foreach (var logType in requiredLogTypes)
{
_logger.LogInformation("Starting to process {logType} logs", logType);
var logTypeInfo = _logTypeDetails.GetInfoForLogType(logType);
var applicablePlugins = loadedPlugins
.Where(plugin => plugin.ConsumedLogTypes.Contains(logType))
.ToList();
try
{
var logTypeProcessingResult = ProcessLogType(logsExtractor.LogSetParts, logTypeInfo, applicablePlugins);
logProcessingStatistics.Add(logType, logTypeProcessingResult);
if (!logTypeProcessingResult.IsSuccessful)
{
failedLogTypeResult = logTypeProcessingResult;
break;
}
_logger.LogInformation("Done processing {logType} logs. {processingResult}", logType, logTypeProcessingResult);
if (logTypeProcessingResult.FilesProcessed > 0)
{
foreach (var plugin in applicablePlugins)
{
pluginsReceivedAnyData.Add(plugin.Name);
}
}
}
catch (Exception ex)
{
var unhandledExceptionMessage = $"Unhandled exception occurred while processing log type {logType}. Exception: {ex.Message}";
_logger.LogError(ex, unhandledExceptionMessage);
var fakeProcessingFileResult = new ProcessFileResult(0, unhandledExceptionMessage, ExitReason.UnclassifiedError);
var fakeProcessingTypeResults = new ProcessLogTypeResult();
fakeProcessingTypeResults.AddProcessingInfo(TimeSpan.Zero, 0, fakeProcessingFileResult);
failedLogTypeResult = fakeProcessingTypeResults;
break;
}
}
_logger.LogInformation("Telling all plugins to complete processing of any cached data");
var pluginsExecutionResults = _pluginManager.SendCompleteProcessingSignalToPlugins(failedLogTypeResult != null);
_logger.LogInformation("Completed reading log set and generating data");
var(errorMessage, existReason) = GetExitReasonAndErrorMessageIfApplicable(failedLogTypeResult, logProcessingStatistics);
var loadedPluginNames = loadedPlugins.Select(plugin => plugin.Name).ToHashSet();
return(new ProcessLogSetResult(
errorMessage,
existReason,
logsExtractor.LogSetSizeBytes,
logsExtractor.IsDirectory,
loadedPluginNames,
logProcessingStatistics,
pluginsExecutionResults,
pluginsReceivedAnyData));
}
public async Task <ProcessFileResult> Get(string id, CancellationToken cancellationToken)
{
var processFile = await _dbContext.ProcessFiles.FirstOrDefaultAsync(_ => _.AggregateId == id, cancellationToken);
return(processFile == null ? null : ProcessFileResult.ToDto(processFile));
}
