public bool TryExtract(string fullPath, Stream stream, string fileName, out ExtractionResult result) { result = null; if (IsExtractable(fileName)) { var encoding = GetFileEncoding(stream); stream.Position = 0; //TODO: buscar una forma mas elegante de hacer esto var text = new StreamReader(stream, encoding).ReadToEnd(); //TODO: mejorar este parche feo y caro para soportar a la vez ansi y utf sin bom if (text.Contains("Ã")) { var bytes = encoding.GetBytes(text); text = Encoding.UTF8.GetString(bytes); } result = new ExtractionResult() { ContentType = "text/plain", PlainContent = text }; return true; } else { return false; } }
public bool TryExtract(string fullPath, Stream stream, string fileName, out ExtractionResult result) { result = null; if (IsExtractable(fileName)) { var encoding = GetFileEncoding(stream); stream.Position = 0; //TODO: buscar una forma mas elegante de hacer esto var text = new StreamReader(stream, encoding).ReadToEnd(); //TODO: mejorar este parche feo y caro para soportar a la vez ansi y utf sin bom if (text.Contains("Ã")) { var bytes = encoding.GetBytes(text); text = Encoding.UTF8.GetString(bytes); } result = new ExtractionResult() { ContentType = "text/plain", PlainContent = text }; return(true); } else { return(false); } }
public TransformResult Transform(ExtractionResult extractionResult) { List <FreightByShipper> _freightByShipperList = (from invoice in extractionResult.Invoices group invoice by invoice.ShipperName into shippers select new FreightByShipper { ShipperName = shippers.Key, Freight = shippers.Sum(invoice => invoice.Freight) }).ToList(); foreach (FreightByShipper freightByShipper in _freightByShipperList) { _outputStream.WriteLine("Shipper: {0}; Freight: {1}", freightByShipper.ShipperName, freightByShipper.Freight); } decimal totalFreight = _freightByShipperList.Sum(f => f.Freight); _outputStream.WriteLine("Total Freight: " + totalFreight); var employeesWithBonus = extractionResult.Employees.ToList(); foreach (Employee employee in employeesWithBonus) { employee.CalculateAndSetBonus(totalFreight); } return(new TransformResult() { Employees = employeesWithBonus, FreightByShippers = _freightByShipperList }); }
private static ExtractionResult <T> Extract <T>(JArray jArr) where T : struct { var result = new ExtractionResult <T>(); var type = typeof(T); Type[] argTypes = { typeof(string), type.MakeByRefType() }; var method = typeof(T).GetMethod("TryParse", BindingFlags.Static | BindingFlags.Public, Type.DefaultBinder, argTypes, null); foreach (var record in jArr) { var parameters = new object[] { record.ToString(), null }; var success = (bool)method.Invoke(null, parameters); if (!success) { result.InvalidValues.Add(record.ToString()); } else { var retVal = (T)parameters[1]; result.Values.Add(retVal); } } return(result); }
public ExtractionResult ExtractText(IndexingRequestDetails details, Stream stream) { using (var reader = new StreamReader(stream, details.Encoding, false, 16, true)) { var doc = XDocument.Load(reader); var returnResult = new ExtractionResult(details); if (Extensions.Count == 0) { returnResult.GenerateSegments(doc, SimpleDegrapher.XElementDegrapher); } else { foreach (var ext in Extensions) { if (ext.TryProcess(returnResult, doc)) { break; } } } returnResult.AnnotateSegments(); return(returnResult); } }
public RunInitializationResult Initialize(RunInitializationRequest request) { if (request.Target.Type != LogsetTarget.File && request.Target.Type != LogsetTarget.Directory) { throw new ArgumentException("Request target must be a file or directory!", "request"); } ExtractionResult extractionResult = ExtractLogset(request.Target, request.RunId); var artifactProcessorLoader = new ArchiveArtifactProcessorLoader(); IArtifactProcessor artifactProcessor = artifactProcessorLoader.LoadArtifactProcessor(extractionResult.RootLogDirectory); string logsetHash = ComputeLogsetHash(extractionResult.RootLogDirectory, artifactProcessor); var pluginLoader = new PluginLoader(request.ArtifactProcessorOptions); ISet <Type> pluginsToExecute = pluginLoader.LoadPlugins(request.RequestedPlugins, artifactProcessor); var extractedTarget = new LogsharkRequestTarget(extractionResult.RootLogDirectory); ISet <string> collectionDependencies; if (request.ParseFullLogset) { collectionDependencies = GetAllSupportedCollections(artifactProcessor, extractionResult.RootLogDirectory); } else { collectionDependencies = pluginLoader.GetCollectionDependencies(pluginsToExecute); } return(new RunInitializationResult(extractedTarget, artifactProcessor, collectionDependencies, logsetHash, pluginsToExecute)); }
private static Model BuildModelsForScenario(ExtractionResult extractionResult, int scenarioId, IEnumerable <Sentence <int> > queries) { var root = new Model { Time = -1, Fluents = null }; var initialModels = new List <Model>(); foreach (var fluents in InitializeFluents(extractionResult.Fluents.Keys.Count())) { initialModels.Add(new Model { Fluents = fluents, Time = 0, CurrentActionId = -1 //wyciągnąć do jakiegoś ładnego defaultu }); } foreach (var initModel in initialModels) { TryInvokeAction(initModel, extractionResult, scenarioId); } return(root); }
public ExtractionResult Extract(Document document) { if (document == null) { throw new ArgumentNullException(nameof(document)); } var text = ProcessPreparation(document.Text); if (!CanProcess(text)) { return(ExtractionResult.None); } var fields = processor.Construct(template, document).ToArray(); var lookup = fields.ToLookup(item => item.Key, item => item.Value, StringComparer.OrdinalIgnoreCase); foreach (var required in template.RequiredFields) { if (!lookup.Contains(required)) { logger.LogWarning("Required field {0} not found", required); return(ExtractionResult.None); } } var result = new ExtractionResult(document, template, fields); return(result); }
/// <summary> /// Extracts dynamic values from a formatted string. /// </summary> /// <param name="str">String including dynamic values</param> /// <param name="format">Format of the string</param> /// <param name="ignoreCase">True, to search case-insensitive.</param> public ExtractionResult Extract(string str, string format, bool ignoreCase = false) { var stringComparison = ignoreCase ? StringComparison.InvariantCultureIgnoreCase : StringComparison.InvariantCulture; if (str == format) //TODO: think on that! { return(new ExtractionResult(true)); } var formatTokens = new FormatStringTokenizer().Tokenize(format); if (formatTokens.IsNullOrEmpty()) { return(new ExtractionResult(str == "")); } var result = new ExtractionResult(true); for (var i = 0; i < formatTokens.Count; i++) { var currentToken = formatTokens[i]; var previousToken = i > 0 ? formatTokens[i - 1] : null; if (currentToken.Type == FormatStringTokenType.ConstantText) { if (i == 0) { if (!str.StartsWith(currentToken.Text, stringComparison)) { result.IsMatch = false; return(result); } str = str.Substring(currentToken.Text.Length); } else { var matchIndex = str.IndexOf(currentToken.Text, stringComparison); if (matchIndex < 0) { result.IsMatch = false; return(result); } Debug.Assert(previousToken != null, "previousToken can not be null since i > 0 here"); result.Matches.Add(new NameValue(previousToken.Text, str.Substring(0, matchIndex))); str = str.Substring(matchIndex + currentToken.Text.Length); } } } var lastToken = formatTokens.Last(); if (lastToken.Type == FormatStringTokenType.DynamicValue) { result.Matches.Add(new NameValue(lastToken.Text, str)); } return(result); }
/// <summary> /// Extracts dynamic values from a formatted string. /// </summary> /// <param name="str">String including dynamic values</param> /// <param name="format">Format of the string</param> /// <param name="ignoreCase">True, to search case-insensitive.</param> /// <param name="splitformatCharacter">format is splitted using this character when provided.</param> public ExtractionResult Extract(string str, string format, bool ignoreCase = false, char?splitformatCharacter = null) { var stringComparison = ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; if (str == format) { return(new ExtractionResult(true)); } var formatTokens = TokenizeFormat(format, splitformatCharacter); if (formatTokens.IsNullOrEmpty()) { return(new ExtractionResult(string.IsNullOrEmpty(str))); } var result = new ExtractionResult(false); for (var i = 0; i < formatTokens.Count; i++) { var currentToken = formatTokens[i]; var previousToken = i > 0 ? formatTokens[i - 1] : null; if (currentToken.Type == FormatStringTokenType.ConstantText) { if (i == 0) { if (str.StartsWith(currentToken.Text, stringComparison)) { str = str.Substring(currentToken.Text.Length); } } else { var matchIndex = str.IndexOf(currentToken.Text, stringComparison); if (matchIndex >= 0) { Debug.Assert(previousToken != null, "previousToken can not be null since i > 0 here"); result.Matches.Add(new NameValue(previousToken.Text, str.Substring(0, matchIndex))); result.IsMatch = true; str = str.Substring(matchIndex + currentToken.Text.Length); } } } } var lastToken = formatTokens.Last(); if (lastToken.Type == FormatStringTokenType.DynamicValue) { result.Matches.Add(new NameValue(lastToken.Text, str)); result.IsMatch = true; } return(result); }
protected internal virtual void Validate(SamlOptions options, ExtractionResult extractionResult, TMessageContext messageContext) { if (options.AcceptSignedMessagesOnly && (!extractionResult.HasValidSignature && !messageContext.HasValidSignature)) { throw new SamlException("Messages without or with invalid signatures are not accepted."); } }
protected internal override void Validate(SamlOptions options, ExtractionResult extractionResult, SamlAuthnResponseMessage messageContext) { if ((DateTime.UtcNow - messageContext.IssueInstant) > options.IssueInstantExpiration) { throw new SamlException("Issue instant is too long ago."); } }
public Algorithm(ExtractionResult result, int scenarioId, IReadOnlyList <Sentence> queries, IModelSaver modelSaver) { _extractionResult = result; _scenarioId = scenarioId; _queries = queries; _endTime = ScenarioHelper.GetScenarioEndTime(result, scenarioId); _invokedActions = new Dictionary <int, int>(); _modelSaver = modelSaver; }
public static int GetScenarioEndTime(ExtractionResult result, int scenarioId) { var scenario = result.Sentences.OfType <Sentence.ScenarioDefinition>().FirstOrDefault(s => s.Name == scenarioId); if (scenario == null) { throw new Exception("Scenario not found"); } return(scenario.Time); }
public static IList <Model> BuildModels(ExtractionResult extractionResult, IEnumerable <Sentence <int> > queries) { var scenarios = new List <Model>(); for (var i = 0; i < extractionResult.Scenarios.Count; ++i) { scenarios[i] = BuildModelsForScenario(extractionResult, i, queries); } return(scenarios); }
public static List <int> GetActionsFromScenario(ExtractionResult result, int scenarioId, int time) { var scenario = result.Sentences.OfType <Sentence.ScenarioDefinition>().FirstOrDefault(s => s.Name == scenarioId); if (scenario == null) { throw new Exception("Scenario not found"); } return(scenario.Actions.Where(a => a.Item2 == time).Select(a => a.Item1).ToList()); }
private ExtractionResult ProcessList(List <List <DataRow> > inputList) { var _result = new ExtractionResult(); _result.TotalProcessedRecords = inputList.Count(); var list = new List <importRecordTmp>(); foreach (var listDR in inputList) { //Skipping the header for (int i = 1; i < listDR.Count(); i++) { //Checking if it is an empty line if (string.IsNullOrEmpty(listDR[i][(int)RecordField.Date].ToString())) { break; } decimal expense = 0; var ValidValue = decimal.TryParse(listDR[i][(int)RecordField.Expense].ToString(), out expense); decimal revenue = 0; if (ValidValue) { ValidValue = decimal.TryParse(listDR[i][(int)RecordField.Revenue].ToString(), out revenue); } DateTime date = new DateTime(); if (ValidValue) { ValidValue = DateTime.TryParse(listDR[i][(int)RecordField.Date].ToString(), out date); } var record = new importRecordTmp { description = listDR[i][(int)RecordField.Description].ToString(), category = listDR[i][(int)RecordField.Category].ToString(), subcategory = listDR[i][(int)RecordField.Subcategory].ToString(), expense = expense, revenue = revenue, comment = listDR[i][(int)RecordField.Comment].ToString(), date = date, importError = !ValidValue }; list.Add(record); } } _result.TotalDiscardedRecord = list.Where(p => p.importError == true).Select(p => p).ToList().Count(); _result.ValidRecords = list.Where(i => !i.importError).ToList(); return(_result); }
public ExtractionResult ExtractText(IndexingRequestDetails details, Stream stream) { using (var reader = new StreamReader(stream, details.Encoding, false, 16, true)) { var document = reader.ReadToEnd(); var lines = document.Split('\r', '\n').Where(s => !string.IsNullOrEmpty(s)).ToArray(); var returnResult = new ExtractionResult(details); returnResult.GenerateSegments(lines, null); returnResult.AnnotateSegments(); return(returnResult); } }
public ExtractionResult ExtractText(IndexingRequestDetails details, Stream stream) { using (var reader = new StreamReader(stream, details.Encoding, false, 16, true)) { var obj = JsonConvert.DeserializeObject(reader.ReadToEnd()); var returnResult = new ExtractionResult(details); returnResult.GenerateSegments(obj, JsonDegrapher); returnResult.AnnotateSegments(); return(returnResult); } }
private static bool ValidateModel(Model model, ExtractionResult extractionResult, int scenarioId) { // ReSharper disable once LoopCanBeConvertedToQuery foreach (var observation in extractionResult.Sentences.OfType <Sentence <int> .ScenarioDefinition>().ToList()[scenarioId] .Observations.Where(item => item.Item2 == model.Time)) { if (!ValidateObservation(observation.Item1, model.Fluents)) { return(false); } } return(true); }
//Returns id of action to be invoked. -1 if no action should be invoked. null if model is contradictory public static int?GetActionToStart(ExtractionResult extractionResult, int scenarioId, int time, bool[] fluents, Dictionary <int, int> invokedActions) { var actionToInvoke = -1; //get actions from scenario var actions = ScenarioHelper.GetActionsFromScenario(extractionResult, scenarioId, time); //get triggered actions var triggeredActions = ActionHelper.CheckTriggerSentences(extractionResult.Sentences, fluents); //get invoked action if (!invokedActions.TryGetValue(time, out var invokedActionId)) { invokedActionId = -1; } if (actions.Count > 1) { return(null); //error } if (actions.Any()) { actionToInvoke = actions.FirstOrDefault(); } if (triggeredActions.Count > 1) { return(null); //error } if (triggeredActions.Any()) { if (actionToInvoke < 0) { actionToInvoke = triggeredActions.FirstOrDefault(); } else if (actionToInvoke != triggeredActions.FirstOrDefault()) //still it can be the same action!!! { return(null); //error - two actions possible } } if (invokedActionId >= 0) { if (actionToInvoke < 0) { actionToInvoke = invokedActionId; } else if (actionToInvoke != invokedActionId) //still it can be the same action!!! { return(null); //error - two actions possible } } return(actionToInvoke); }
public void ValidateShouldNotThrowExceptionIfMessageIsSigned() { var parsingResult = new FakeSamlMessage { HasValidSignature = true }; var extractionResult = new ExtractionResult { HasValidSignature = false }; _options.AcceptSignedMessagesOnly = true; _target.Validate(_options, extractionResult, parsingResult); }
public void ValidateShouldThrowExceptionIfMessageIsUnsignedButNeedsToBe() { var parsingResult = new FakeSamlMessage { HasValidSignature = false }; var extractionResult = new ExtractionResult { HasValidSignature = false }; _options.AcceptSignedMessagesOnly = true; Assert.Throws <SamlException>(() => _target.Validate(_options, extractionResult, parsingResult)); }
public AlgorithmResultModel(Dictionary <int, SimpleModelSaver> models, ExtractionResult extractionResult) { var reverseFluents = extractionResult.Fluents.ToDictionary(x => x.Value, x => x.Key); var reverseActions = extractionResult.Actions.ToDictionary(x => x.Value, x => x.Key); var reverseScenarios = extractionResult.Scenarios.ToDictionary(x => x.Value, x => x.Key); foreach (var scenario in models) { var name = reverseScenarios[scenario.Key]; var result = new ScenarioResultsModel { Name = name, ModelsCount = scenario.Value.TotalModelsCount }; for (var i = 0; i < scenario.Value.SavedModels.Count; i++) { var model = scenario.Value.SavedModels[i]; var modelModel = new ModelModel { Name = $"Model {i + 1}", HistoryTimeColumns = GetDataGridColumns(model.EndTime, true), OcclusionTimeColumns = GetDataGridColumns(model.EndTime), ActionsTimeColumns = GetDataGridColumns(model.EndTime) }; modelModel.Actions.Add(new HistoryModel { Values = model.Actions.Select(a => a == -1 ? "" : reverseActions[a]).ToArray() }); modelModel.Occlusion.Add(new HistoryModel { Values = model.Occlusion.Select(x => x != null ? string.Join(",", x.Select(o => reverseFluents[o])) : "").ToArray() }); for (var j = 0; j < reverseFluents.Count; j++) { modelModel.History.Add(new HistoryModel { Name = reverseFluents[j], Values = model.History.Select(h => h[j].ToString()).ToArray() }); } result.Models.Add(modelModel); } if (result.Models.Any()) { ScenarioResults.Add(result); } } }
public static bool ValidateModelWithObservation(bool[] fluents, int time, ExtractionResult result, int scenarioId) { var scenario = result.Sentences.OfType <Sentence.ScenarioDefinition>().FirstOrDefault(s => s.Name == scenarioId); if (scenario == null) { return(false); } foreach (var scenarioObservation in scenario.Observations.Where(o => o.Item2 == time)) { if (!ExpressionHelper.IsExpressionTrue(scenarioObservation.Item1, fluents)) { return(false); } } return(true); }
public void GetDataFromInvoicesAndEmployeesFakes() { // Arrange var invoiceRepository = new FakeInvoiceRepository(); var employeeRepository = new FakeEmployeeRepository(); var service = new ExtractionService(invoiceRepository, employeeRepository, Console.Out); // Act ExtractionResult result = service.Extract(); // Assert Assert.That(invoiceRepository.WasCalled); Assert.That(employeeRepository.WasCalled == false); }
public ExtractionResult ExtractText(IndexingRequestDetails details, Stream stream) { using (var reader = new StreamReader(stream, details.Encoding, false, 16, true)) { var rows = new List <string>(); var row = string.Empty; while ((row = reader.ReadLine()) != null) { var entry = string.Join("\r\n", row.Split('\t').AsEnumerable().Reverse()); rows.Add(entry); } var returnResult = new ExtractionResult(details); returnResult.GenerateSegments(rows, null); returnResult.AnnotateSegments(); return(returnResult); } }
public bool TryExtract(string fullPath, Stream stream, string fileName, out ExtractionResult result) { var builder = new CleanTextBuilder(); var reader = new FilterReader(fullPath, fileName); if (reader.Filtered) { builder.Add(reader); result = new ExtractionResult() { ContentType = null, PlainContent = builder.ToString() }; return true; } result = null; return false; }
public static List <ActionDescription> ExtractActions(this ExtractionResult result) { var actions = new List <ActionDescription>(); foreach (var a in result.Sentences) { switch (a) { case SenI.Causes c: actions.Add(new ActionDescription(c.Action, c.Duration, c.Result, c.Condition)); break; case SenI.Releases r: actions.Add(new ActionDescription(r.Action, r.Duration, r.Result, r.Condition)); break; } } return(actions); }
public void GetDataFromInvoicesAndEmployeesMocks() { // Arrange var invoiceRepository = new Mock <IInvoiceRepository>(); var employeeRepository = new Mock <IEmployeeRepository>(); invoiceRepository.Setup(m => m.List(It.IsAny <DateTime>())).Returns(new List <Invoice>()); var service = new ExtractionService(invoiceRepository.Object, employeeRepository.Object, Console.Out); // Act ExtractionResult result = service.Extract(); // Assert invoiceRepository.VerifyAll(); employeeRepository.Verify(e => e.List(), Times.Never()); }
public ExtractionResult Extract() { var result = new ExtractionResult(); result.Invoices = _invoiceRepository.List(new DateTime(1996, 1, 1)); foreach (Invoice invoice in result.Invoices) { _outputStream.WriteLine("{0}-{1}-{2}", invoice.ShipperName, invoice.ShipAddress, invoice.ShipCity); } if (result.Invoices.Count() > 0) { result.Employees = _employeeRepository.List(); foreach (Employee employee in result.Employees) { _outputStream.WriteLine(employee); } } return(result); }
public bool TryExtract(string fullPath, Stream stream, string fileName, out ExtractionResult result) { var builder = new CleanTextBuilder(); var reader = new FilterReader(fullPath, fileName); if (reader.Filtered) { builder.Add(reader); result = new ExtractionResult() { ContentType = null, PlainContent = builder.ToString() }; return(true); } result = null; return(false); }
/// <summary> /// Extracts dynamic values from a formatted string. /// </summary> /// <param name="str">String including dynamic values</param> /// <param name="format">Format of the string</param> /// <param name="ignoreCase">True, to search case-insensitive.</param> public ExtractionResult Extract(string str, string format, bool ignoreCase = false) { var stringComparison = ignoreCase ? StringComparison.InvariantCultureIgnoreCase : StringComparison.InvariantCulture; if (str == format) //TODO: think on that! { return new ExtractionResult(true); } var formatTokens = new FormatStringTokenizer().Tokenize(format); if (formatTokens.IsNullOrEmpty()) { return new ExtractionResult(str == ""); } var result = new ExtractionResult(true); for (var i = 0; i < formatTokens.Count; i++) { var currentToken = formatTokens[i]; var previousToken = i > 0 ? formatTokens[i - 1] : null; if (currentToken.Type == FormatStringTokenType.ConstantText) { if (i == 0) { if (!str.StartsWith(currentToken.Text, stringComparison)) { result.IsMatch = false; return result; } str = str.Substring(currentToken.Text.Length); } else { var matchIndex = str.IndexOf(currentToken.Text, stringComparison); if (matchIndex < 0) { result.IsMatch = false; return result; } Debug.Assert(previousToken != null, "previousToken can not be null since i > 0 here"); result.Matches.Add(new NameValue(previousToken.Text, str.Substring(0, matchIndex))); str = str.Substring(matchIndex + currentToken.Text.Length); } } } var lastToken = formatTokens.Last(); if (lastToken.Type == FormatStringTokenType.DynamicValue) { result.Matches.Add(new NameValue(lastToken.Text, str)); } return result; }
public void Write(IExtractionArguments arguments, IExtractionResults extractionResults, string sourceSystem) { _log.DebugFormat("Writing extractionResults for {0}", extractionResults); var sourceSystemEntity = _dataContext.SourceSystems.FirstOrDefault(p => p.Name == sourceSystem); if (sourceSystemEntity == null) throw new ArgumentException("SourceSystem does not exist in the database", "sourceSystem"); var makeEntity = _dataContext.Makes.FirstOrDefault(p => p.MakeName == arguments.Make && p.SourceSystem == sourceSystemEntity); if (makeEntity == null) throw new ArgumentException("Make does not exist in the database", "arguments"); var modelEntity = makeEntity.Models.FirstOrDefault(p => p.ModelName == arguments.Model); if (modelEntity == null) throw new ArgumentException("Model does not exist in the database", "arguments"); var resultEntity = new ExtractionResult() { Model = modelEntity, From = arguments.From, To = arguments.To, ExtractionDateTime = DateTime.Now }; _dataContext.ExtractionResults.InsertOnSubmit(resultEntity); _dataContext.SubmitChanges(); foreach (var vehicle in extractionResults.Vehicles) { _dataContext.Vehicles.InsertOnSubmit(new Vehicle() { ExtractionResult = resultEntity, Milage = SafeInt( vehicle.Milage), Year = vehicle.Year, Price = Convert.ToDecimal( vehicle.Price), Title = vehicle.Title }); } _dataContext.SubmitChanges(); }
public PluginExtractionResult(ExtractionResult status,string message) { init(status, message,null); }
public PluginExtractionResult(ExtractionResult status,string message,Exception error) { init(status, message, error); }
private void init(ExtractionResult status, string message, Exception error) { this.Status = status; this.Message = message; this.Error = error; }