public void IndexSettingsCreatePropertyMappingForTopDocument(EntityContextInfo entityInfo, MappingDefinition mappingDefinition, ElasticJsonWriter elasticCrudJsonWriter) { var elasticSearchMapping = _elasticSerializerConfiguration.ElasticMappingResolver.GetElasticSearchMapping(entityInfo.EntityType); elasticSearchMapping.TraceProvider = _traceProvider; elasticSearchMapping.SaveChildObjectsAsWellAsParent = _elasticSerializerConfiguration.SaveChildObjectsAsWellAsParent; elasticSearchMapping.ProcessChildDocumentsAsSeparateChildIndex = _elasticSerializerConfiguration .ProcessChildDocumentsAsSeparateChildIndex; IndexCreateCreatePropertyMappingForEntityForParentDocument(entityInfo, elasticSearchMapping, mappingDefinition, elasticCrudJsonWriter); if (_elasticSerializerConfiguration.ProcessChildDocumentsAsSeparateChildIndex) { if (elasticSearchMapping.ChildIndexEntities.Count > 0) { // Only save the top level items now elasticSearchMapping.SaveChildObjectsAsWellAsParent = false; foreach (var item in elasticSearchMapping.ChildIndexEntities) { IndexCreateCreatePropertyMappingForChildDocument(elasticCrudJsonWriter, entityInfo, elasticSearchMapping, item, mappingDefinition); } } } elasticSearchMapping.ChildIndexEntities.Clear(); }
private void SaveCurrentMapping(string mappingFileName) { // TO DO - set drop down/mapping file values MappingDefinition currentMappingDefinition = new MappingDefinition(); currentMappingDefinition.DefinitionFile = ((ListBoxPair)DefinitionFileList.SelectedItem).Value; currentMappingDefinition.CompanyName = CompanyNameTextBox.Text.Trim(); currentMappingDefinition.MappingFilters = MappingFilterList; currentMappingDefinition.RequiresLevel2File = checkBoxLevel2.IsChecked ?? false; currentMappingDefinition.RequiresOptOutFile = checkBoxOptOut.IsChecked ?? false; currentMappingDefinition.CanHaveErrorFile = checkBoxErrorFile.IsChecked ?? false; MappingField mappingField = new MappingField(); for (int idx = 0; idx < SampleHeaderRow.Count(); idx++) { mappingField = new MappingField(); mappingField.SampleID = idx + 1; mappingField.SampleField = SampleHeaderRow[idx]; mappingField.SampleValue = SampleDataRow[idx]; var item = MappingSettingList[idx]; mappingField.MiddlewareFieldIdentifier = item.Key; currentMappingDefinition.MappingFields.Add(mappingField); } CSVConvertorFileManagement.SaveMappingFile(currentMappingDefinition, mappingFileName, CSVAppFolderMapping, ".xml"); }
public void CreateIndexSettingsAndMappingsForDocument(string index, IndexSettings indexSettings, IndexAliases indexAliases, IndexWarmers indexWarmers, EntityContextInfo entityInfo, MappingDefinition mappingDefinition) { if (_processedItems.Contains("_index" + index)) { return; } _processedItems.Add("_index" + index); var elasticCrudJsonWriter = new ElasticJsonWriter(); elasticCrudJsonWriter.JsonWriter.WriteStartObject(); CreateIndexSettings(elasticCrudJsonWriter, indexSettings); indexAliases.WriteJson(elasticCrudJsonWriter); indexWarmers.WriteJson(elasticCrudJsonWriter); IndexSettingsCreatePropertyMappingForTopDocument(entityInfo, mappingDefinition, elasticCrudJsonWriter); elasticCrudJsonWriter.JsonWriter.WriteEndObject(); CreateIndexCommand(elasticCrudJsonWriter.GetJsonString(), index); }
public void ConvertSimpleTypedTypeWithAutoMap() { var doc = Parsers.Delimitered.GetParser().SetHeaderRow(true) .Parse(TestFilePath); var data = doc.ConvertTo <TypedClass>(MappingDefinition.CreateMappingDefinition().AddMappingFromDocumentWithHeaders(doc)); Assert.IsTrue(data.Any()); }
public IndexDefinition() { // settings the default values IndexSettings = new IndexSettings { NumberOfReplicas = 1, NumberOfShards = 5 }; IndexAliases = new IndexAliases(); Mapping = new MappingDefinition(); }
private void SetAdditionFileDisplay(MappingDefinition currentMappingDefinition) { loadSelectedFile.IsEnabled = false; additionalFileStatusTable.DataContext = null; AdditionalFileStatusList = new ObservableCollection <AdditionalFileStatus>(); var radioButtons = additionalFiletypeGroup.Children.OfType <RadioButton>(); foreach (var rButton in radioButtons) { rButton.IsChecked = false; rButton.IsEnabled = false; } //textBlockAdditionalFileStatus.Text = string.Format("Level 2 records: {0}", currentMappingDefinition.Level2Records.Count()); selectedFileType.Text = "None"; errorsRadioButton.IsEnabled = currentMappingDefinition.CanHaveErrorFile; level2RadioButton.IsEnabled = currentMappingDefinition.RequiresLevel2File; optoutChangesRadioButton.IsEnabled = currentMappingDefinition.RequiresOptOutFile; optoutNewRadioButton.IsEnabled = currentMappingDefinition.RequiresOptOutFile; AdditionalFileStatus additionalFileStatus = new AdditionalFileStatus(); if (currentMappingDefinition.RequiresLevel2File) { level2RadioButton.IsEnabled = true; additionalFileStatus = new AdditionalFileStatus(); additionalFileStatus.FileType = "Level 2"; additionalFileStatus.RowCount = currentMappingDefinition.Level2Records.Count(); additionalFileStatus.DateLastUpdated = currentMappingDefinition.DateLevel2LastLoaded == DateTime.MinValue?"-": currentMappingDefinition.DateLevel2LastLoaded.ToShortDateString(); AdditionalFileStatusList.Add(additionalFileStatus); } if (currentMappingDefinition.RequiresOptOutFile) { optoutNewRadioButton.IsEnabled = true; additionalFileStatus = new AdditionalFileStatus(); additionalFileStatus.FileType = "Opt out file"; additionalFileStatus.RowCount = currentMappingDefinition.OptOutRecords.Count(); additionalFileStatus.DateLastUpdated = currentMappingDefinition.DateOptOutRecordsLastLoaded == DateTime.MinValue ? "-" : currentMappingDefinition.DateOptOutRecordsLastLoaded.ToShortDateString(); AdditionalFileStatusList.Add(additionalFileStatus); } if (currentMappingDefinition.CanHaveErrorFile) { errorsRadioButton.IsEnabled = true; additionalFileStatus = new AdditionalFileStatus(); additionalFileStatus.FileType = "Errors file"; additionalFileStatus.RowCount = currentMappingDefinition.FileErrorRecords.Count(); additionalFileStatus.DateLastUpdated = currentMappingDefinition.DateErrorRecordsLastLoaded == DateTime.MinValue ? "-" : currentMappingDefinition.DateErrorRecordsLastLoaded.ToShortDateString(); AdditionalFileStatusList.Add(additionalFileStatus); } additionalFileStatusTable.AutoGenerateColumns = true; additionalFileStatusTable.ItemsSource = AdditionalFileStatusList; }
public static MappingDefinition ReadMappingFile(string fileName, string applicationFolder, string definitionExtension) { MappingDefinition mappingFields = new MappingDefinition(); XmlSerializer serializer = new XmlSerializer(typeof(MappingDefinition)); string path = Path.Combine(applicationFolder, MappingSubDirectory, fileName + definitionExtension); StreamReader reader = new StreamReader(path); mappingFields = (MappingDefinition)serializer.Deserialize(reader); mappingFields.HasAdditionalFile = mappingFields.CanHaveErrorFile || mappingFields.RequiresLevel2File || mappingFields.RequiresOptOutFile; reader.Close(); return(mappingFields); }
/// <summary> /// Create a new mapping for the child type in the parent document index /// </summary> /// <param name="entityInfo"> </param> /// <param name="elasticMapping"> </param> /// <param name="item"> </param> /// <param name="mappingDefinition"> definition for the type mappings </param> private void CreatePropertyMappingForChildDocument(EntityContextInfo entityInfo, ElasticMapping elasticMapping, EntityContextInfo item, MappingDefinition mappingDefinition) { var childMapping = _elasticSerializerConfiguration.ElasticMappingResolver.GetElasticSearchMapping(item.EntityType); var parentMapping = _elasticSerializerConfiguration.ElasticMappingResolver.GetElasticSearchMapping(item.ParentEntityType); var childType = childMapping.GetDocumentType(item.EntityType); var parentType = parentMapping.GetDocumentType(item.ParentEntityType); var processedId = childType + "_" + parentType; if (_processedItems.Contains(childType)) { var test = CommandTypes.Find(t => t.StartsWith(childType)); if (test != processedId) { throw new ElasticException("InitMappings: Not supported, child documents can only have one parent"); } return; } _processedItems.Add(childType); CommandTypes.Add(processedId); var elasticCrudJsonWriter = new ElasticJsonWriter(); elasticCrudJsonWriter.JsonWriter.WriteStartObject(); elasticCrudJsonWriter.JsonWriter.WritePropertyName(childType); elasticCrudJsonWriter.JsonWriter.WriteStartObject(); mappingDefinition.Source.WriteJson(elasticCrudJsonWriter); mappingDefinition.All.WriteJson(elasticCrudJsonWriter); CreateParentMappingForDocument( elasticCrudJsonWriter, elasticMapping.GetDocumentType(item.ParentEntityType)); if (item.RoutingDefinition.RoutingId != null && _elasticSerializerConfiguration.UserDefinedRouting) { CreateForceRoutingMappingForDocument(elasticCrudJsonWriter); } ProccessPropertyMappingsWithoutTypeName(elasticCrudJsonWriter, item, childMapping); elasticCrudJsonWriter.JsonWriter.WriteEndObject(); elasticCrudJsonWriter.JsonWriter.WriteEndObject(); CreateMappingCommandForTypeWithExistingIndex(elasticCrudJsonWriter.GetJsonString(), elasticMapping.GetIndexForType(entityInfo.EntityType), childMapping.GetDocumentType(item.EntityType)); }
public static bool SaveMappingFile(MappingDefinition mappingDefinition, string fileName, string applicationFolder, string definitionExtension) { XmlSerializer serializer = new XmlSerializer(typeof(MappingDefinition)); Directory.CreateDirectory(Path.Combine(applicationFolder, MappingSubDirectory)); string path = Path.Combine(applicationFolder, MappingSubDirectory, fileName + definitionExtension); StreamWriter writer = new StreamWriter(path, false); serializer.Serialize(writer, mappingDefinition); writer.Close(); return(true); }
private void MappingFileList_SelectionChanged(object sender, SelectionChangedEventArgs e) { if (MappingFileList.SelectedIndex >= 1) { CurrentMappingFilename = ((ListBoxPair)MappingFileList.SelectedItem).Value; CurrentMappingDefinition = CSVConvertorFileManagement.ReadMappingFile(CurrentMappingFilename, CSVAppFolder, ".xml"); if (CurrentMappingDefinition.HasAdditionalFile) { SetAdditionFileDisplay(CurrentMappingDefinition); AdditionalFilePanel.Visibility = Visibility.Visible; } else { AdditionalFilePanel.Visibility = Visibility.Hidden; //textBlockAdditionalFileStatus.Text = string.Empty; } } }
private void ConvertFile_Click(object sender, RoutedEventArgs ev) { // check everything is ready, file to process, destination and mapping file if (OriginalFileName == string.Empty || !File.Exists(OriginalFile.Text.Trim())) { MessageBox.Show("Please select a file to process"); return; } if (MappingFileList.SelectedIndex < 1) { MessageBox.Show("Please select a mapping"); return; } string temp = DestinationLocation.Text.Trim(); if (temp == string.Empty || !Directory.Exists(temp)) { MessageBox.Show("Please select a destination location"); return; } CurrentMappingDefinition = CSVConvertorFileManagement.ReadMappingFile(((ListBoxPair)MappingFileList.SelectedItem).Value, CSVAppFolder, ".xml"); // finally, check mapping file for level 2 records if (CurrentMappingDefinition.RequiresLevel2File && CurrentMappingDefinition.Level2Records.Count() == 0) { MessageBox.Show("There are no level 2 records loaded"); return; } int rowsProcessed = ConvertCSVFile(OriginalFile.Text.Trim(), FirstRowHeadersCheckBox.IsChecked ?? false); if (rowsProcessed < 0) { return; } { MessageBox.Show("Processing complete, rows output: " + rowsProcessed); } }
public async Task <ResultDetails <string> > CreateTypeMappingForIndexAsync <T>(MappingDefinition mappingDefinition) { if (mappingDefinition == null) { throw new ElasticsearchCrudException("CreateTypeMappingForIndexAsync: A mapping definition with the parent index is required"); } _traceProvider.Trace(TraceEventType.Verbose, "{0}: CreateTypeMappingForIndex Elasticsearch started", "ElasticsearchContextIndexMapping"); var resultDetails = new ResultDetails <string> { Status = HttpStatusCode.InternalServerError }; try { var indexMappings = new IndexMappings(_traceProvider, _elasticsearchSerializerConfiguration); var item = Activator.CreateInstance <T>(); var entityContextInfo = new EntityContextInfo { RoutingDefinition = mappingDefinition.RoutingDefinition, Document = item, EntityType = typeof(T), Id = "0" }; if (string.IsNullOrEmpty(mappingDefinition.Index)) { string index = _elasticsearchSerializerConfiguration.ElasticsearchMappingResolver.GetElasticSearchMapping(entityContextInfo.EntityType).GetIndexForType(entityContextInfo.EntityType); MappingUtils.GuardAgainstBadIndexName(index); mappingDefinition.Index = index; } indexMappings.CreatePropertyMappingForTopDocument(entityContextInfo, mappingDefinition); await indexMappings.Execute(_client, _connectionString, _traceProvider, _cancellationTokenSource); return(resultDetails); } catch (OperationCanceledException oex) { _traceProvider.Trace(TraceEventType.Warning, oex, "{1}: CreateTypeMappingForIndexAsync Request OperationCanceledException: {0}", oex.Message, "ElasticsearchContextIndexMapping"); resultDetails.Description = "OperationCanceledException"; return(resultDetails); } }
public ResultDetails <string> CreateTypeMappingForIndex <T>(MappingDefinition mappingDefinition) { var syncExecutor = new SyncExecute(_traceProvider); return(syncExecutor.ExecuteResultDetails(() => CreateTypeMappingForIndexAsync <T>(mappingDefinition))); }
/// <summary> /// Create a new index for the parent document /// </summary> /// <param name="entityInfo"> </param> /// <param name="elasticMapping"> </param> /// <param name="mappingDefinition"> mapping definitions for the index type </param> private void IndexCreateCreatePropertyMappingForEntityForParentDocument(EntityContextInfo entityInfo, ElasticMapping elasticMapping, MappingDefinition mappingDefinition, ElasticJsonWriter elasticCrudJsonWriter) { var itemType = elasticMapping.GetDocumentType(entityInfo.EntityType); if (_processedItems.Contains("_mapping" + itemType)) { return; } _processedItems.Add("_mapping" + itemType); //elasticCrudJsonWriter.JsonWriter.WriteStartObject(); elasticCrudJsonWriter.JsonWriter.WritePropertyName("mappings"); elasticCrudJsonWriter.JsonWriter.WriteStartObject(); elasticCrudJsonWriter.JsonWriter.WritePropertyName(itemType); elasticCrudJsonWriter.JsonWriter.WriteStartObject(); mappingDefinition.Source.WriteJson(elasticCrudJsonWriter); mappingDefinition.All.WriteJson(elasticCrudJsonWriter); if (entityInfo.RoutingDefinition.RoutingId != null && _elasticSerializerConfiguration.UserDefinedRouting) { CreateForceRoutingMappingForDocument(elasticCrudJsonWriter); } if (entityInfo.RoutingDefinition.ParentId != null) { CreateParentMappingForDocument( elasticCrudJsonWriter, elasticMapping.GetDocumentType(entityInfo.ParentEntityType)); } ProccessPropertyMappingsWithoutTypeName(elasticCrudJsonWriter, entityInfo, elasticMapping); elasticCrudJsonWriter.JsonWriter.WriteEndObject(); if (_elasticSerializerConfiguration.ProcessChildDocumentsAsSeparateChildIndex) { if (elasticMapping.ChildIndexEntities.Count > 0) { elasticMapping.SaveChildObjectsAsWellAsParent = false; foreach (var item in elasticMapping.ChildIndexEntities) { IndexCreateCreatePropertyMappingForChildDocument(elasticCrudJsonWriter, entityInfo, elasticMapping, item, mappingDefinition); } } } elasticCrudJsonWriter.JsonWriter.WriteEndObject(); //elasticCrudJsonWriter.JsonWriter.WriteEndObject(); //CreateMappingCommandForTypeWithExistingIndex(elasticCrudJsonWriter.GetJsonString(), mappingDefinition.Index, itemType); }
public void SetUp() { mSut = new MappingDefinition(); }
/// <summary> /// Create a new index for the parent document /// </summary> /// <param name="entityInfo"></param> /// <param name="elasticsearchMapping"></param> /// <param name="mappingDefinition">mapping definitions for the index type</param> private void CreatePropertyMappingForEntityForParentDocument(EntityContextInfo entityInfo, ElasticsearchMapping elasticsearchMapping, MappingDefinition mappingDefinition) { var itemType = elasticsearchMapping.GetDocumentType(entityInfo.EntityType); if (_processedItems.Contains("_mapping" + itemType)) { return; } _processedItems.Add("_mapping" + itemType); var elasticsearchCrudJsonWriter = new ElasticsearchCrudJsonWriter(); elasticsearchCrudJsonWriter.JsonWriter.WriteStartObject(); elasticsearchCrudJsonWriter.JsonWriter.WritePropertyName(itemType); elasticsearchCrudJsonWriter.JsonWriter.WriteStartObject(); mappingDefinition.Source.WriteJson(elasticsearchCrudJsonWriter); mappingDefinition.All.WriteJson(elasticsearchCrudJsonWriter); mappingDefinition.Analyzer.WriteJson(elasticsearchCrudJsonWriter); if (entityInfo.RoutingDefinition.RoutingId != null && _elasticsearchSerializerConfiguration.UserDefinedRouting) { CreateForceRoutingMappingForDocument(elasticsearchCrudJsonWriter); } if (entityInfo.RoutingDefinition.ParentId != null) { CreateParentMappingForDocument( elasticsearchCrudJsonWriter, elasticsearchMapping.GetDocumentType(entityInfo.ParentEntityType)); } ProccessPropertyMappingsWithoutTypeName(elasticsearchCrudJsonWriter, entityInfo, elasticsearchMapping); elasticsearchCrudJsonWriter.JsonWriter.WriteEndObject(); elasticsearchCrudJsonWriter.JsonWriter.WriteEndObject(); CreateMappingCommandForTypeWithExistingIndex(elasticsearchCrudJsonWriter.GetJsonString(), mappingDefinition.Index, itemType); }
private void SetupExistingMapping() { // load from file if (MappingFileList.SelectedIndex < 1) { MessageBox.Show("No mapping selected"); return; } MappingDefinition mappingDefinition = CSVConvertorFileManagement.ReadMappingFile(((ListBoxPair)MappingFileList.SelectedItem).Value, CSVAppFolderMapping, ".xml"); // file/combo settings MappingNameTextBox.Text = ((ListBoxPair)MappingFileList.SelectedItem).Value; DefinitionFileList.SelectedValue = mappingDefinition.DefinitionFile; LoadMiddlewareDefinition(mappingDefinition.DefinitionFile); MappingSettingList = new ObservableCollection <ListBoxPair>(); ListBoxPair listBoxPair = new ListBoxPair(); int mappingFieldID = 0; string middlewareFieldIdentifier = string.Empty; SampleHeaderRow = new List <string>(); SampleDataRow = new List <string>(); foreach (MappingField mappingField in mappingDefinition.MappingFields) { int idx = mappingField.SampleID; // set sample value and restore Sample and Header arrays SampleHeaderRow.Add(mappingField.SampleField); SampleDataRow.Add(mappingField.SampleValue); middlewareFieldIdentifier = mappingField.MiddlewareFieldIdentifier; listBoxPair = new ListBoxPair(); if (Int32.TryParse(mappingField.MiddlewareFieldIdentifier, out mappingFieldID)) { // convention is that unset mapping has negative integer ID // if mapping set then delete from DefintionFields and Add to MappingFields listBoxPair = new ListBoxPair(((idx + 1) * -1).ToString(), "-"); } else { // or set actual mapping and remove from definition list listBoxPair = DefinitionFields.FirstOrDefault(x => x.Key == middlewareFieldIdentifier); DefinitionFields.Remove(listBoxPair); } MappingSettingList.Add(listBoxPair); } SetDefinitionListBox(); SetMappingList(); ClearFilterSettings(); DisplaySampleFile(); SetFilterFieldNames(); CompanyNameTextBox.Text = mappingDefinition.CompanyName; MappingFilterList = mappingDefinition.MappingFilters; checkBoxLevel2.IsChecked = mappingDefinition.RequiresLevel2File; checkBoxOptOut.IsChecked = mappingDefinition.RequiresOptOutFile; checkBoxErrorFile.IsChecked = mappingDefinition.CanHaveErrorFile; DisplayCurrentFilterList(); SetScrollViewer(); }
public CSVDataProcesses(MappingDefinition mappingDefinition) { MappingDefinition = mappingDefinition; }