public void IndexSettingsCreatePropertyMappingForTopDocument(EntityContextInfo entityInfo,
MappingDefinition mappingDefinition, ElasticJsonWriter elasticCrudJsonWriter)
{
var elasticSearchMapping =
_elasticSerializerConfiguration.ElasticMappingResolver.GetElasticSearchMapping(entityInfo.EntityType);
elasticSearchMapping.TraceProvider = _traceProvider;
elasticSearchMapping.SaveChildObjectsAsWellAsParent =
_elasticSerializerConfiguration.SaveChildObjectsAsWellAsParent;
elasticSearchMapping.ProcessChildDocumentsAsSeparateChildIndex = _elasticSerializerConfiguration
.ProcessChildDocumentsAsSeparateChildIndex;
IndexCreateCreatePropertyMappingForEntityForParentDocument(entityInfo, elasticSearchMapping,
mappingDefinition, elasticCrudJsonWriter);
if (_elasticSerializerConfiguration.ProcessChildDocumentsAsSeparateChildIndex)
{
if (elasticSearchMapping.ChildIndexEntities.Count > 0)
{
// Only save the top level items now
elasticSearchMapping.SaveChildObjectsAsWellAsParent = false;
foreach (var item in elasticSearchMapping.ChildIndexEntities)
{
IndexCreateCreatePropertyMappingForChildDocument(elasticCrudJsonWriter, entityInfo,
elasticSearchMapping, item, mappingDefinition);
}
}
}
elasticSearchMapping.ChildIndexEntities.Clear();
}
private void SaveCurrentMapping(string mappingFileName)
{
// TO DO - set drop down/mapping file values
MappingDefinition currentMappingDefinition = new MappingDefinition();
currentMappingDefinition.DefinitionFile = ((ListBoxPair)DefinitionFileList.SelectedItem).Value;
currentMappingDefinition.CompanyName = CompanyNameTextBox.Text.Trim();
currentMappingDefinition.MappingFilters = MappingFilterList;
currentMappingDefinition.RequiresLevel2File = checkBoxLevel2.IsChecked ?? false;
currentMappingDefinition.RequiresOptOutFile = checkBoxOptOut.IsChecked ?? false;
currentMappingDefinition.CanHaveErrorFile = checkBoxErrorFile.IsChecked ?? false;
MappingField mappingField = new MappingField();
for (int idx = 0; idx < SampleHeaderRow.Count(); idx++)
{
mappingField = new MappingField();
mappingField.SampleID = idx + 1;
mappingField.SampleField = SampleHeaderRow[idx];
mappingField.SampleValue = SampleDataRow[idx];
var item = MappingSettingList[idx];
mappingField.MiddlewareFieldIdentifier = item.Key;
currentMappingDefinition.MappingFields.Add(mappingField);
}
CSVConvertorFileManagement.SaveMappingFile(currentMappingDefinition, mappingFileName, CSVAppFolderMapping, ".xml");
}
public void CreateIndexSettingsAndMappingsForDocument(string index,
IndexSettings indexSettings,
IndexAliases indexAliases,
IndexWarmers indexWarmers,
EntityContextInfo entityInfo,
MappingDefinition mappingDefinition)
{
if (_processedItems.Contains("_index" + index))
{
return;
}
_processedItems.Add("_index" + index);
var elasticCrudJsonWriter = new ElasticJsonWriter();
elasticCrudJsonWriter.JsonWriter.WriteStartObject();
CreateIndexSettings(elasticCrudJsonWriter, indexSettings);
indexAliases.WriteJson(elasticCrudJsonWriter);
indexWarmers.WriteJson(elasticCrudJsonWriter);
IndexSettingsCreatePropertyMappingForTopDocument(entityInfo, mappingDefinition, elasticCrudJsonWriter);
elasticCrudJsonWriter.JsonWriter.WriteEndObject();
CreateIndexCommand(elasticCrudJsonWriter.GetJsonString(), index);
}
public void ConvertSimpleTypedTypeWithAutoMap()
{
var doc = Parsers.Delimitered.GetParser().SetHeaderRow(true)
.Parse(TestFilePath);
var data = doc.ConvertTo <TypedClass>(MappingDefinition.CreateMappingDefinition().AddMappingFromDocumentWithHeaders(doc));
Assert.IsTrue(data.Any());
}
public IndexDefinition()
{
// settings the default values
IndexSettings = new IndexSettings {
NumberOfReplicas = 1, NumberOfShards = 5
};
IndexAliases = new IndexAliases();
Mapping = new MappingDefinition();
}
private void SetAdditionFileDisplay(MappingDefinition currentMappingDefinition)
{
loadSelectedFile.IsEnabled = false;
additionalFileStatusTable.DataContext = null;
AdditionalFileStatusList = new ObservableCollection <AdditionalFileStatus>();
var radioButtons = additionalFiletypeGroup.Children.OfType <RadioButton>();
foreach (var rButton in radioButtons)
{
rButton.IsChecked = false;
rButton.IsEnabled = false;
}
//textBlockAdditionalFileStatus.Text = string.Format("Level 2 records: {0}", currentMappingDefinition.Level2Records.Count());
selectedFileType.Text = "None";
errorsRadioButton.IsEnabled = currentMappingDefinition.CanHaveErrorFile;
level2RadioButton.IsEnabled = currentMappingDefinition.RequiresLevel2File;
optoutChangesRadioButton.IsEnabled = currentMappingDefinition.RequiresOptOutFile;
optoutNewRadioButton.IsEnabled = currentMappingDefinition.RequiresOptOutFile;
AdditionalFileStatus additionalFileStatus = new AdditionalFileStatus();
if (currentMappingDefinition.RequiresLevel2File)
{
level2RadioButton.IsEnabled = true;
additionalFileStatus = new AdditionalFileStatus();
additionalFileStatus.FileType = "Level 2";
additionalFileStatus.RowCount = currentMappingDefinition.Level2Records.Count();
additionalFileStatus.DateLastUpdated = currentMappingDefinition.DateLevel2LastLoaded == DateTime.MinValue?"-": currentMappingDefinition.DateLevel2LastLoaded.ToShortDateString();
AdditionalFileStatusList.Add(additionalFileStatus);
}
if (currentMappingDefinition.RequiresOptOutFile)
{
optoutNewRadioButton.IsEnabled = true;
additionalFileStatus = new AdditionalFileStatus();
additionalFileStatus.FileType = "Opt out file";
additionalFileStatus.RowCount = currentMappingDefinition.OptOutRecords.Count();
additionalFileStatus.DateLastUpdated = currentMappingDefinition.DateOptOutRecordsLastLoaded == DateTime.MinValue ? "-" : currentMappingDefinition.DateOptOutRecordsLastLoaded.ToShortDateString();
AdditionalFileStatusList.Add(additionalFileStatus);
}
if (currentMappingDefinition.CanHaveErrorFile)
{
errorsRadioButton.IsEnabled = true;
additionalFileStatus = new AdditionalFileStatus();
additionalFileStatus.FileType = "Errors file";
additionalFileStatus.RowCount = currentMappingDefinition.FileErrorRecords.Count();
additionalFileStatus.DateLastUpdated = currentMappingDefinition.DateErrorRecordsLastLoaded == DateTime.MinValue ? "-" : currentMappingDefinition.DateErrorRecordsLastLoaded.ToShortDateString();
AdditionalFileStatusList.Add(additionalFileStatus);
}
additionalFileStatusTable.AutoGenerateColumns = true;
additionalFileStatusTable.ItemsSource = AdditionalFileStatusList;
}
public static MappingDefinition ReadMappingFile(string fileName, string applicationFolder, string definitionExtension)
{
MappingDefinition mappingFields = new MappingDefinition();
XmlSerializer serializer = new XmlSerializer(typeof(MappingDefinition));
string path = Path.Combine(applicationFolder, MappingSubDirectory, fileName + definitionExtension);
StreamReader reader = new StreamReader(path);
mappingFields = (MappingDefinition)serializer.Deserialize(reader);
mappingFields.HasAdditionalFile = mappingFields.CanHaveErrorFile || mappingFields.RequiresLevel2File || mappingFields.RequiresOptOutFile;
reader.Close();
return(mappingFields);
}
/// <summary>
/// Create a new mapping for the child type in the parent document index
/// </summary>
/// <param name="entityInfo"> </param>
/// <param name="elasticMapping"> </param>
/// <param name="item"> </param>
/// <param name="mappingDefinition"> definition for the type mappings </param>
private void CreatePropertyMappingForChildDocument(EntityContextInfo entityInfo, ElasticMapping elasticMapping,
EntityContextInfo item, MappingDefinition mappingDefinition)
{
var childMapping =
_elasticSerializerConfiguration.ElasticMappingResolver.GetElasticSearchMapping(item.EntityType);
var parentMapping =
_elasticSerializerConfiguration.ElasticMappingResolver.GetElasticSearchMapping(item.ParentEntityType);
var childType = childMapping.GetDocumentType(item.EntityType);
var parentType = parentMapping.GetDocumentType(item.ParentEntityType);
var processedId = childType + "_" + parentType;
if (_processedItems.Contains(childType))
{
var test = CommandTypes.Find(t => t.StartsWith(childType));
if (test != processedId)
{
throw new ElasticException("InitMappings: Not supported, child documents can only have one parent");
}
return;
}
_processedItems.Add(childType);
CommandTypes.Add(processedId);
var elasticCrudJsonWriter = new ElasticJsonWriter();
elasticCrudJsonWriter.JsonWriter.WriteStartObject();
elasticCrudJsonWriter.JsonWriter.WritePropertyName(childType);
elasticCrudJsonWriter.JsonWriter.WriteStartObject();
mappingDefinition.Source.WriteJson(elasticCrudJsonWriter);
mappingDefinition.All.WriteJson(elasticCrudJsonWriter);
CreateParentMappingForDocument(
elasticCrudJsonWriter,
elasticMapping.GetDocumentType(item.ParentEntityType));
if (item.RoutingDefinition.RoutingId != null && _elasticSerializerConfiguration.UserDefinedRouting)
{
CreateForceRoutingMappingForDocument(elasticCrudJsonWriter);
}
ProccessPropertyMappingsWithoutTypeName(elasticCrudJsonWriter, item, childMapping);
elasticCrudJsonWriter.JsonWriter.WriteEndObject();
elasticCrudJsonWriter.JsonWriter.WriteEndObject();
CreateMappingCommandForTypeWithExistingIndex(elasticCrudJsonWriter.GetJsonString(),
elasticMapping.GetIndexForType(entityInfo.EntityType), childMapping.GetDocumentType(item.EntityType));
}
public static bool SaveMappingFile(MappingDefinition mappingDefinition, string fileName, string applicationFolder, string definitionExtension)
{
XmlSerializer serializer = new XmlSerializer(typeof(MappingDefinition));
Directory.CreateDirectory(Path.Combine(applicationFolder, MappingSubDirectory));
string path = Path.Combine(applicationFolder, MappingSubDirectory, fileName + definitionExtension);
StreamWriter writer = new StreamWriter(path, false);
serializer.Serialize(writer, mappingDefinition);
writer.Close();
return(true);
}
private void MappingFileList_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
if (MappingFileList.SelectedIndex >= 1)
{
CurrentMappingFilename = ((ListBoxPair)MappingFileList.SelectedItem).Value;
CurrentMappingDefinition = CSVConvertorFileManagement.ReadMappingFile(CurrentMappingFilename, CSVAppFolder, ".xml");
if (CurrentMappingDefinition.HasAdditionalFile)
{
SetAdditionFileDisplay(CurrentMappingDefinition);
AdditionalFilePanel.Visibility = Visibility.Visible;
}
else
{
AdditionalFilePanel.Visibility = Visibility.Hidden;
//textBlockAdditionalFileStatus.Text = string.Empty;
}
}
}
private void ConvertFile_Click(object sender, RoutedEventArgs ev)
{
// check everything is ready, file to process, destination and mapping file
if (OriginalFileName == string.Empty || !File.Exists(OriginalFile.Text.Trim()))
{
MessageBox.Show("Please select a file to process");
return;
}
if (MappingFileList.SelectedIndex < 1)
{
MessageBox.Show("Please select a mapping");
return;
}
string temp = DestinationLocation.Text.Trim();
if (temp == string.Empty || !Directory.Exists(temp))
{
MessageBox.Show("Please select a destination location");
return;
}
CurrentMappingDefinition = CSVConvertorFileManagement.ReadMappingFile(((ListBoxPair)MappingFileList.SelectedItem).Value, CSVAppFolder, ".xml");
// finally, check mapping file for level 2 records
if (CurrentMappingDefinition.RequiresLevel2File && CurrentMappingDefinition.Level2Records.Count() == 0)
{
MessageBox.Show("There are no level 2 records loaded");
return;
}
int rowsProcessed = ConvertCSVFile(OriginalFile.Text.Trim(), FirstRowHeadersCheckBox.IsChecked ?? false);
if (rowsProcessed < 0)
{
return;
}
{
MessageBox.Show("Processing complete, rows output: " + rowsProcessed);
}
}
public async Task <ResultDetails <string> > CreateTypeMappingForIndexAsync <T>(MappingDefinition mappingDefinition)
{
if (mappingDefinition == null)
{
throw new ElasticsearchCrudException("CreateTypeMappingForIndexAsync: A mapping definition with the parent index is required");
}
_traceProvider.Trace(TraceEventType.Verbose, "{0}: CreateTypeMappingForIndex Elasticsearch started", "ElasticsearchContextIndexMapping");
var resultDetails = new ResultDetails <string> {
Status = HttpStatusCode.InternalServerError
};
try
{
var indexMappings = new IndexMappings(_traceProvider, _elasticsearchSerializerConfiguration);
var item = Activator.CreateInstance <T>();
var entityContextInfo = new EntityContextInfo
{
RoutingDefinition = mappingDefinition.RoutingDefinition,
Document = item,
EntityType = typeof(T),
Id = "0"
};
if (string.IsNullOrEmpty(mappingDefinition.Index))
{
string index = _elasticsearchSerializerConfiguration.ElasticsearchMappingResolver.GetElasticSearchMapping(entityContextInfo.EntityType).GetIndexForType(entityContextInfo.EntityType);
MappingUtils.GuardAgainstBadIndexName(index);
mappingDefinition.Index = index;
}
indexMappings.CreatePropertyMappingForTopDocument(entityContextInfo, mappingDefinition);
await indexMappings.Execute(_client, _connectionString, _traceProvider, _cancellationTokenSource);
return(resultDetails);
}
catch (OperationCanceledException oex)
{
_traceProvider.Trace(TraceEventType.Warning, oex, "{1}: CreateTypeMappingForIndexAsync Request OperationCanceledException: {0}", oex.Message,
"ElasticsearchContextIndexMapping");
resultDetails.Description = "OperationCanceledException";
return(resultDetails);
}
}
public ResultDetails <string> CreateTypeMappingForIndex <T>(MappingDefinition mappingDefinition)
{
var syncExecutor = new SyncExecute(_traceProvider);
return(syncExecutor.ExecuteResultDetails(() => CreateTypeMappingForIndexAsync <T>(mappingDefinition)));
}
/// <summary>
/// Create a new index for the parent document
/// </summary>
/// <param name="entityInfo"> </param>
/// <param name="elasticMapping"> </param>
/// <param name="mappingDefinition"> mapping definitions for the index type </param>
private void IndexCreateCreatePropertyMappingForEntityForParentDocument(EntityContextInfo entityInfo,
ElasticMapping elasticMapping, MappingDefinition mappingDefinition, ElasticJsonWriter elasticCrudJsonWriter)
{
var itemType = elasticMapping.GetDocumentType(entityInfo.EntityType);
if (_processedItems.Contains("_mapping" + itemType))
{
return;
}
_processedItems.Add("_mapping" + itemType);
//elasticCrudJsonWriter.JsonWriter.WriteStartObject();
elasticCrudJsonWriter.JsonWriter.WritePropertyName("mappings");
elasticCrudJsonWriter.JsonWriter.WriteStartObject();
elasticCrudJsonWriter.JsonWriter.WritePropertyName(itemType);
elasticCrudJsonWriter.JsonWriter.WriteStartObject();
mappingDefinition.Source.WriteJson(elasticCrudJsonWriter);
mappingDefinition.All.WriteJson(elasticCrudJsonWriter);
if (entityInfo.RoutingDefinition.RoutingId != null && _elasticSerializerConfiguration.UserDefinedRouting)
{
CreateForceRoutingMappingForDocument(elasticCrudJsonWriter);
}
if (entityInfo.RoutingDefinition.ParentId != null)
{
CreateParentMappingForDocument(
elasticCrudJsonWriter,
elasticMapping.GetDocumentType(entityInfo.ParentEntityType));
}
ProccessPropertyMappingsWithoutTypeName(elasticCrudJsonWriter, entityInfo, elasticMapping);
elasticCrudJsonWriter.JsonWriter.WriteEndObject();
if (_elasticSerializerConfiguration.ProcessChildDocumentsAsSeparateChildIndex)
{
if (elasticMapping.ChildIndexEntities.Count > 0)
{
elasticMapping.SaveChildObjectsAsWellAsParent = false;
foreach (var item in elasticMapping.ChildIndexEntities)
{
IndexCreateCreatePropertyMappingForChildDocument(elasticCrudJsonWriter, entityInfo,
elasticMapping, item, mappingDefinition);
}
}
}
elasticCrudJsonWriter.JsonWriter.WriteEndObject();
//elasticCrudJsonWriter.JsonWriter.WriteEndObject();
//CreateMappingCommandForTypeWithExistingIndex(elasticCrudJsonWriter.GetJsonString(), mappingDefinition.Index, itemType);
}
public void SetUp()
{
mSut = new MappingDefinition();
}
/// <summary>
/// Create a new index for the parent document
/// </summary>
/// <param name="entityInfo"></param>
/// <param name="elasticsearchMapping"></param>
/// <param name="mappingDefinition">mapping definitions for the index type</param>
private void CreatePropertyMappingForEntityForParentDocument(EntityContextInfo entityInfo, ElasticsearchMapping elasticsearchMapping, MappingDefinition mappingDefinition)
{
var itemType = elasticsearchMapping.GetDocumentType(entityInfo.EntityType);
if (_processedItems.Contains("_mapping" + itemType))
{
return;
}
_processedItems.Add("_mapping" + itemType);
var elasticsearchCrudJsonWriter = new ElasticsearchCrudJsonWriter();
elasticsearchCrudJsonWriter.JsonWriter.WriteStartObject();
elasticsearchCrudJsonWriter.JsonWriter.WritePropertyName(itemType);
elasticsearchCrudJsonWriter.JsonWriter.WriteStartObject();
mappingDefinition.Source.WriteJson(elasticsearchCrudJsonWriter);
mappingDefinition.All.WriteJson(elasticsearchCrudJsonWriter);
mappingDefinition.Analyzer.WriteJson(elasticsearchCrudJsonWriter);
if (entityInfo.RoutingDefinition.RoutingId != null && _elasticsearchSerializerConfiguration.UserDefinedRouting)
{
CreateForceRoutingMappingForDocument(elasticsearchCrudJsonWriter);
}
if (entityInfo.RoutingDefinition.ParentId != null)
{
CreateParentMappingForDocument(
elasticsearchCrudJsonWriter,
elasticsearchMapping.GetDocumentType(entityInfo.ParentEntityType));
}
ProccessPropertyMappingsWithoutTypeName(elasticsearchCrudJsonWriter, entityInfo, elasticsearchMapping);
elasticsearchCrudJsonWriter.JsonWriter.WriteEndObject();
elasticsearchCrudJsonWriter.JsonWriter.WriteEndObject();
CreateMappingCommandForTypeWithExistingIndex(elasticsearchCrudJsonWriter.GetJsonString(), mappingDefinition.Index, itemType);
}
private void SetupExistingMapping()
{
// load from file
if (MappingFileList.SelectedIndex < 1)
{
MessageBox.Show("No mapping selected");
return;
}
MappingDefinition mappingDefinition = CSVConvertorFileManagement.ReadMappingFile(((ListBoxPair)MappingFileList.SelectedItem).Value, CSVAppFolderMapping, ".xml");
// file/combo settings
MappingNameTextBox.Text = ((ListBoxPair)MappingFileList.SelectedItem).Value;
DefinitionFileList.SelectedValue = mappingDefinition.DefinitionFile;
LoadMiddlewareDefinition(mappingDefinition.DefinitionFile);
MappingSettingList = new ObservableCollection <ListBoxPair>();
ListBoxPair listBoxPair = new ListBoxPair();
int mappingFieldID = 0;
string middlewareFieldIdentifier = string.Empty;
SampleHeaderRow = new List <string>();
SampleDataRow = new List <string>();
foreach (MappingField mappingField in mappingDefinition.MappingFields)
{
int idx = mappingField.SampleID;
// set sample value and restore Sample and Header arrays
SampleHeaderRow.Add(mappingField.SampleField);
SampleDataRow.Add(mappingField.SampleValue);
middlewareFieldIdentifier = mappingField.MiddlewareFieldIdentifier;
listBoxPair = new ListBoxPair();
if (Int32.TryParse(mappingField.MiddlewareFieldIdentifier, out mappingFieldID))
{
// convention is that unset mapping has negative integer ID
// if mapping set then delete from DefintionFields and Add to MappingFields
listBoxPair = new ListBoxPair(((idx + 1) * -1).ToString(), "-");
}
else
{
// or set actual mapping and remove from definition list
listBoxPair = DefinitionFields.FirstOrDefault(x => x.Key == middlewareFieldIdentifier);
DefinitionFields.Remove(listBoxPair);
}
MappingSettingList.Add(listBoxPair);
}
SetDefinitionListBox();
SetMappingList();
ClearFilterSettings();
DisplaySampleFile();
SetFilterFieldNames();
CompanyNameTextBox.Text = mappingDefinition.CompanyName;
MappingFilterList = mappingDefinition.MappingFilters;
checkBoxLevel2.IsChecked = mappingDefinition.RequiresLevel2File;
checkBoxOptOut.IsChecked = mappingDefinition.RequiresOptOutFile;
checkBoxErrorFile.IsChecked = mappingDefinition.CanHaveErrorFile;
DisplayCurrentFilterList();
SetScrollViewer();
}
public CSVDataProcesses(MappingDefinition mappingDefinition)
{
MappingDefinition = mappingDefinition;
}
