private void openConfigurationFileToolStripMenuItem_Click(object sender, EventArgs e)
{
var theDialog = new OpenFileDialog
{
Title = @"Open Json Configuration File",
Filter = @"Text files|*.txt",
InitialDirectory = @"" + GlobalParameters.ConfigurationPath + ""
};
if (theDialog.ShowDialog() != DialogResult.OK)
{
return;
}
try
{
var myStream = theDialog.OpenFile();
using (myStream)
{
richTextBoxInformation.Clear();
var chosenFile = theDialog.FileName;
// Load from disk into memory
JsonExportSetting.LoadJsonConfigurationFile(chosenFile);
// Update values on form
LocalInitialiseJsonExtractSettings();
}
}
catch (Exception ex)
{
MessageBox.Show("Error: Could not read file from disk. Original error: " + ex.Message, "An issues has been encountered", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
public FormJsonConfiguration(FormManageMetadata parent)
{
_myParent = parent;
InitializeComponent();
// Make sure the configuration information is available in this form.
try
{
var configurationFileName = GlobalParameters.ConfigurationPath + GlobalParameters.JsonExportConfigurationFileName + '_' + GlobalParameters.WorkingEnvironment + GlobalParameters.FileExtension;
// If the config file does not exist yet, create it by calling the EnvironmentConfiguration Class
if (!File.Exists(configurationFileName))
{
JsonExportSetting.CreateDummyJsonConfigurationFile(configurationFileName);
}
// Load the validation settings file using the paths retrieved from the application root contents (configuration path)
JsonExportSetting.LoadJsonConfigurationFile(configurationFileName);
richTextBoxInformation.Text += $"The Json extract configuration file {configurationFileName} has been loaded.";
// Apply the values to the form
LocalInitialiseJsonExtractSettings();
}
catch (Exception)
{
// Do nothing
}
}
/// <summary>
/// Save settings
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void toolStripMenuItemSaveSettings_Click(object sender, EventArgs e)
{
try
{
// Connection
var stringSourceConnection = "";
stringSourceConnection = checkBoxSourceConnectionKey.Checked ? "True" : "False";
JsonExportSetting.GenerateDataObjectConnection = stringSourceConnection;
// Database extension
var stringDatabaseExtension = "";
stringDatabaseExtension = checkBoxDatabaseExtension.Checked ? "True" : "False";
JsonExportSetting.GenerateDatabaseAsExtension = stringDatabaseExtension;
// Schema extension
var stringSchemaExtension = "";
stringSchemaExtension = checkBoxSchemaExtension.Checked ? "True" : "False";
JsonExportSetting.GenerateSchemaAsExtension = stringSchemaExtension;
// Type classification
var stringTypeClassification = "";
stringTypeClassification = checkBoxAddType.Checked ? "True" : "False";
JsonExportSetting.GenerateTypeAsClassification = stringTypeClassification;
// Source data types
var stringSourceDataTypes = "";
stringSourceDataTypes = checkBoxSourceDataType.Checked ? "True" : "False";
JsonExportSetting.GenerateSourceDataItemTypes = stringSourceDataTypes;
// Target data types
var stringTargetDataTypes = "";
stringTargetDataTypes = checkBoxTargetDataType.Checked ? "True" : "False";
JsonExportSetting.GenerateTargetDataItemTypes = stringTargetDataTypes;
// Add metadata related data object
var stringAddMetadataConnection = "";
stringAddMetadataConnection = checkBoxAddMetadataConnection.Checked ? "True" : "False";
JsonExportSetting.AddMetadataAsRelatedDataObject = stringAddMetadataConnection;
// Add next up related data objects from lineage
var stringAddNextUpObjects = "";
stringAddNextUpObjects = checkBoxNextUpDataObjects.Checked ? "True" : "False";
JsonExportSetting.AddUpstreamDataObjectsAsRelatedDataObject = stringAddNextUpObjects;
// Write to disk
JsonExportSetting.SaveJsonConfigurationFile();
richTextBoxInformation.Text = @"The values have been successfully saved.";
}
catch (Exception ex)
{
MessageBox.Show(@"Error: Could not write values to memory and disk. Original error: " + ex.Message, @"An issues has been encountered", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
/// <summary>
/// Commit the changes to memory, save the configuration settings to disk and create a backup.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void SaveConfigurationFileToolStripMenuItem_Click(object sender, EventArgs e)
{
#region root path file
// Update the paths in memory
GlobalParameters.OutputPath = textBoxOutputPath.Text;
GlobalParameters.ConfigurationPath = textBoxConfigurationPath.Text;
var localEnvironment = (KeyValuePair <TeamWorkingEnvironment, string>)comboBoxEnvironments.SelectedItem;
GlobalParameters.WorkingEnvironment = localEnvironment.Key.environmentKey;
GlobalParameters.WorkingEnvironmentInternalId = localEnvironment.Key.environmentInternalId;
// Save the paths from memory to disk.
UpdateRootPathFile();
#endregion
// Make sure the new paths as updated are available upon save for backup etc.
// Check if the paths and files are available, just to be sure.
FileHandling.InitialisePath(GlobalParameters.ConfigurationPath);
FileHandling.InitialisePath(GlobalParameters.OutputPath);
TeamConfiguration.CreateDummyEnvironmentConfigurationFile(GlobalParameters.ConfigurationPath + GlobalParameters.ConfigFileName + '_' + GlobalParameters.WorkingEnvironment + GlobalParameters.FileExtension);
ValidationSetting.CreateDummyValidationFile(GlobalParameters.ConfigurationPath + GlobalParameters.ValidationFileName + '_' + GlobalParameters.WorkingEnvironment + GlobalParameters.FileExtension);
JsonExportSetting.CreateDummyJsonConfigurationFile(GlobalParameters.ConfigurationPath + GlobalParameters.JsonExportConfigurationFileName + '_' + GlobalParameters.WorkingEnvironment + GlobalParameters.FileExtension);
// Create a file backup for the configuration file
try
{
TeamUtility.CreateFileBackup(GlobalParameters.ConfigurationPath + GlobalParameters.ConfigFileName + '_' + GlobalParameters.WorkingEnvironment + GlobalParameters.FileExtension);
richTextBoxInformation.Text = "A backup of the current configuration was made at " + DateTime.Now + " in " + textBoxConfigurationPath.Text + ".\r\n";
}
catch (Exception)
{
richTextBoxInformation.Text = "TEAM was unable to create a backup of the configuration file.";
}
// Update the in-memory variables for use throughout the application, to commit the saved changes for runtime use.
// This is needed before saving to disk, as the EnvironmentConfiguration Class retrieves the values from memory.
UpdateConfigurationInMemory();
// Save the information
LocalTeamEnvironmentConfiguration.SaveConfigurationFile();
parentFormMain.RevalidateFlag = true;
}
/// <summary>
/// Updates an input DataObject with a Database and Schema extension (two key/value pairs) based on its connection properties (TeamConnection object).
/// </summary>
/// <param name="dataObject"></param>
/// <param name="teamConnection"></param>
/// <param name="jsonExportSetting"></param>
/// <returns></returns>
public static DataObject SetDataObjectDatabaseExtension(DataObject dataObject, TeamConnection teamConnection, JsonExportSetting jsonExportSetting)
{
if (jsonExportSetting.GenerateDatabaseAsExtension == "True" && dataObject.dataObjectConnection != null)
{
List <Extension> extensions = new List <Extension>();
var extension = new Extension
{
key = "database",
value = teamConnection.DatabaseServer.DatabaseName,
description = "database name"
};
extensions.Add(extension);
if (dataObject.dataObjectConnection.extensions is null)
{
dataObject.dataObjectConnection.extensions = extensions;
}
else
{
dataObject.dataObjectConnection.extensions.AddRange(extensions);
}
}
return(dataObject);
}
/// <summary>
/// Add a Data Object Connection based on the TeamConnection details.
/// </summary>
/// <param name="dataObject"></param>
/// <param name="teamConnection"></param>
/// <param name="jsonExportSetting"></param>
/// <returns></returns>
public static DataObject SetDataObjectConnection(DataObject dataObject, TeamConnection teamConnection, JsonExportSetting jsonExportSetting)
{
if (jsonExportSetting.GenerateDataObjectConnection == "True")
{
// Add dataObjectConnection, including connection string (to Data Object).
var localDataConnection = new DataConnection();
localDataConnection.dataConnectionString = teamConnection.ConnectionKey;
dataObject.dataObjectConnection = localDataConnection;
}
return(dataObject);
}
/// <summary>
/// Create Data Object in TEAM.
/// </summary>
/// <param name="dataObjectName"></param>
/// <param name="teamConnection"></param>
/// <param name="jsonExportSetting"></param>
/// <returns></returns>
public static DataObject CreateDataObject(string dataObjectName, TeamConnection teamConnection, JsonExportSetting jsonExportSetting)
{
DataObject localDataObject = new DataObject();
localDataObject.name = dataObjectName;
// Data Object Connection
localDataObject = SetDataObjectConnection(localDataObject, teamConnection, jsonExportSetting);
// Source and target connection information as extensions
localDataObject = SetDataObjectDatabaseExtension(localDataObject, teamConnection, jsonExportSetting);
localDataObject = SetDataObjectSchemaExtension(localDataObject, teamConnection, jsonExportSetting);
// Add classifications
localDataObject = SetDataObjectTypeClassification(localDataObject, jsonExportSetting);
return(localDataObject);
}
/// <summary>
/// Create the special-type metadata Data Object.
/// </summary>
/// <param name="metaDataConnection"></param>
/// <param name="jsonExportSetting"></param>
/// <returns></returns>
public static DataObject CreateMetadataDataObject(TeamConnection metaDataConnection, JsonExportSetting jsonExportSetting)
{
DataObject localDataObject = new DataObject();
if (jsonExportSetting.AddMetadataAsRelatedDataObject == "True")
{
localDataObject = CreateDataObject("Metadata", metaDataConnection, jsonExportSetting);
// Override classification
if (jsonExportSetting.GenerateTypeAsClassification == "True")
{
localDataObject.dataObjectClassification[0].classification = "Metadata";
}
}
return(localDataObject);
}
public static DataObject SetDataObjectTypeClassification(DataObject dataObject, JsonExportSetting jsonExportSetting)
{
if (jsonExportSetting.GenerateTypeAsClassification == "True")
{
List <Classification> localClassifications = new List <Classification>();
Classification localClassification = new Classification();
var tableType = MetadataHandling.GetDataObjectType(dataObject.name, "", FormBase.TeamConfiguration);
localClassification.classification = tableType.ToString();
localClassifications.Add(localClassification);
if (dataObject.dataObjectClassification is null)
{
dataObject.dataObjectClassification = localClassifications;
}
else
{
dataObject.dataObjectClassification.AddRange(localClassifications);
}
}
return(dataObject);
}
public static List <DataObject> SetLineageRelatedDataObjectList(DataTable dataObjectMappingDataTable, string targetDataObjectName, JsonExportSetting jsonExportSetting)
{
List <DataObject> dataObjectList = new List <DataObject>();
if (jsonExportSetting.AddUpstreamDataObjectsAsRelatedDataObject == "True")
{
// Find the corresponding row in the Data Object Mapping grid
DataRow[] DataObjectMappings = dataObjectMappingDataTable.Select("[" + TableMappingMetadataColumns.SourceTable + "] = '" + targetDataObjectName + "'");
foreach (DataRow DataObjectMapping in DataObjectMappings)
{
var localDataObjectName = DataObjectMapping[TableMappingMetadataColumns.TargetTable.ToString()].ToString();
var localDataObjectConnectionInternalId = DataObjectMapping[TableMappingMetadataColumns.TargetConnection.ToString()].ToString();
TeamConnection localConnection = FormBase.GetTeamConnectionByConnectionId(localDataObjectConnectionInternalId);
// Set the name and further settings.
dataObjectList.Add(CreateDataObject(localDataObjectName, localConnection, jsonExportSetting));
}
}
return(dataObjectList);
}
public FormMain()
{
InitializeComponent();
// Set the version of the build for everything
const string versionNumberForTeamApplication = "v1.6.3";
Text = "TEAM - Taxonomy for ETL Automation Metadata " + versionNumberForTeamApplication;
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"The TEAM root path is {GlobalParameters.RootPath}."));
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"The TEAM script path is {GlobalParameters.ScriptPath}."));
richTextBoxInformation.AppendText("Initialising the application.\r\n\r\n");
// Root paths (mandatory TEAM directories)
// Make sure the application and custom location directories exist as per the start-up default.
try
{
LocalTeamEnvironmentConfiguration.InitialiseEnvironmentPaths();
}
catch (Exception ex)
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, $"An error was encountered while creating the standard TEAM paths: \r\n\r\n{ex}"));
}
#region Load the root path configuration settings (user defined paths and working environment)
// Load the root file, to be able to locate the (customisable) configuration file.
// This file contains the configuration directory, the output directory and the working environment.
string rootPathFileName = GlobalParameters.CorePath + GlobalParameters.PathFileName + GlobalParameters.FileExtension;
try
{
LocalTeamEnvironmentConfiguration.LoadRootPathFile(rootPathFileName, GlobalParameters.ConfigurationPath, GlobalParameters.OutputPath);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information,
$"The core configuration file {rootPathFileName} has been loaded."));
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error,
$"The core configuration file {rootPathFileName} could not be loaded. Is there a Configuration directory in the TEAM installation location?"));
}
// Environments file
string environmentFile = GlobalParameters.CorePath + GlobalParameters.JsonEnvironmentFileName + GlobalParameters.JsonExtension;
try
{
TeamEnvironmentCollection.LoadTeamEnvironmentCollection(environmentFile);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information,
$"The environment file {environmentFile} has been loaded."));
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error,
$"The environment file {environmentFile} could not be loaded. Does the file exists in the designated (root) location?"));
}
#endregion
#region Check if user configured paths exists (now that they have been loaded from the root file), and create dummy Configuration and Validation files if necessary
// Configuration Path
try
{
FileHandling.InitialisePath(GlobalParameters.ConfigurationPath);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"The user defined configuration path {GlobalParameters.ConfigurationPath} is available."));
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, "The directories required to operate TEAM are not available and can not be created. Do you have administrative privileges in the installation directory to create these additional directories?"));
}
// Output Path
try
{
FileHandling.InitialisePath(GlobalParameters.OutputPath);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"The user defined output path {GlobalParameters.OutputPath} is available."));
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, "The directories required to operate TEAM are not available and can not be created. Do you have administrative privileges in the installation directory to create these additional directories?"));
}
// Create a dummy configuration file if it does not exist.
var configurationFileName =
GlobalParameters.ConfigurationPath +
GlobalParameters.ConfigFileName + '_' +
GlobalParameters.WorkingEnvironment +
GlobalParameters.FileExtension;
try
{
if (!File.Exists(configurationFileName))
{
TeamConfiguration.CreateDummyEnvironmentConfigurationFile(configurationFileName);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"A new configuration file {configurationFileName} was created."));
}
else
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"The existing configuration file {configurationFileName} was detected."));
}
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, $"An issue was encountered creating or detecting the configuration paths for {configurationFileName}."));
}
// Create a default validation file if the file does not exist as expected.
var validationFileName =
GlobalParameters.ConfigurationPath +
GlobalParameters.ValidationFileName + '_' +
GlobalParameters.WorkingEnvironment +
GlobalParameters.FileExtension;
try
{
ValidationSetting.CreateDummyValidationFile(validationFileName);
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, $"An issue was encountered creating or detecting the configuration paths for {validationFileName}."));
}
// Create a default json configuration file if the file does not exist as expected.
var jsonConfigurationFileName =
GlobalParameters.ConfigurationPath +
GlobalParameters.JsonExportConfigurationFileName + '_' +
GlobalParameters.WorkingEnvironment +
GlobalParameters.FileExtension;
try
{
JsonExportSetting.CreateDummyJsonConfigurationFile(jsonConfigurationFileName);
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, $"An issue was encountered creating or detecting the configuration paths for {jsonConfigurationFileName}."));
}
#endregion
// Load the connections file for the respective environment.
var connectionFileName =
GlobalParameters.ConfigurationPath +
GlobalParameters.JsonConnectionFileName + '_' +
GlobalParameters.WorkingEnvironment +
GlobalParameters.JsonExtension;
TeamConfiguration.ConnectionDictionary = TeamConnectionFile.LoadConnectionFile(connectionFileName);
#region Load configuration file
// Load the available configuration file into memory.
var configurationFile = GlobalParameters.ConfigurationPath + GlobalParameters.ConfigFileName + '_' + GlobalParameters.WorkingEnvironment + GlobalParameters.FileExtension;
try
{
// Load the configuration file.
TeamConfiguration.LoadTeamConfigurationFile(configurationFile);
GlobalParameters.EnvironmentMode = TeamConfiguration.EnvironmentMode;
// Retrieve the events into the main event log.
GlobalParameters.TeamEventLog.MergeEventLog(TeamConfiguration.ConfigurationSettingsEventLog);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, $"The user configuration settings ({configurationFile}) have been loaded."));
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, $"An issue was encountered loading the user configuration file ({configurationFile})."));
}
#endregion
//FormBase.EnvironmentVersion.GetEnvironmentVersionsFromFile(Path.GetDirectoryName(configurationFile));
TeamVersionList.LoadVersionList(GlobalParameters.CorePath + GlobalParameters.VersionFileName + GlobalParameters.JsonExtension);
// Load the pattern definition file.
try
{
GlobalParameters.PatternDefinitionList = LoadPatternDefinition.DeserializeLoadPatternDefinition(GlobalParameters.LoadPatternPath + GlobalParameters.LoadPatternDefinitionFile);
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Information, "The pattern definition file was loaded successfully."));
}
catch
{
GlobalParameters.TeamEventLog.Add(Event.CreateNewEvent(EventTypes.Error, "An issue was encountered loading the pattern definition file."));
}
// Report the events (including errors) back to the user
int errorCounter = 0;
foreach (Event individualEvent in GlobalParameters.TeamEventLog)
{
if (individualEvent.eventCode == (int)EventTypes.Error)
{
errorCounter++;
}
}
if (errorCounter > 0)
{
richTextBoxInformation.AppendText(
$"{errorCounter} error(s) have been found at startup. Please check the Event Log in the menu.\r\n\r\n");
}
TestConnections();
//Startup information
richTextBoxInformation.AppendText("\r\nApplication initialised - the Taxonomy of ETL Automation Metadata (TEAM). \r\n");
richTextBoxInformation.AppendText("Welcome to version " + versionNumberForTeamApplication + ".\r\n\r\n");
labelWorkingEnvironment.Text = GlobalParameters.WorkingEnvironment; //+"("+GlobalParameters.WorkingEnvironmentInternalId+")";
labelEnvironmentMode.Text = GlobalParameters.EnvironmentMode.ToString();
}
