public static void TestForeignKeys(EduHubSchema Schema) { foreach (var childField in Schema.Entities.SelectMany(e => e.Fields).Where(f => f.ForeignParentKey.EntityName != null)) { var foreignParent = childField.GetForeignParent(); if (foreignParent == null) { throw new InvalidOperationException("Foreign parent not found"); } var foreignIndex = foreignParent.GetIndex(); if (foreignIndex == null) { throw new InvalidOperationException("Foreign parent is not indexed"); } if (!foreignIndex.IsUnique) { throw new InvalidOperationException("Foreign index is not unique"); } var childIndex = childField.GetIndex(); if (childIndex == null) { throw new InvalidOperationException("Child field is not indexed"); } } }
public static void ApplyVersion60Overrides(EduHubSchema schema) { // Records in datasets without a unique index cannot be deleted // and this breaks delta/partial dataset support. // In v60 unique indexes on 'TID' (Transaction ID) columns were removed // for a large number of datasets. All currently available data confirms // that these columns continue to be unique and make a good unique key. // This patch restores unique indexes for affected datasets without unique // columns, but with a TID column which was previously unique. // This is a deviation from the official schema, but necessary at this point. // restore unique TID index for datasets with no unique index foreach (var entityName in new string[] { "ARF", "CRF", "DFF", "DRF", "GLCF", "GLCFPREV", "GLF", "GLFPREV", "PEF" }) { if (schema.Contains(entityName)) { RestoreTidUniqueColumn(schema[entityName]); } } // primary key constraints were removed from the official schema, but // there serve an important purpose in our recreation foreach (var entity in schema.Entities) { RestorePrimaryKey(entity); } }
public static void EnsureNavigationFieldIndexes(EduHubSchema schema) { foreach (var childField in schema.Entities.SelectMany(e => e.Fields).Where(f => f.ForeignParentKey.EntityName != null)) { if (childField.GetIndex() == null) { var entity = childField.Entity; var fields = new List <EduHubField>() { childField }; var indexName = $"Index_{string.Join("_", fields.Select(f => f.Name))}"; var index = new EduHubIndex(entity, indexName, fields.AsReadOnly(), false, false, false); entity.AddIndex(index); } } }
static void Main(string[] args) { var c7SchemaFilename = args[0]; var csvDirectory = args[1]; var output = args[2]; // Parse C7 Schema File var c7Elements = C7Parser.ParseSchemaFile(c7SchemaFilename); var schema = EduHubSchema.FromC7Schema(c7Elements); // Augment with Database Information DbParser.AugmentSchemaFromCsv(csvDirectory, schema); // Apply customizations Customizations.EnsureNavigationFieldIndexes(schema); // Validate schema SchemaTests.TestForeignKeys(schema); var schemaXml = schema.ToXElement(); schemaXml.Save(output); }
public static void AugmentSchemaFromSql(string ConnectionString, EduHubSchema Schema) { var tables = new List <SysTable>(); var types = new List <SysType>(); var columns = new List <SysColumn>(); var indexes = new List <SysIndex>(); var indexColumns = new List <SysIndexColumn>(); using (SqlConnection dbConnection = new SqlConnection(ConnectionString)) { dbConnection.Open(); // Tables const string sqlTables = @"SELECT name, object_id FROM sys.tables"; using (var dbCommand = new SqlCommand(sqlTables, dbConnection)) { using (var dbReader = dbCommand.ExecuteReader()) { while (dbReader.Read()) { tables.Add(new SysTable() { name = dbReader.GetString(0), object_id = dbReader.GetInt32(1) }); } } } // Types const string sqlTypes = @"SELECT name, user_type_id FROM sys.types"; using (var dbCommand = new SqlCommand(sqlTypes, dbConnection)) { using (var dbReader = dbCommand.ExecuteReader()) { while (dbReader.Read()) { types.Add(new SysType() { name = dbReader.GetString(0), user_type_id = dbReader.GetInt32(1) }); } } } // Columns const string sqlColumns = @"SELECT object_id, name, column_id, user_type_id, max_length, is_nullable, is_identity FROM sys.columns"; using (var dbCommand = new SqlCommand(sqlColumns, dbConnection)) { using (var dbReader = dbCommand.ExecuteReader()) { while (dbReader.Read()) { columns.Add(new SysColumn() { object_id = dbReader.GetInt32(0), name = dbReader.GetString(1), column_id = dbReader.GetInt32(2), user_type_id = dbReader.GetInt32(3), max_length = dbReader.GetInt16(4), is_nullable = dbReader.GetBoolean(5), is_identity = dbReader.GetBoolean(6) }); } } } // Indexes const string sqlIndexes = @"SELECT object_id, name, index_id, type, is_unique, is_primary_key FROM sys.indexes"; using (var dbCommand = new SqlCommand(sqlIndexes, dbConnection)) { using (var dbReader = dbCommand.ExecuteReader()) { while (dbReader.Read()) { indexes.Add(new SysIndex() { object_id = dbReader.GetInt32(0), name = dbReader.IsDBNull(1) ? null : dbReader.GetString(1), index_id = dbReader.GetInt32(2), type = dbReader.GetByte(3), is_unique = dbReader.GetBoolean(4), is_primary_key = dbReader.GetBoolean(5) }); } } } // Index Columns const string sqlIndexColumns = @"SELECT object_id, index_id, column_id, key_ordinal FROM sys.index_columns"; using (var dbCommand = new SqlCommand(sqlIndexColumns, dbConnection)) { using (var dbReader = dbCommand.ExecuteReader()) { while (dbReader.Read()) { indexColumns.Add(new SysIndexColumn() { object_id = dbReader.GetInt32(0), index_id = dbReader.GetInt32(1), column_id = dbReader.GetInt32(2), key_ordinal = dbReader.GetByte(3) }); } } } dbConnection.Close(); } AugmentSchema(Schema, tables, types, columns, indexes, indexColumns); }
private static void AugmentSchema(EduHubSchema Schema, List <SysTable> Tables, List <SysType> Types, List <SysColumn> Columns, List <SysIndex> Indexes, List <SysIndexColumn> IndexColumns) { var tableLookup = Tables.ToDictionary(t => t.name, t => t.object_id, StringComparer.OrdinalIgnoreCase); var typeLookup = Types.ToDictionary(t => t.user_type_id, t => t.name); var columnsLookup = Columns .GroupBy(c => c.object_id) .ToDictionary(g => g.Key, g => g.OrderBy(c => c.column_id).ToList()); var indexLookup = Indexes .GroupBy(i => i.object_id) .ToDictionary(g => g.Key, g => g.OrderBy(i => i.index_id).ToList()); var indexColumnLookup = IndexColumns .GroupBy(ic => ic.object_id) .ToDictionary(g => g.Key, g => g.GroupBy(ic => ic.index_id).ToDictionary(g2 => g2.Key, g2 => g2.OrderBy(ic => ic.key_ordinal).ToList())); foreach (var entity in Schema.Entities) { // Table ID if (!tableLookup.TryGetValue(entity.Name, out int tableObjectId)) { throw new InvalidOperationException($"Unknown Entity: {entity.Name}"); } else { // COLUMNS // Validate schema conformance; determine nullable and identity columns if (columnsLookup.TryGetValue(tableObjectId, out List <SysColumn> entityColumns)) { foreach (var column in entityColumns) { var field = entity.Fields.First(f => f.Name.Equals(column.name, StringComparison.OrdinalIgnoreCase)); field.SqlType = typeLookup[column.user_type_id]; // Check Framework Type if (field.Type != GetFrameworkType(field.SqlType)) { // Overrides if (field.Entity.Name == "SPFSTORE" && field.Name == "ASSOCIATION_TYPE" && field.Type == "short" && field.SqlType == "int") { field.Type = "int"; } else { throw new InvalidOperationException("Entity field type didn't match database schema"); } } switch (field.SqlType) { case "varchar": case "varbinary": if (field.TypeMaxLength == 0) { field.TypeMaxLength = column.max_length; } // Overrides if (field.Entity.Name == "SPFSTORE" && field.Name == "PHYSICAL_LOCATION" && field.TypeMaxLength == 255 && column.max_length == 500) { field.TypeMaxLength = column.max_length; break; } if (field.TypeMaxLength != column.max_length) { throw new InvalidOperationException("Entity field max length didn't match database schema"); } break; } if (!field.IsNullable && column.is_nullable) { throw new InvalidOperationException("Entity field nullable didn't match database schema"); } if (field.IsNullable && !column.is_nullable) { field.IsNullable = false; } if (field.IsIdentity && !column.is_identity) { throw new InvalidOperationException("Entity field identity didn't match database schema"); } if (!field.IsIdentity && column.is_identity) { field.IsIdentity = true; } } } // INDEXES // Determine Indexes if (indexLookup.TryGetValue(tableObjectId, out List <SysIndex> entityIndexes)) { foreach (var entityIndex in entityIndexes) { List <SysIndexColumn> indexColumns = indexColumnLookup[entityIndex.object_id][entityIndex.index_id]; List <EduHubField> fields = indexColumns .Select(ic => entity.Fields.First(f => f.Name.Equals(entityColumns.First(c => c.column_id == ic.column_id).name, StringComparison.OrdinalIgnoreCase))) .ToList(); if (fields.Count == 0) { throw new InvalidOperationException("Unexpected index with no columns"); } var indexName = $"Index_{string.Join("_", fields.Select(f => f.Name))}"; var index = new EduHubIndex( Entity: entity, Name: indexName, Fields: fields.AsReadOnly(), IsPrimary: entityIndex.is_primary_key, IsUnique: entityIndex.is_unique, IsClustered: entityIndex.type == 1); // 1 = Clustered, 2 = Non Clustered // Check for existing Index with matching Fields; if matched, add unique or shortest name; var matchingIndex = entity.Indexes.FirstOrDefault(ei => ei.Fields.Count == index.Fields.Count && ei.Fields.All(f => index.Fields.Contains(f))); if (matchingIndex != null) { if ((!matchingIndex.IsUnique && index.IsUnique) || // New Index is unique (!matchingIndex.IsPrimary && index.IsPrimary) || // New Index is primary (!matchingIndex.IsClustered && index.IsClustered) || // New Index is clustered (index.Name.Length < matchingIndex.Name.Length)) // New Index has shorter name { // Remove existing, add new entity.RemoveIndex(matchingIndex); entity.AddIndex(index); } } else { entity.AddIndex(index); } } } // Ensure identity columns have index foreach (var identityField in entity.Fields.Where(f => f.IsIdentity)) { var name = $"Index_{identityField.Name}"; var index = new EduHubIndex( Entity: entity, Name: name, Fields: new List <EduHubField>() { identityField }.AsReadOnly(), IsPrimary: false, IsUnique: true, IsClustered: false); // Check for existing Index with matching Fields; if matched, add unique or shortest name; var matchingIndex = entity.Indexes.FirstOrDefault(ei => ei.Fields.Count == index.Fields.Count && ei.Fields.All(f => index.Fields.Contains(f))); if (matchingIndex != null) { if ((!matchingIndex.IsUnique && index.IsUnique) || // New Index is unique (index.Name.Length < matchingIndex.Name.Length)) // New Index has shorter name { if (matchingIndex.IsClustered) { throw new InvalidOperationException("Shouldn't replace clustered indexes"); } if (matchingIndex.IsPrimary) { throw new InvalidOperationException("Shouldn't replace primary indexes"); } // Remove existing, add new entity.RemoveIndex(matchingIndex); entity.AddIndex(index); } } else { entity.AddIndex(index); } } } } }
public static void AugmentSchemaFromCsv(string CsvDirectory, EduHubSchema Schema) { var tables = new List <SysTable>(); using (var stream = new FileStream(Path.Combine(CsvDirectory, "SQL_Schema_Tables.csv"), FileMode.Open)) { using (var reader = new CsvReader(stream)) { tables = SysBuilder.ImportCsv <SysTable>(reader, (ColumnName: "SYSSCHOBJS.NAME", Mapper: (t, v) => t.name = v), (ColumnName: "OBJECT_ID", Mapper: (t, v) => t.object_id = int.Parse(v)) ); } } var types = new List <SysType>(); using (var stream = new FileStream(Path.Combine(CsvDirectory, "SQL_Schema_Types.csv"), FileMode.Open)) { using (var reader = new CsvReader(stream)) { types = SysBuilder.ImportCsv <SysType>(reader, (ColumnName: "SYSSCALARTYPES.NAME", Mapper: (t, v) => t.name = v), (ColumnName: "USER_TYPE_ID", Mapper: (t, v) => t.user_type_id = int.Parse(v)) ); } } var columns = new List <SysColumn>(); using (var stream = new FileStream(Path.Combine(CsvDirectory, "SQL_Schema_Columns.csv"), FileMode.Open)) { using (var reader = new CsvReader(stream)) { columns = SysBuilder.ImportCsv <SysColumn>(reader, (ColumnName: "OBJECT_ID", Mapper: (t, v) => t.object_id = int.Parse(v)), (ColumnName: "SYSCOLPARS.NAME", Mapper: (t, v) => t.name = v), (ColumnName: "COLUMN_ID", Mapper: (t, v) => t.column_id = int.Parse(v)), (ColumnName: "USER_TYPE_ID", Mapper: (t, v) => t.user_type_id = int.Parse(v)), (ColumnName: "MAX_LENGTH", Mapper: (t, v) => t.max_length = int.Parse(v)), (ColumnName: "IS_NULLABLE", Mapper: (t, v) => t.is_nullable = v.AsBool()), (ColumnName: "IS_IDENTITY", Mapper: (t, v) => t.is_identity = v.AsBool()) ); } } var indexes = new List <SysIndex>(); using (var stream = new FileStream(Path.Combine(CsvDirectory, "SQL_Schema_Indexes.csv"), FileMode.Open)) { using (var reader = new CsvReader(stream)) { indexes = SysBuilder.ImportCsv <SysIndex>(reader, (ColumnName: "OBJECT_ID", Mapper: (t, v) => t.object_id = int.Parse(v)), (ColumnName: "SYSIDXSTATS.NAME", Mapper: (t, v) => t.name = v), (ColumnName: "INDEX_ID", Mapper: (t, v) => t.index_id = int.Parse(v)), (ColumnName: "SYSIDXSTATS.TYPE", Mapper: (t, v) => t.type = int.Parse(v)), (ColumnName: "IS_UNIQUE", Mapper: (t, v) => t.is_unique = v.AsBool()), (ColumnName: "IS_PRIMARY_KEY", Mapper: (t, v) => t.is_primary_key = v.AsBool()) ); } } var indexColumns = new List <SysIndexColumn>(); using (var stream = new FileStream(Path.Combine(CsvDirectory, "SQL_Schema_IndexColumns.csv"), FileMode.Open)) { using (var reader = new CsvReader(stream)) { indexColumns = SysBuilder.ImportCsv <SysIndexColumn>(reader, (ColumnName: "OBJECT_ID", Mapper: (t, v) => t.object_id = int.Parse(v)), (ColumnName: "INDEX_ID", Mapper: (t, v) => t.index_id = int.Parse(v)), (ColumnName: "COLUMN_ID", Mapper: (t, v) => t.column_id = int.Parse(v)), (ColumnName: "KEY_ORDINAL", Mapper: (t, v) => t.key_ordinal = byte.Parse(v)) ); } } AugmentSchema(Schema, tables, types, columns, indexes, indexColumns); }