/// <summary> /// Loads the data. /// </summary> public override void LoadData() { try { var tableIndexIsOff = false; // Get list of files matching mask // TODO: Throw an error if the path doesn't exist? if (Directory.Exists(baseDataDir)) { var files = Directory.GetFiles(baseDataDir, Fileprefix + "-*.csv"); PreProcessFile(ref files); foreach (var file in files) { VersionStrategy.Filename = file; if (!VersionStrategy.IsLoaded() && (ImportType == BaseDataImportStrategyType.Append || VersionStrategy.IsNewest())) { // start transaction Logger.Write($"Processing base data update for type {typeof(TEntity).Name} from file {file}"); var rows = 0; // Verify data file exists. if (!File.Exists(Path.Combine(baseDataDir, file))) { Logger.Warning("Import file \"{0}\" missing from the base data resources directory.", file); return; } using (var session = DataProvider.SessionFactory.OpenStatelessSession()) { // Turn off indexes. if (TurnOffIndexesDuringImpport && !tableIndexIsOff) { DisableTableIndexes(); tableIndexIsOff = true; } // Turncate the table if it's a replace strategy if (ImportType == BaseDataImportStrategyType.Replace) { using (var cmd = session.Connection.CreateCommand()) { if (cmd.Connection.State == ConnectionState.Closed) { cmd.Connection.Open(); } cmd.CommandTimeout = 900; cmd.CommandText = string.Format("TRUNCATE TABLE {0}", tableName); cmd.ExecuteNonQuery(); } } var builder = new OleDbConnectionStringBuilder(); builder.Provider = "Microsoft.ACE.OLEDB.12.0"; builder.DataSource = baseDataDir; builder["Extended Properties"] = "text;HDR=YES;FMT=Delimited"; using (var conn = new OleDbConnection(builder.ConnectionString)) { conn.Open(); var sql = string.Format("SELECT * FROM [{0}]", Path.GetFileName(file)); using (var cmd = new OleDbCommand(sql, conn)) { var reader = cmd.ExecuteReader(); using (var bulkImporter = new BulkInsert <TEntity, TKey>(session.Connection)) { bulkImporter.ConnectionRequested += (o, e) => e.Data = session.Connection as SqlConnection; bulkImporter.Prepare(); bulkImporter.BatchSize = 1000; // TODO: Parameterize? if (reader == null) { Logger.Warning( "A problem occurred while trying to read CSV file \"{0}\". Please make sure that the file is properly formated and has data.", file); return; } if (reader.HasRows) { while (reader.Read()) { var temp = LoadFromReader(reader); if (temp != null) { bulkImporter.Insert(temp); } rows++; } } } } } } SchemaVersion version; using (var session = DataProvider.SessionFactory.OpenSession()) { version = VersionStrategy.GetVersion(session); session.SaveOrUpdate(version); session.Flush(); } // commit transaction Logger.Information( $"Base data update completed for type {typeof(TEntity).Name}: {rows} rows inserted or updated; now at schema version {version}"); } } if (TurnOffIndexesDuringImpport && tableIndexIsOff) { Task.Factory.StartNew(() => EnableTableIndexes(tableName), TaskCreationOptions.LongRunning); } } } catch (Exception ex) { Logger.Write(ex, "Error loading base data for entity {0}", typeof(TEntity).Name); } }
/// <summary> /// Loads the data. /// </summary> public override void LoadData() { try { if (!VersionStrategy.IsLoaded() && VersionStrategy.IsNewest()) { Logger.Information($"Processing base data update for type {typeof(TEntity).Name}"); var rows = 0; // start transaction using (var session = DataProvider.SessionFactory.OpenStatelessSession()) { // Always turncate for enum types using (var cmd = session.Connection.CreateCommand()) { if (cmd.Connection.State == ConnectionState.Closed) { cmd.Connection.Open(); } cmd.CommandTimeout = 900; cmd.CommandText = string.Format("TRUNCATE TABLE {0}", tableName); cmd.ExecuteNonQuery(); } // Loop through values in the enum. using (var bulkImporter = new BulkInsert <TEntity, TKey>(session.Connection)) { bulkImporter.ConnectionRequested += (o, e) => { e.Data = session.Connection as SqlConnection; }; bulkImporter.Prepare(); bulkImporter.BatchSize = 1000; Array values = Enum.GetValues(typeof(TEnum)); foreach (var val in values) { var temp = GetEntity(val); if (temp != null) { bulkImporter.Insert(temp); rows++; } } } } SchemaVersion version; using (var session = DataProvider.SessionFactory.OpenSession()) { version = VersionStrategy.GetVersion(session); session.SaveOrUpdate(version); session.Flush(); } Logger.Information($"Base data update completed for type {typeof(TEntity).Name}: {rows} rows inserted or updated; now at schema version {version}"); } } catch (Exception ex) { Logger.Write(ex, "Error importing {0} enumeration values for data type {1}", typeof(TEnum).Name, typeof(TEntity).Name); } }
/// <summary> /// Loads the data. /// </summary> public override void LoadData() { bool tableIndexIsOff = false; try { // Get list of files matching mask // TODO: Throw an error if the path doesn't exist? if (Directory.Exists(baseDataDir)) { var files = Directory.GetFiles(baseDataDir, Fileprefix + "*.csv").ToList(); // Turn off indexes. if (files.Any() && TurnOffIndexesDuringImpport && AllowDisableIndexesDuringImport) { DisableTableIndexes(); tableIndexIsOff = true; } foreach (var file in files) { VersionStrategy.Filename = file; if (!VersionStrategy.IsLoaded() && (ImportType == BaseDataImportStrategyType.Append || VersionStrategy.IsNewest())) { Logger.Write($"Processing base data update for type {typeof(TEntity).Name} from file {file}"); var rows = 0; // start transaction // Verify format file exists. if (!File.Exists(Path.Combine(baseDataDir, FormatFile))) { Logger.Warning("Format file \"{0}\" missing from the base data resources directory.", FormatFile); return; } // Verify data file exists. if (!File.Exists(Path.Combine(baseDataDir, file))) { Logger.Warning("Import file \"{0}\" missing from the base data resources directory.", file); return; } using (var session = DataProvider.SessionFactory.OpenStatelessSession()) { // Turn off indexes. // Please do not remove as we may add back. Jason //if (TurnOffIndexesDuringImpport && !tableIndexIsOff) //{ // DisableTableIndexes(); // tableIndexIsOff = true; //} // Turncate the table if it's a replace strategy if (ImportType == BaseDataImportStrategyType.Replace) { using (var cmd = session.Connection.CreateCommand()) { if (cmd.Connection.State == ConnectionState.Closed) { cmd.Connection.Open(); } cmd.CommandTimeout = 900; cmd.CommandText = string.Format("TRUNCATE TABLE {0}", tableName); cmd.ExecuteNonQuery(); } } var con = session.Connection; ProvideFeedback(string.Format("Importing file {0}", file)); using (var cmd2 = con.CreateCommand()) { cmd2.CommandText = "BULK INSERT " + tableName + " FROM '" + Path.Combine(baseDataDir, file) + "' WITH (FIRSTROW = 2, FORMATFILE = '" + Path.Combine(baseDataDir, FormatFile) + "')"; cmd2.CommandTimeout = 6000; rows = cmd2.ExecuteNonQuery(); } } SchemaVersion version; using (var session = DataProvider.SessionFactory.OpenSession()) { // TODO: Add functionality to update existing version row in all scenarios to avoid multiple entries.- Jason version = VersionStrategy.GetVersion(session); session.SaveOrUpdate(version); session.Flush(); } Logger.Information( $"Base data update completed for type {typeof(TEntity).Name}: {rows} rows inserted or updated; now at schema version {version}"); } } } } catch (Exception ex) { Logger.Write(ex, "Error loading base data for type {0}", typeof(TEntity).Name); } finally { if (TurnOffIndexesDuringImpport && AllowEnableIndexesDuringImport && tableIndexIsOff) { Task.Factory.StartNew(() => { EnableTableIndexes(tableName); }, TaskCreationOptions.LongRunning); } } }