//private BaseDataVersionStrategy GetVersioningStrategy() //{ // var entityType = typeof (TEntity); // var attributes = entityType.GetCustomAttributes(typeof(BaseDataImportAttribute), true) as BaseDataImportAttribute[]; // if (attributes == null || !attributes.Any()) // return null; // or throw exception // return attributes[0].CreateVersionStrategy<TEntity>(); //} /// <summary> /// Loads the data. /// </summary> public virtual void LoadData() { try { ProvideFeedback(string.Format("Loading {0}", Inflector.Pluralize(Inflector.Titleize2(typeof(TEntity).Name)))); string tableName = typeof(TEntity).EntityTableName(); using (var session = this.SessionProvider.SessionFactory.OpenStatelessSession()) { session.CreateSQLQuery(EnableDisableNonClusteredIndexes(tableName, true)) //ALL .ExecuteUpdate(); } using (var session = this.SessionProvider.SessionFactory.OpenStatelessSession()) { EntityStrategy.CurrentSession = session; using (var reader = DataProvider[Reader.Name]) { if (!reader.AlreadyImported()) { using (var bulkImporter = new BulkInsert <TEntity, TKey>(session.Connection)) { bulkImporter.ConnectionRequested += (o, e) => { e.Data = session.Connection as SqlConnection; }; bulkImporter.Prepare(); bulkImporter.BatchSize = BatchSize;; while (reader.Read()) { var temp = EntityStrategy.LoadFromReader(reader); bulkImporter.Insert(temp); } } } } } Task.Factory.StartNew(() => { using (var session = this.SessionProvider.SessionFactory.OpenStatelessSession()) { session.CreateSQLQuery(EnableDisableNonClusteredIndexes(tableName)) //ALL .SetTimeout(600) .ExecuteUpdate(); } }, TaskCreationOptions.LongRunning); } catch (Exception ex) { Logger.Write(ex, $"Error importing data for entity type {typeof(TEntity).Name} using strategy {typeof(TStrategy).Name}"); } }
/// <summary> /// Loads the data. /// </summary> public override void LoadData() { try { if (!VersionStrategy.IsLoaded() && VersionStrategy.IsNewest()) { Logger.Information($"Processing base data update for type {typeof(TEntity).Name}"); var rows = 0; // start transaction using (var session = DataProvider.SessionFactory.OpenStatelessSession()) { // Always turncate for enum types using (var cmd = session.Connection.CreateCommand()) { if (cmd.Connection.State == ConnectionState.Closed) { cmd.Connection.Open(); } cmd.CommandTimeout = 900; cmd.CommandText = string.Format("TRUNCATE TABLE {0}", tableName); cmd.ExecuteNonQuery(); } // Loop through values in the enum. using (var bulkImporter = new BulkInsert <TEntity, TKey>(session.Connection)) { bulkImporter.ConnectionRequested += (o, e) => { e.Data = session.Connection as SqlConnection; }; bulkImporter.Prepare(); bulkImporter.BatchSize = 1000; Array values = Enum.GetValues(typeof(TEnum)); foreach (var val in values) { var temp = GetEntity(val); if (temp != null) { bulkImporter.Insert(temp); rows++; } } } } SchemaVersion version; using (var session = DataProvider.SessionFactory.OpenSession()) { version = VersionStrategy.GetVersion(session); session.SaveOrUpdate(version); session.Flush(); } Logger.Information($"Base data update completed for type {typeof(TEntity).Name}: {rows} rows inserted or updated; now at schema version {version}"); } } catch (Exception ex) { Logger.Write(ex, "Error importing {0} enumeration values for data type {1}", typeof(TEnum).Name, typeof(TEntity).Name); } }
/// <summary> /// Loads the data. /// </summary> public override void LoadData() { try { var tableIndexIsOff = false; // Get list of files matching mask // TODO: Throw an error if the path doesn't exist? if (Directory.Exists(baseDataDir)) { var files = Directory.GetFiles(baseDataDir, Fileprefix + "-*.csv"); PreProcessFile(ref files); foreach (var file in files) { VersionStrategy.Filename = file; if (!VersionStrategy.IsLoaded() && (ImportType == BaseDataImportStrategyType.Append || VersionStrategy.IsNewest())) { // start transaction Logger.Write($"Processing base data update for type {typeof(TEntity).Name} from file {file}"); var rows = 0; // Verify data file exists. if (!File.Exists(Path.Combine(baseDataDir, file))) { Logger.Warning("Import file \"{0}\" missing from the base data resources directory.", file); return; } using (var session = DataProvider.SessionFactory.OpenStatelessSession()) { // Turn off indexes. if (TurnOffIndexesDuringImpport && !tableIndexIsOff) { DisableTableIndexes(); tableIndexIsOff = true; } // Turncate the table if it's a replace strategy if (ImportType == BaseDataImportStrategyType.Replace) { using (var cmd = session.Connection.CreateCommand()) { if (cmd.Connection.State == ConnectionState.Closed) { cmd.Connection.Open(); } cmd.CommandTimeout = 900; cmd.CommandText = string.Format("TRUNCATE TABLE {0}", tableName); cmd.ExecuteNonQuery(); } } var builder = new OleDbConnectionStringBuilder(); builder.Provider = "Microsoft.ACE.OLEDB.12.0"; builder.DataSource = baseDataDir; builder["Extended Properties"] = "text;HDR=YES;FMT=Delimited"; using (var conn = new OleDbConnection(builder.ConnectionString)) { conn.Open(); var sql = string.Format("SELECT * FROM [{0}]", Path.GetFileName(file)); using (var cmd = new OleDbCommand(sql, conn)) { var reader = cmd.ExecuteReader(); using (var bulkImporter = new BulkInsert <TEntity, TKey>(session.Connection)) { bulkImporter.ConnectionRequested += (o, e) => e.Data = session.Connection as SqlConnection; bulkImporter.Prepare(); bulkImporter.BatchSize = 1000; // TODO: Parameterize? if (reader == null) { Logger.Warning( "A problem occurred while trying to read CSV file \"{0}\". Please make sure that the file is properly formated and has data.", file); return; } if (reader.HasRows) { while (reader.Read()) { var temp = LoadFromReader(reader); if (temp != null) { bulkImporter.Insert(temp); } rows++; } } } } } } SchemaVersion version; using (var session = DataProvider.SessionFactory.OpenSession()) { version = VersionStrategy.GetVersion(session); session.SaveOrUpdate(version); session.Flush(); } // commit transaction Logger.Information( $"Base data update completed for type {typeof(TEntity).Name}: {rows} rows inserted or updated; now at schema version {version}"); } } if (TurnOffIndexesDuringImpport && tableIndexIsOff) { Task.Factory.StartNew(() => EnableTableIndexes(tableName), TaskCreationOptions.LongRunning); } } } catch (Exception ex) { Logger.Write(ex, "Error loading base data for entity {0}", typeof(TEntity).Name); } }