public void Execute(string fullFilename, ConnectionSettings settings, ITaskObserver taskObserver)
{
string scriptFilename = getFilename(fullFilename);
if (_executionTracker.TestDataScriptAlreadyExecuted(settings, scriptFilename))
{
taskObserver.Log(string.Format("Skipping (already executed): {0}", scriptFilename));
}
else
{
string sql = _fileSystem.ReadTextFile(fullFilename);
if (!_executor.ScriptSupportsTransactions(sql))
{
taskObserver.Log(string.Format("Executing: {0}", scriptFilename));
_executor.ExecuteNonQuery(settings, sql, true);
_executionTracker.MarkTestDataScriptAsExecuted(settings, scriptFilename, taskObserver);
}
else
{
taskObserver.Log(string.Format("Executing: {0} in a transaction", scriptFilename));
_executor.ExecuteNonQueryTransactional(settings, sql);
_executionTracker.MarkTestDataScriptAsExecuted(settings, scriptFilename, taskObserver);
}
}
}
public void RunsScriptsWithTransactionalStopWordsNonTransactional()
{
ConnectionSettings settings = getConnectionSettings();
string scriptFile = @"c:\scripts\Update\01_Test.sql";
string fileContents = "CREATE DATABASE ...";
MockRepository mocks = new MockRepository();
IScriptExecutionTracker executionTracker = mocks.StrictMock <IScriptExecutionTracker>();
IFileSystem fileSystem = mocks.StrictMock <IFileSystem>();
IQueryExecutor queryExecutor = mocks.StrictMock <IQueryExecutor>();
ITaskObserver taskObserver = mocks.StrictMock <ITaskObserver>();
Expect.Call(executionTracker.ScriptAlreadyExecuted(settings, "01_Test.sql")).Return(false);
taskObserver.Log("Executing: 01_Test.sql");
Expect.Call(fileSystem.ReadTextFile(scriptFile)).Return(fileContents);
Expect.Call(queryExecutor.ScriptSupportsTransactions(fileContents)).Return(false);
queryExecutor.ExecuteNonQuery(settings, fileContents, true);
executionTracker.MarkScriptAsExecuted(settings, "01_Test.sql", taskObserver);
mocks.ReplayAll();
IChangeScriptExecutor executor = new ChangeScriptExecutor(executionTracker, queryExecutor, fileSystem);
executor.Execute(scriptFile, settings, taskObserver);
mocks.VerifyAll();
}
public void ExecuteIfChanged(string fullFilename, ConnectionSettings settings, ITaskObserver taskObserver, bool logOnly = false)
{
string scriptFilename = getFilename(fullFilename);
var scriptFileMD5 = GetFileMD5Hash(fullFilename);
if (_executionTracker.EverytimeScriptShouldBeExecuted(settings, scriptFilename, scriptFileMD5))
{
if (!logOnly)
{
string sql = _fileSystem.ReadTextFile(fullFilename);
taskObserver.Log(string.Format("Executing: {0}{1}", getLastFolderName(fullFilename), scriptFilename));
_executor.ExecuteNonQuery(settings, sql, true);
}
else
{
taskObserver.Log(string.Format("Executing: {0}{1} in log only mode", getLastFolderName(fullFilename), scriptFilename));
}
_executionTracker.MarkScriptAsExecuted(settings, scriptFilename, taskObserver, scriptFileMD5);
}
else
{
taskObserver.Log(string.Format("Skipping (unchanged): {0}{1}", getLastFolderName(fullFilename), scriptFilename));
}
}
public void Execute(string fullFilename, ConnectionSettings settings, ITaskObserver taskObserver, bool logOnly = false)
{
string scriptFilename = getFilename(fullFilename);
if (_executionTracker.ScriptAlreadyExecuted(settings, scriptFilename))
{
taskObserver.Log(string.Format("Skipping (already executed): {0}{1}", getLastFolderName(fullFilename),scriptFilename));
}
else
{
if (!logOnly)
{
string sql = _fileSystem.ReadTextFile(fullFilename);
if (!ScriptSupportsTransactions(sql))
{
taskObserver.Log(string.Format("Executing: {0}{1}", getLastFolderName(fullFilename),scriptFilename));
_executor.ExecuteNonQuery(settings, sql, true);
}
else
{
taskObserver.Log(string.Format("Executing: {0}{1} in a transaction", getLastFolderName(fullFilename),scriptFilename));
_executor.ExecuteNonQueryTransactional(settings, sql);
}
}
else
{
taskObserver.Log(string.Format("Executing: {0}{1} in log only mode",getLastFolderName(fullFilename), scriptFilename));
}
_executionTracker.MarkScriptAsExecuted(settings, scriptFilename, taskObserver);
}
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
try
{
var version = _queryExecutor.ReadFirstColumnAsStringArray(taskAttributes.ConnectionSettings, "select @@version")[0];
taskObserver.Log("Running against: " + version);
//can't kill connections or enter single user mode in Azure
var sql = string.Format("drop database [{0}]", taskAttributes.ConnectionSettings.Database);
if (!version.Contains("SQL Azure"))
{
_connectionDropper.Drop(taskAttributes.ConnectionSettings, taskObserver);
sql = string.Format("ALTER DATABASE [{0}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE drop database [{0}]", taskAttributes.ConnectionSettings.Database);
}
string message = string.Format("Dropping database: {0}\n", taskAttributes.ConnectionSettings.Database);
taskObserver.Log(message);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql);
}
catch (Exception)
{
//if the database doesn't exist just close any open connections and move on
SqlConnection.ClearAllPools();
taskObserver.Log(string.Format("Database '{0}' could not be dropped.", taskAttributes.ConnectionSettings.Database));
}
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
if (!_queryExecutor.CheckDatabaseExists(taskAttributes.ConnectionSettings))
{
return;
}
var version = _queryExecutor.ReadFirstColumnAsStringArray(taskAttributes.ConnectionSettings, "select @@version")[0];
taskObserver.Log("Running against: " + version);
//can't kill connections or enter single user mode in Azure
var sql = string.Format("drop database [{0}]", taskAttributes.ConnectionSettings.Database);
if (!version.Contains("SQL Azure"))
{
_connectionDropper.Drop(taskAttributes.ConnectionSettings, taskObserver);
sql = string.Format("ALTER DATABASE [{0}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE drop database [{0}]", taskAttributes.ConnectionSettings.Database);
}
string message = string.Format("Dropping database: {0}\n", taskAttributes.ConnectionSettings.Database);
taskObserver.Log(message);
try
{
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql);
}
catch (Exception)
{
taskObserver.Log(string.Format("Database '{0}' could not be dropped.", taskAttributes.ConnectionSettings.Database));
}
}
private IEnumerable <IInfosOpérationDépôt> TéléchargerCabsCore(IEnumerable <IParution> parutionsCiblées,
IDépôtCabPao dépôtCabPao,
IDépôtCab dépôtCabExterne,
bool conversionVersTiff,
ITaskObserver observer)
{
observer.NotifyProgress(this, 0);
var i = 0;
int total = parutionsCiblées.Count();
var téléchargementTasks =
(from p in parutionsCiblées
select Task.Factory.StartNew(
() => TéléchargerCabSiNécessaireCore(p, dépôtCabExterne, dépôtCabPao, conversionVersTiff, observer),
TaskCreationOptions.AttachedToParent)
.ContinueWith(
t =>
{
int step = Interlocked.Increment(ref i);
observer.NotifyProgress(this, step * 100 / total);
return(t.Result);
},
TaskContinuationOptions.AttachedToParent
)).ToList();
var résultat = (from task in téléchargementTasks where task.Result != null select task.Result).ToList();
observer.NotifyProgress(this, 100);
return(résultat);
}
public void Versions_database()
{
string assembly = SqlDatabaseManager.SQL_FILE_ASSEMBLY;
string sqlFile = string.Format(SqlDatabaseManager.SQL_FILE_TEMPLATE, "VersionDatabase");
ConnectionSettings settings = new ConnectionSettings(String.Empty, String.Empty, false, String.Empty, String.Empty);
string sqlScript = "SQL script...";
MockRepository mocks = new MockRepository();
IResourceFileLocator fileLocator = mocks.CreateMock <IResourceFileLocator>();
IQueryExecutor queryExecutor = mocks.CreateMock <IQueryExecutor>();
ITaskObserver taskObserver = mocks.CreateMock <ITaskObserver>();
using (mocks.Record())
{
Expect.Call(fileLocator.ReadTextFile(assembly, sqlFile)).Return(sqlScript);
Expect.Call(queryExecutor.ExecuteScalarInteger(settings, sqlScript)).Return(7);
taskObserver.SetVariable("usdDatabaseVersion", "7");
}
using (mocks.Playback())
{
IDatabaseVersioner versioner = new DatabaseVersioner(fileLocator, queryExecutor);
versioner.VersionDatabase(settings, taskObserver);
}
mocks.VerifyAll();
}
public void Correctly_drops_connections()
{
string assembly = SqlDatabaseManager.SQL_FILE_ASSEMBLY;
string sqlFile = string.Format(SqlDatabaseManager.SQL_FILE_TEMPLATE, "DropConnections");
ConnectionSettings settings = new ConnectionSettings("server", "MyDatabase", true, null, null);
MockRepository mocks = new MockRepository();
ITaskObserver taskObserver = mocks.CreateMock <ITaskObserver>();
IResourceFileLocator fileLocator = mocks.CreateMock <IResourceFileLocator>();
ITokenReplacer replacer = mocks.CreateMock <ITokenReplacer>();
IQueryExecutor queryExecutor = mocks.CreateMock <IQueryExecutor>();
using (mocks.Record())
{
taskObserver.Log("Dropping connections for database MyDatabase\n");
Expect.Call(fileLocator.ReadTextFile(assembly, sqlFile)).Return("Unformatted SQL");
replacer.Text = "Unformatted SQL";
replacer.Replace("DatabaseName", "MyDatabase");
Expect.Call(replacer.Text).Return("Formatted SQL");
queryExecutor.ExecuteNonQuery(settings, "Formatted SQL", false);
}
using (mocks.Playback())
{
IDatabaseConnectionDropper dropper = new DatabaseConnectionDropper(fileLocator, replacer, queryExecutor);
dropper.Drop(settings, taskObserver);
}
mocks.VerifyAll();
}
public void Execute(string fullFilename, ConnectionSettings settings, ITaskObserver taskObserver, bool logOnly = false)
{
string scriptFilename = getFilename(fullFilename);
if (_executionTracker.ScriptAlreadyExecuted(settings, scriptFilename))
{
taskObserver.Log(string.Format("Skipping (already executed): {0}{1}", getLastFolderName(fullFilename), scriptFilename));
}
else
{
if (!logOnly)
{
string sql = _fileSystem.ReadTextFile(fullFilename);
if (!_executor.ScriptSupportsTransactions(sql))
{
taskObserver.Log(string.Format("Executing: {0}{1}", getLastFolderName(fullFilename), scriptFilename));
_executor.ExecuteNonQuery(settings, sql, true);
}
else
{
taskObserver.Log(string.Format("Executing: {0}{1} in a transaction", getLastFolderName(fullFilename), scriptFilename));
_executor.ExecuteNonQueryTransactional(settings, sql);
}
}
else
{
taskObserver.Log(string.Format("Executing: {0}{1} in log only mode", getLastFolderName(fullFilename), scriptFilename));
}
_executionTracker.MarkScriptAsExecuted(settings, scriptFilename, taskObserver);
}
}
private void ChargerListeDonnéesCab(ITaskObserver observer)
{
_listeDonnéesCab =
(from compte in Configuration.Comptes.AsParallel()
select CabsPourCompte(compte, observer.InPart())).SelectMany(
cabEnum => cabEnum).ToList();
}
public void CorrectlyExecutesScriptIfItHasntAlreadyBeenExecuted()
{
var settings = new ConnectionSettings("server", "db", true, null, null);
string scriptFile = @"c:\scripts\TestData\01_Test.sql";
string fileContents = "file contents...";
MockRepository mocks = new MockRepository();
IScriptExecutionTracker executionTracker = mocks.StrictMock <IScriptExecutionTracker>();
IFileSystem fileSystem = mocks.StrictMock <IFileSystem>();
IQueryExecutor queryExecutor = mocks.StrictMock <IQueryExecutor>();
ITaskObserver taskObserver = mocks.StrictMock <ITaskObserver>();
Expect.Call(executionTracker.TestDataScriptAlreadyExecuted(settings, "01_Test.sql")).Return(false);
taskObserver.Log("Executing: 01_Test.sql in a transaction");
Expect.Call(fileSystem.ReadTextFile(scriptFile)).Return(fileContents);
Expect.Call(queryExecutor.ScriptSupportsTransactions(fileContents)).Return(true);
queryExecutor.ExecuteNonQueryTransactional(settings, fileContents);
executionTracker.MarkTestDataScriptAsExecuted(settings, "01_Test.sql", taskObserver);
mocks.ReplayAll();
ITestDataScriptExecutor executor = new TestDataScriptExecutor(executionTracker, queryExecutor, fileSystem);
executor.Execute(scriptFile, settings, taskObserver);
mocks.VerifyAll();
}
public void Execute(string fullFilename, ConnectionSettings settings, ITaskObserver taskObserver)
{
string scriptFilename = getFilename(fullFilename);
if (_executionTracker.TestDataScriptAlreadyExecuted(settings, scriptFilename))
{
taskObserver.Log(string.Format("Skipping (already executed): {0}", scriptFilename));
}
else
{
string sql = _fileSystem.ReadTextFile(fullFilename);
if (!_executor.ScriptSupportsTransactions(sql))
{
taskObserver.Log(string.Format("Executing: {0}", scriptFilename));
_executor.ExecuteNonQuery(settings, sql, true);
_executionTracker.MarkTestDataScriptAsExecuted(settings, scriptFilename, taskObserver);
}
else
{
taskObserver.Log(string.Format("Executing: {0} in a transaction", scriptFilename));
_executor.ExecuteNonQueryTransactional(settings, sql);
_executionTracker.MarkTestDataScriptAsExecuted(settings, scriptFilename, taskObserver);
}
}
}
private string ListingCabFtp(ITaskObserver observer)
{
var request = RequêteFtp(_UrlRacine);
request.Method = WebRequestMethods.Ftp.ListDirectoryDetails;
return(OuvrirPage(request, observer));
}
private IEnumerable <ICab> CabsPourPage(string page, TypeEditeur éditeur, ITaskObserver observer)
{
observer.NotifyProgress(this, 0);
var cabs = new List <ICab>();
var matches = _RegexCab.Matches(page);
using (var steps = observer.GetSteps(matches.Count).GetEnumerator())
{
foreach (Match mCab in matches)
{
observer.NotifyProgress(this, steps.Current);
var donnéesBrutes = CréerDonnéesBrutes(mCab, éditeur);
if (donnéesBrutes != null)
{
var cab = CréerCab(donnéesBrutes);
if (cab != null)
{
cabs.Add(cab);
}
}
steps.MoveNext();
}
}
observer.NotifyProgress(this, 100);
return(cabs);
}
private IEnumerable <ICab> CabsPourCompte(CompteCab compte, ITaskObserver observer)
{
observer.NotifyProgress(this, 0);
var cabs = new List <ICab>();
string pageRacine = OuvrirPage(RequêteDossierRacine(compte), observer.InSlice(0, 7));
cabs.AddRange(CabsPourPage(pageRacine, compte.Editeur, observer.InSlice(7, 10)));
ITaskObserver loopObs = observer.InSlice(10, 100);
var matches = _RegexDossier.Matches(pageRacine);
var slices = loopObs.GetSlices(matches.Count).GetEnumerator();
foreach (Match mDossier in matches)
{
ITaskObserver sliceObs = loopObs.InSlice(slices.Current);
var gDossier = mDossier.Groups["url_rep"];
if (gDossier != null && !String.IsNullOrEmpty(gDossier.Value))
{
string dossier = gDossier.Value;
string pageDossier = OuvrirPage(RequêteDossier(compte, dossier), sliceObs.InSlice(0, 70));
if (!String.IsNullOrEmpty(pageDossier))
{
cabs.AddRange(CabsPourPage(pageDossier, compte.Editeur, sliceObs.InSlice(70, 100)));
}
}
slices.MoveNext();
}
observer.NotifyProgress(this, 100);
return(cabs);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
string sql = string.Format("create database [{0}]", taskAttributes.ConnectionSettings.Database);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql, false);
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "ExistingSchema", taskObserver);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
string sql = string.Format("create database [{0}]", taskAttributes.ConnectionSettings.Database);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql, false);
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "ExistingSchema", taskObserver);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
string sql = string.Format("create database [{0}]", taskAttributes.ConnectionSettings.Database);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql);
taskObserver.Log(string.Format("Run scripts in Create folder."));
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Create", taskObserver);
}
public void MarkTestDataScriptAsExecuted(ConnectionSettings settings, string scriptFilename, ITaskObserver task)
{
string insertTemplate =
"insert into usd_AppliedDatabaseTestDataScript (ScriptFile, DateApplied) values ('{0}', getdate())";
string sql = string.Format(insertTemplate, scriptFilename);
_executor.ExecuteNonQueryTransactional(settings, sql);
}
private ICab IntégrerCabTéléchargé(ICab cab, string cheminFichierCab, ITaskObserver observer)
{
observer.NotifyInfo(this, $"Vérification et intégration du nouveau Cab téléchargé [{cheminFichierCab}]");
var newCab = IntégrerCabTéléchargé(cab, cheminFichierCab);
observer.NotifyInfo(this, $"Nouveau Cab téléchargé [{cheminFichierCab}] vérifié et intégré : [{newCab}]");
return(newCab);
}
public void VersionDatabase(ConnectionSettings settings, ITaskObserver taskObserver)
{
string assembly = SqlDatabaseManager.SQL_FILE_ASSEMBLY;
string sqlFile = string.Format(SqlDatabaseManager.SQL_FILE_TEMPLATE, "VersionDatabase");
string sql = _fileLocator.ReadTextFile(assembly, sqlFile);
string version = _executor.ExecuteScalarInteger(settings, sql).ToString();
taskObserver.SetVariable(_databaseVersionPropertyName, version);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
string sql = string.Format("create database [{0}]", taskAttributes.ConnectionSettings.Database);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql);
taskObserver.Log(string.Format("Run scripts in Create folder."));
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Create", taskObserver);
}
public void VersionDatabase(ConnectionSettings settings, ITaskObserver taskObserver)
{
string assembly = SqlDatabaseManager.SQL_FILE_ASSEMBLY;
string sqlFile = string.Format(SqlDatabaseManager.SQL_FILE_TEMPLATE, "VersionDatabase");
string sql = _fileLocator.ReadTextFile(assembly, sqlFile);
string version = _executor.ExecuteScalarInteger(settings, sql).ToString();
taskObserver.SetVariable(_databaseVersionPropertyName, version);
}
/// <inheritdoc />
public void Initialise(IComputationHandler handler)
{
ValidateAssignedComponents();
_logger.Info($"Initialising trainer \"{Name}\" with handler {handler}...");
ITaskObserver prepareTask = SigmaEnvironment.TaskManager.BeginTask(TaskType.Prepare, $"Preparing trainer {Name}");
int initialisedNDArrayCount;
int initialisedNumberCount;
Network.AssociatedHandler = Operator.Handler;
if (Network.Initialised && !ForceInitialisation)
{
initialisedNumberCount = 0;
initialisedNDArrayCount = 0;
_logger.Info($"Skipping network initialisation because network was already initialised and force initialisation flag is set to false...");
}
else
{
InitialiseNetwork(handler, out initialisedNumberCount, out initialisedNDArrayCount);
}
Operator.Sigma = Sigma;
Operator.Handler = Operator.Handler ?? handler;
Operator.Network = Network;
Operator.Trainer = this;
// attach all given hooks
foreach (IHook hook in _globalHooks)
{
if (!Operator.AttachGlobalHook(hook))
{
_logger.Debug($"Skipped attaching global hook {hook} in trainer \"{Name}\", operator refused to attach it.");
}
}
foreach (IHook hook in _localHooks)
{
if (!Operator.AttachLocalHook(hook))
{
_logger.Debug($"Skipped attaching local hook {hook} in trainer \"{Name}\", operator refused to attach it.");
}
}
UpdateRegistry();
_initialised = true;
SigmaEnvironment.TaskManager.EndTask(prepareTask);
_logger.Info($"Done initialising trainer \"{Name}\" for handler {handler}, initialised {initialisedNDArrayCount} ndarrays and {initialisedNumberCount} numbers.");
}
public void TestTaskManagerBegin()
{
TaskManager manager = new TaskManager();
ITaskObserver task = manager.BeginTask(TaskType.Preprocess, "mnistdataset", exposed: false, indeterminate: true);
Assert.AreEqual(TaskType.Preprocess, task.Type);
Assert.AreEqual("mnistdataset", task.Description);
Assert.IsFalse(task.Exposed);
Assert.Less(task.Progress, 0.0f);
}
public override void ChargerCabs(ITaskObserver observer)
{
try
{
ChargerListeDonnéesCab(observer);
}
catch (Exception ex)
{
throw new Exception("Problème durant la recherche des CAB sur le site NMPP", ex);
}
}
public void ExecuteTestDataScriptsInFolder(TaskAttributes taskAttributes, string scriptDirectory, ITaskObserver taskObserver)
{
_schemaInitializer.EnsureTestDataSchemaCreated(taskAttributes.ConnectionSettings);
var sqlFilenames = _fileLocator.GetSqlFilenames(taskAttributes.ScriptDirectory, scriptDirectory);
foreach (string sqlFilename in sqlFilenames)
{
_testDataScriptExecutor.Execute(sqlFilename, taskAttributes.ConnectionSettings, taskObserver);
}
}
public override void ChargerCabs(ITaskObserver observer)
{
try
{
ChargerDossiersPao(observer);
}
catch (Exception ex)
{
throw new Exception($"Problème durant la lecture des dossiers de [{LibelléCourt}]", ex);
}
}
public void ExecuteRunAlwaysScriptsInFolder(TaskAttributes taskAttributes, string scriptDirectory, ITaskObserver taskObserver)
{
var sqlFilenames = _fileLocator.GetSqlFilenames(taskAttributes.ScriptDirectory, scriptDirectory);
foreach (string sqlFilename in sqlFilenames)
{
_scriptExecutor.ExecuteAlways(sqlFilename, taskAttributes.ConnectionSettings, taskObserver, taskAttributes.LogOnly);
}
_versioner.VersionDatabase(taskAttributes.ConnectionSettings, taskObserver);
}
public void Upgrade(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
string initializationMessage = _logMessageGenerator.GetInitialMessage(taskAttributes);
taskObserver.Log(initializationMessage);
IEnumerable<IDatabaseActionExecutor> executors = _actionExecutorFactory.GetExecutors(taskAttributes.RequestedDatabaseAction);
foreach (IDatabaseActionExecutor executor in executors)
{
executor.Execute(taskAttributes, taskObserver);
}
}
public AutomationTaskParameter(string clientID = "", string taskName = "", string commandURL = "", string acknowledgeURL = "",
Logger logger = null, bool isRepeat = false, int interval = 1000, ITaskObserver observer = null)
{
_client_id = clientID;
_name = taskName;
_interval = interval;
_url_cmd = commandURL;
_url_ack = acknowledgeURL;
_logger = logger;
_repeat = isRepeat;
_observer = observer;
}
public void TestTaskManagerCancel()
{
TaskManager manager = new TaskManager();
ITaskObserver task = manager.BeginTask(TaskType.Preprocess, "mnistdataset", exposed: true, indeterminate: true);
Assert.IsTrue(manager.GetTasks().Contains(task));
manager.CancelTask(task);
Assert.IsFalse(manager.GetTasks().Contains(task));
Assert.AreEqual(TaskObserveStatus.Canceled, task.Status);
}
public void Upgrade(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
string initializationMessage = _logMessageGenerator.GetInitialMessage(taskAttributes);
taskObserver.Log(initializationMessage);
IEnumerable <IDatabaseActionExecutor> executors = _actionExecutorFactory.GetExecutors(taskAttributes.RequestedDatabaseAction);
foreach (IDatabaseActionExecutor executor in executors)
{
executor.Execute(taskAttributes, taskObserver);
}
}
public void Import(string excelFile, string server, string database, bool integrated, string username, string password,
ITaskObserver taskObserver)
{
string message = string.Format("\nImporting Excel File '{0}' to Database '{1}' on Server '{2}'\n", excelFile, database, server);
taskObserver.Log(message);
var settings = new ConnectionSettings(server, database, integrated, username, password);
string sql = _fileLocator.ReadTextFile("Tarantino.Core", "Tarantino.Core.DatabaseManager.SqlFiles.ImportExcel.sql");
_tokenReplacer.Text = sql;
_tokenReplacer.Replace("DATABASE", database);
_tokenReplacer.Replace("EXCEL_FILE", excelFile);
_queryExecutor.ExecuteNonQuery(settings, _tokenReplacer.Text, false);
}
private Dictionary <string, INDArray> ExtractDirectFrom(object[] data, int blockIndex, IComputationHandler handler)
{
Dictionary <string, INDArray> namedBlocks = new Dictionary <string, INDArray>();
ITaskObserver prepareTask = SigmaEnvironment.TaskManager.BeginTask(TaskType.Prepare, "preparing extractors for dataset \"" + Name + "\"", indeterminate: true);
PrepareExtractors();
SigmaEnvironment.TaskManager.EndTask(prepareTask);
ITaskObserver extractTask = SigmaEnvironment.TaskManager.BeginTask(TaskType.Extract, $"extracting block {blockIndex} for dataset \"{Name}\"", indeterminate: true);
int extractorIndex = 0;
foreach (IRecordExtractor extractor in _recordExtractors)
{
_logger.Debug($"Extracting hierarchically from extractor {extractor} at index {extractorIndex}...");
Dictionary <string, INDArray> subNamedBlock = extractor.ExtractHierarchicalFrom(data[extractorIndex++], TargetBlockSizeRecords, handler);
//check if block size is 0, indicating we reached the end of the stream
if (subNamedBlock == null)
{
_lastAvailableBlockIndex = blockIndex - 1;
_logger.Debug($"Cannot extract block {blockIndex} for handler {handler}, the underlying stream for extractor {extractor} is unable to retrieve any more records. End of stream most likely reached.");
SigmaEnvironment.TaskManager.CancelTask(extractTask);
return(null);
}
foreach (string name in subNamedBlock.Keys)
{
if (namedBlocks.ContainsKey(name))
{
SigmaEnvironment.TaskManager.CancelTask(extractTask);
throw new ArgumentException($"Section name collision: {name} is already used by another extractor, current extractor {extractor} cannot use it again.");
}
else
{
namedBlocks.Add(name, subNamedBlock[name]);
}
}
}
SigmaEnvironment.TaskManager.EndTask(extractTask);
return(namedBlocks);
}
public void MarkScriptAsExecuted(ConnectionSettings settings, string scriptFilename, ITaskObserver task, string hash = "")
{
//for everytime scripts just delete the row. We could update it but either way has the same result
if (!string.IsNullOrEmpty(hash))
{
string deleteTemplate = "delete from usd_AppliedDatabaseScript where ScriptFile = '{0}'";
string deletesql = string.Format(deleteTemplate, scriptFilename);
_executor.ExecuteNonQueryTransactional(settings, deletesql);
}
string insertTemplate = "insert into usd_AppliedDatabaseScript (ScriptFile, DateApplied, hash) values ('{0}', getdate(), '{1}')";
string sql = string.Format(insertTemplate, scriptFilename, hash);
_executor.ExecuteNonQueryTransactional(settings, sql);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
_connectionDropper.Drop(taskAttributes.ConnectionSettings, taskObserver);
var sql = string.Format("ALTER DATABASE [{0}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE drop database [{0}]", taskAttributes.ConnectionSettings.Database);
try
{
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql, false);
}
catch (Exception)
{
taskObserver.Log(string.Format("Database '{0}' could not be dropped.", taskAttributes.ConnectionSettings.Database));
}
}
public void TestTaskManagerGetTasks()
{
TaskManager manager = new TaskManager();
ITaskObserver task1 = manager.BeginTask(TaskType.Download, "mnistdataset", exposed: true);
ITaskObserver task2 = manager.BeginTask(TaskType.Download, "mnistdataset", exposed: false);
ITaskObserver task3 = manager.BeginTask(TaskType.Download, "mnistdataset", exposed: true);
IEnumerable <ITaskObserver> tasks = manager.GetTasks(TaskType.Download);
Assert.IsTrue(tasks.Contains(task1));
Assert.IsFalse(tasks.Contains(task2));
Assert.IsTrue(tasks.Contains(task3));
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
_connectionDropper.Drop(taskAttributes.ConnectionSettings, taskObserver);
var sql = string.Format("ALTER DATABASE [{0}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE drop database [{0}]", taskAttributes.ConnectionSettings.Database);
try
{
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql, false);
}
catch(Exception)
{
taskObserver.Log(string.Format("Database '{0}' could not be dropped.", taskAttributes.ConnectionSettings.Database));
}
}
public void Import(string excelFile, string server, string database, bool integrated, string username, string password,
ITaskObserver taskObserver)
{
string message = string.Format("\nImporting Excel File '{0}' to Database '{1}' on Server '{2}'\n", excelFile, database, server);
taskObserver.Log(message);
var settings = new ConnectionSettings(server, database, integrated, username, password);
string sql = _fileLocator.ReadTextFile("Tarantino.Core", "Tarantino.Core.DatabaseManager.SqlFiles.ImportExcel.sql");
_tokenReplacer.Text = sql;
_tokenReplacer.Replace("DATABASE", database);
_tokenReplacer.Replace("EXCEL_FILE", excelFile);
_queryExecutor.ExecuteNonQuery(settings, _tokenReplacer.Text, false);
}
public void ExecuteScriptsInFolder(TaskAttributes taskAttributes, string scriptDirectory, ITaskObserver taskObserver)
{
_schemaInitializer.EnsureSchemaCreated(taskAttributes.ConnectionSettings);
var sqlFilenames = _fileLocator.GetSqlFilenames(taskAttributes.ScriptDirectory, scriptDirectory);
var filteredFilenames = _fileFilterService.GetFilteredFilenames(sqlFilenames, taskAttributes.SkipFileNameContaining);
foreach (string sqlFilename in filteredFilenames)
{
_scriptExecutor.Execute(sqlFilename, taskAttributes.ConnectionSettings, taskObserver);
}
_versioner.VersionDatabase(taskAttributes.ConnectionSettings, taskObserver);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
if (!_queryExecutor.CheckDatabaseExists(taskAttributes.ConnectionSettings))
{
taskObserver.Log(string.Format("Database does not exist. Attempting to create database before updating."));
string sql = string.Format("create database [{0}]", taskAttributes.ConnectionSettings.Database);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql);
taskObserver.Log(string.Format("Run scripts in Create folder."));
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Create", taskObserver);
}
taskObserver.Log(string.Format("Run scripts in Update folder."));
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Update", taskObserver);
taskObserver.Log(string.Format("Run scripts in Everytime folder."));
_folderExecutor.ExecuteChangedScriptsInFolder(taskAttributes, "Everytime", taskObserver);
}
public void Drop(ConnectionSettings settings, ITaskObserver taskObserver)
{
string message = string.Format("Dropping connections for database {0}\n", settings.Database);
taskObserver.Log(message);
string assembly = SqlDatabaseManager.SQL_FILE_ASSEMBLY;
string sqlFile = string.Format(SqlDatabaseManager.SQL_FILE_TEMPLATE, "DropConnections");
string sql = _fileLocator.ReadTextFile(assembly, sqlFile);
_replacer.Text = sql;
_replacer.Replace("DatabaseName", settings.Database);
sql = _replacer.Text;
_executor.ExecuteNonQuery(settings, sql);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
if (!_queryExecutor.CheckDatabaseExists(taskAttributes.ConnectionSettings))
{
taskObserver.Log(string.Format("Database does not exist. Attempting to create database before updating."));
string sql = string.Format("create database [{0}]", taskAttributes.ConnectionSettings.Database);
_queryExecutor.ExecuteNonQuery(taskAttributes.ConnectionSettings, sql);
taskObserver.Log(string.Format("Run scripts in Create folder."));
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Create", taskObserver);
}
taskObserver.Log(string.Format("Run scripts in Update folder."));
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Update", taskObserver);
taskObserver.Log(string.Format("Run scripts in Everytime folder."));
_folderExecutor.ExecuteChangedScriptsInFolder(taskAttributes, "Everytime", taskObserver);
}
public void CancelTask(ITaskObserver task)
{
if (task.Status != TaskObserveStatus.Running)
{
//nothing to do here, task is not even running
return;
}
OnTaskCanceled(task);
task.Status = TaskObserveStatus.Canceled;
lock (_runningObservers)
{
_runningObservers.Remove(task);
}
}
public void Drop(ConnectionSettings settings, ITaskObserver taskObserver)
{
string message = string.Format("Dropping connections for database {0}\n", settings.Database);
taskObserver.Log(message);
string assembly = SqlDatabaseManager.SQL_FILE_ASSEMBLY;
string sqlFile = string.Format(SqlDatabaseManager.SQL_FILE_TEMPLATE, "DropConnections");
string sql = _fileLocator.ReadTextFile(assembly, sqlFile);
_replacer.Text = sql;
_replacer.Replace("DatabaseName", settings.Database);
sql = _replacer.Text;
_executor.ExecuteNonQuery(settings, sql, false);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
_folderExecutor.ExecuteTestDataScriptsInFolder(taskAttributes, "TestData", taskObserver);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Update", taskObserver);
}
public void Execute(TaskAttributes taskAttributes, ITaskObserver taskObserver)
{
taskAttributes.LogOnly = true;
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Create", taskObserver);
_folderExecutor.ExecuteScriptsInFolder(taskAttributes, "Update", taskObserver);
}