public bool Execute(ISessionContext context)
{
var logger = context.GetLogger();
var options = context.Options;
try
{
var inputPath = options.Get("JobLoader.SearchPath", System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Jobloader", "Input", "*"));
var backupPath = options.Get("JobLoader.BackupPath", System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Jobloader", "Backup"));
if (String.IsNullOrEmpty(inputPath))
{
throw new ArgumentNullException("JobLoader.SearchPath");
}
var searchOptions = System.IO.SearchOption.AllDirectories;
var dir = System.IO.Path.GetDirectoryName(inputPath);
var searchPattern = System.IO.Path.GetFileName(inputPath);
foreach (var f in System.IO.Directory.EnumerateFiles(dir, searchPattern, searchOptions).ToList())
{
logger.Info("File found: " + f);
// load and parse file content
var encoding = SimpleHelpers.FileEncoding.DetectFileEncoding(f);
if (encoding == null)
{
logger.Debug("Ignoring file: could not detect file encoding");
continue;
}
var json = System.IO.File.ReadAllText(f, encoding);
var item = Newtonsoft.Json.JsonConvert.DeserializeObject <PipelineJob> (json);
// save to database
if (item != null)
{
item.SetScheduler(item.Scheduler != null ? item.Scheduler.ToArray() : new string[0]);
_storageContext.SavePipelineJob(item);
logger.Info("Pipeline imported: " + item.Id);
}
// move file
if (!String.IsNullOrEmpty(backupPath))
{
(new System.IO.DirectoryInfo(backupPath)).Create();
System.IO.File.Move(f, System.IO.Path.Combine(backupPath, System.IO.Path.GetFileName(f)));
logger.Debug("File moved to backup folder");
}
}
}
catch (Exception ex)
{
context.Error = ex.Message;
logger.Error(ex);
return(false);
}
return(true);
}
public bool Execute(ISessionContext context)
{
_logger = context.GetLogger();
var _options = context.Options;
try
{
var processFilename = _options.Get("processFilename", "");
if (String.IsNullOrWhiteSpace(processFilename))
{
throw new ArgumentNullException("processFilename");
}
var taskWorkDir = _options.Get("processWorkingDirectory", "");
if (String.IsNullOrWhiteSpace(taskWorkDir))
{
taskWorkDir = System.IO.Path.GetDirectoryName(processFilename);
}
fireAndForget = _options.Get("fireAndForget", false);
// save ammount of log to produce
_maxLogLines = 10 - (int)_logger.GetLogLevel();
return(ExecuteProcess(processFilename,
_options.Get("parameters", ""),
TimeSpan.FromSeconds(_options.Get("waitInSeconds", 0)),
!fireAndForget,
!fireAndForget,
taskWorkDir));
}
catch (Exception ex)
{
context.Error = ex.Message;
_logger.Error(ex);
return(false);
}
}
public bool Execute(ISessionContext context)
{
var logger = context.GetLogger();
var options = context.Options;
AWSS3Helper s3 = null;
List <string> files = null;
FileTransferDetails parsedErrorLocation = null;
try
{
var inputSearchPath = options.Get("inputSearchPath", "");
if (String.IsNullOrEmpty(inputSearchPath))
{
throw new ArgumentNullException("inputSearchPath");
}
var backupLocation = options.Get("backupLocation", "");
if (String.IsNullOrEmpty(backupLocation))
{
throw new ArgumentNullException("backupLocation");
}
var loadScript = options.Get("sqlScriptPath", "");
if (String.IsNullOrEmpty(loadScript))
{
throw new ArgumentNullException("sqlScriptPath");
}
var errorLocation = options.Get("errorLocation", "");
if (String.IsNullOrEmpty(errorLocation))
{
throw new ArgumentNullException("errorLocation");
}
var customCSharpScriptPath = options.Get("customCSharpScriptPath", "");
if (String.IsNullOrEmpty(customCSharpScriptPath))
{
throw new ArgumentNullException("customCSharpScriptPath");
}
// prepare paths
var parsedInput = FileTransferDetails.ParseSearchPath(inputSearchPath);
var parsedLoadScript = FileTransferDetails.ParseSearchPath(loadScript);
var parsedBackupLocation = FileTransferDetails.ParseSearchPath(backupLocation);
parsedErrorLocation = FileTransferDetails.ParseSearchPath(errorLocation);
var parsedCustomCSharpScriptPath = FileTransferDetails.ParseSearchPath(customCSharpScriptPath);
// open s3 connection
s3 = new AWSS3Helper(options.Get("awsAccessKey", ""), options.Get("awsSecretAccessKey", ""), parsedInput.BucketName, Amazon.RegionEndpoint.USEast1, true);
var csharpScript = s3.ReadFileAsText(parsedCustomCSharpScriptPath.FilePath, true);
// generate code
var evaluator = ScriptEvaluator.CompileAndCreateModel(csharpScript);
if (evaluator.HasError || evaluator.Model == null)
{
throw new Exception("Script compilation error. " + (evaluator.Message ?? "<empty>"));
}
// 1. check if there is any new file
files = GetFilesFromS3(s3, parsedInput).ToList();
if (files.Any())
{
logger.Info("Files found: " + files.Count);
}
else
{
logger.Debug("No file found");
return(false);
}
var connectionString = RedshiftHelper.GetConnectionString(context);
foreach (var f in files)
{
var sqlScript = s3.ReadFileAsText(parsedLoadScript.FilePath, true);
if (String.IsNullOrEmpty(sqlScript))
{
throw new Exception("invalid sql script");
}
using (var conn = new Npgsql.NpgsqlConnection(connectionString))
{
conn.Open();
var fullFilename = System.IO.Path.Combine("s3://", parsedInput.BucketName, f.Trim()).Replace('\\', '/');
options.Set("InputFilename", fullFilename);
evaluator.Model.Initialize(conn, s3, context);
evaluator.Model.BeforeExecution();
sqlScript = evaluator.Model.PrepareSqlCOPYCommand(sqlScript);
// Create a PostgeSQL connection string.
ExecuteRedshiftLoad(connectionString, logger, sqlScript, new List <string> ()
{
f
}, parsedInput);
logger.Debug("Moving files to backup folder");
evaluator.Model.AfterExecution();
// move files
var destName = System.IO.Path.Combine(parsedBackupLocation.FilePath, System.IO.Path.GetFileName(f));
s3.MoveFile(f, destName, false);
}
logger.Success("Done");
}
}
catch (Exception ex)
{
context.Error = ex.Message;
logger.Error(ex);
try
{
if (files != null && s3 != null)
{
// move files
foreach (var f in files)
{
var destName = System.IO.Path.Combine(parsedErrorLocation.FilePath, System.IO.Path.GetFileName(f));
s3.MoveFile(f, destName, false);
}
}
}
catch { }
return(false);
}
return(true);
}
public bool Execute(ISessionContext context)
{
var logger = context.GetLogger();
var options = context.Options;
AWSS3Helper s3 = null;
try
{
var loadScript = options.Get("sqlScriptPath", "");
var customCSharpScriptPath = options.Get("customCSharpScriptPath", "");
if ((!String.IsNullOrWhiteSpace(loadScript)) && (!String.IsNullOrWhiteSpace(customCSharpScriptPath)))
{
throw new Exception("No action configured");
}
// prepare paths
var parsedLoadScript = FileTransferDetails.ParseSearchPath(loadScript);
var parsedCustomCSharpScriptPath = FileTransferDetails.ParseSearchPath(customCSharpScriptPath);
// open s3 connection
s3 = new AWSS3Helper(options.Get("awsAccessKey", ""), options.Get("awsSecretAccessKey", ""), parsedLoadScript.BucketName, Amazon.RegionEndpoint.USEast1, true);
// load sql script
string sqlScript = null;
if (!String.IsNullOrWhiteSpace(loadScript))
{
sqlScript = s3.ReadFileAsText(parsedLoadScript.FilePath, true);
}
// generate code
IAWSRedshiftPluginDynamicScript customCode = null;
if (!String.IsNullOrWhiteSpace(customCSharpScriptPath))
{
// load custom code
var csharpScript = s3.ReadFileAsText(parsedCustomCSharpScriptPath.FilePath, true);
var evaluator = ScriptEvaluator.CompileAndCreateModel(csharpScript);
if (evaluator.HasError || evaluator.Model == null)
{
throw new Exception("Script compilation error. " + (evaluator.Message ?? "<empty>"));
}
customCode = evaluator.Model;
}
// execute commands
using (var conn = new Npgsql.NpgsqlConnection(RedshiftHelper.GetConnectionString(context)))
{
conn.Open();
if (customCode != null)
{
logger.Debug("Custom csharp code Initialize");
customCode.Initialize(conn, s3, context);
logger.Debug("Custom csharp code BeforeExecution");
customCode.BeforeExecution();
logger.Debug("Custom csharp code PrepareSqlCOPYCommand");
if (!String.IsNullOrEmpty(sqlScript))
{
sqlScript = customCode.PrepareSqlCOPYCommand(sqlScript);
}
}
if (!String.IsNullOrEmpty(sqlScript))
{
logger.Debug("SQL command start");
try
{
conn.Execute(sqlScript);
}
catch (Exception ex)
{
// do nothing in case of timeout... some operations may take a while to complete...
if (ex.Message.IndexOf("timeout", StringComparison.OrdinalIgnoreCase) < 0)
{
throw ex;
}
logger.Info("SQL command executed, but is still running (connection timeout)...");
}
logger.Debug("SQL command end");
}
if (customCode != null)
{
logger.Debug("Custom csharp code AfterExecution");
customCode.AfterExecution();
}
}
logger.Success("Done");
}
catch (Exception ex)
{
context.Error = ex.Message;
logger.Error(ex);
return(false);
}
return(true);
}
/// <summary>
/// Implements this plugin execution's logic
/// </summary>
public bool Execute(ISessionContext context)
{
var logger = context.GetLogger();
var options = context.Options;
try
{
// Parsing Parameters
String notificationType = options.Get("notificationType", "EMAIL");
String contacts = options.Get("contacts", String.Empty);
String subject = options.Get("subject", String.Empty);
String message = options.Get("message", String.Empty);
// Email Specific Configurations
String EmailFrom = options.Get("emailFrom", String.Empty);
String EmailLogin = options.Get("emailLogin", String.Empty);
String EmailPassword = options.Get("emailPassword", String.Empty);
String EmailSmtp = options.Get("emailSmtp", "");
int EmailPort = options.Get("emailPort", 587); // Gmail port is the default
bool EmailSsl = options.Get("emailSsl", false);
bool EmailIsHtml = options.Get("emailIsHtml", true);
// Setting up Configuration
NotificationService.Configuration = new NotificationConfig()
{
EmailFrom = EmailFrom,
EmailLogin = EmailLogin,
EmailPassword = EmailPassword,
EmailSmtp = EmailSmtp,
EmailPort = EmailPort,
EmailSsl = EmailSsl,
EmailIsHtml = EmailIsHtml
};
// Picking up Notification Type
NotificationTypes notifType = NotificationTypes.EMAIL;
switch (notificationType.ToUpperInvariant())
{
case "SMS":
notifType = NotificationTypes.SMS;
break;
case "CALL":
notifType = NotificationTypes.CALL;
break;
case "EMAIL":
notifType = NotificationTypes.EMAIL;
break;
}
// Parsing Contacts List
List <String> contactsList = contacts.Split(' ', ',', ';', '|').Where(i => !String.IsNullOrEmpty(i)).ToList();
// Sending Notification
var notificationResponse = NotificationService.Send(notifType, contactsList, message, subject);
// Checking for error
if (!notificationResponse.Status)
{
logger.Error("Error Sending Email:" + notificationResponse.Message);
}
return(notificationResponse.Status);
}
catch (Exception ex)
{
context.Error = ex.Message;
logger.Error(ex);
return(false);
}
}
public bool Execute(ISessionContext context)
{
var _logger = context.GetLogger();
var _options = context.Options;
IFileTransfer input = null;
string lastFile = null;
int filesCount = 0;
int maxFilesCount = Int32.MaxValue;
var fileTransferService = context.GetContainer().GetInstanceOf <IFileService> ();
try
{
var searchPath = _options.Get("inputPath", "");
if (String.IsNullOrEmpty(searchPath))
{
throw new ArgumentNullException("inputPath");
}
var deleteSourceFile = _options.Get <bool>("deleteSourceFile", false);
// prepare paths
input = fileTransferService.Open(searchPath, _options);
if (!input.IsOpened())
{
throw new Exception("Invalid searchPath: " + searchPath);
}
maxFilesCount = _options.Get("maxFileCount", maxFilesCount);
if (maxFilesCount <= 0)
{
maxFilesCount = Int32.MaxValue;
}
var defaultEncoding = Encoding.GetEncoding(_options.Get("encoding", "ISO-8859-1"));
// open connection
string line;
Layout layout = new Layout();
foreach (var f in input.GetFileStreams())
{
lastFile = f.FileName;
_logger.Info("File found: " + lastFile);
filesCount++;
// read file
using (var reader = new StreamReader(f.FileStream, defaultEncoding, true))
{
int n = 1;
while ((line = reader.ReadLine()) != null)
{
context.Emit(layout.Create()
.Set("fileName", f.FileName)
.Set("fileNumber", filesCount)
.Set("filePath", searchPath)
.Set("lineNumber", n++)
.Set("line", line));
}
}
// If backup folder exists, move file
if (!String.IsNullOrWhiteSpace(_options.Get("backupLocation", "")))
{
// TODO: implement move operation if location are the same!
using (var backupLocation = fileTransferService.Open(_options.Get("backupLocation", ""), _options))
{
var destName = backupLocation.Details.GetDestinationPath(f.FileName);
backupLocation.SendFile(input.GetFileStream(f.FileName).FileStream, destName, true);
_logger.Info("Backup file created: " + destName);
}
}
// If DeleSource is set
if (deleteSourceFile)
{
input.RemoveFile(f.FileName);
_logger.Info("File deleted: " + f.FileName);
}
// limit
if (filesCount >= maxFilesCount)
{
break;
}
}
if (filesCount > 0)
{
_logger.Success("Done");
return(true);
}
else
{
_logger.Debug("No Files Found on: " + searchPath);
return(true);
}
}
catch (Exception ex)
{
context.Error = ex.Message;
_logger.Error(ex);
try
{
if (lastFile != null && input != null && !String.IsNullOrEmpty(_options.Get("errorLocation", "")))
{
// move files
using (var parsedErrorLocation = fileTransferService.Open(_options.Get("errorLocation", ""), _options))
{
var destName = parsedErrorLocation.Details.GetDestinationPath(lastFile);
parsedErrorLocation.SendFile(input.GetFileStream(lastFile).FileStream, destName, true);
}
}
}
catch { }
return(false);
}
finally
{
if (input != null)
{
input.Dispose();
}
}
}
public bool Execute(ISessionContext context)
{
_context = context;
_logger = context.GetLogger();
_options = context.Options;
IFileTransfer input = null;
_fileTransferService = context.GetContainer().GetInstanceOf <IFileService> ();
_deleteSourceFile = _options.Get <bool> ("deleteSourceFile", false);
_retryCount = _options.Get <int> ("retryCount", 2);
_retryWait = TimeSpan.FromMilliseconds(_options.Get <int> ("retryWaitMs", 250));
try
{
var searchPath = _options.Get("inputPath", _options.Get("searchPath", ""));
if (String.IsNullOrEmpty(searchPath))
{
throw new ArgumentNullException("inputPath");
}
if (String.IsNullOrEmpty(_options.Get("outputPath", "")))
{
throw new ArgumentNullException("outputPath");
}
_maxFilesCount = _options.Get("maxFileCount", _maxFilesCount);
if (_maxFilesCount <= 0)
{
_maxFilesCount = Int32.MaxValue;
}
// prepare paths
input = _fileTransferService.Open(searchPath, _options);
if (!input.IsOpened())
{
throw new Exception(String.Format("Invalid inputPath, {0}: {1}", input.LastError ?? "", searchPath));
}
// try move files
ParallelTasks <FileTransferInfo> .Process(input.ListFiles().Take(_maxFilesCount), 0, _options.Get("maxConcurrency", 2), f => MoveFileInternal(f, searchPath));
if (!String.IsNullOrEmpty(input.LastError))
{
_logger.Warn(input.LastError);
}
if (_filesCount > 0)
{
_logger.Success("Done");
return(true);
}
else
{
_logger.Debug("No Files Found on: " + searchPath);
return(true);
}
}
catch (Exception ex)
{
context.Error = ex.Message;
_logger.Error(ex);
return(false);
}
finally
{
if (input != null)
{
input.Dispose();
}
}
}
public bool Execute(ISessionContext context)
{
var logger = context.GetLogger();
var options = context.Options;
AWSS3Helper s3 = null;
List <string> files = null;
FileTransferDetails parsedErrorLocation = null;
try
{
var inputSearchPath = options.Get("inputSearchPath", "");
if (String.IsNullOrEmpty(inputSearchPath))
{
throw new ArgumentNullException("inputSearchPath");
}
var backupLocation = options.Get("backupLocation", "");
if (String.IsNullOrEmpty(backupLocation))
{
throw new ArgumentNullException("backupLocation");
}
var loadScript = options.Get("sqlScriptPath", "");
if (String.IsNullOrEmpty(loadScript))
{
throw new ArgumentNullException("sqlScriptPath");
}
var errorLocation = options.Get("errorLocation", "");
if (String.IsNullOrEmpty(errorLocation))
{
throw new ArgumentNullException("errorLocation");
}
// prepare paths
var parsedInput = FileTransferDetails.ParseSearchPath(inputSearchPath);
var parsedLoadScript = FileTransferDetails.ParseSearchPath(loadScript);
var parsedBackupLocation = FileTransferDetails.ParseSearchPath(backupLocation);
parsedErrorLocation = FileTransferDetails.ParseSearchPath(errorLocation);
// open s3 connection
s3 = new AWSS3Helper(options.Get("awsAccessKey", ""), options.Get("awsSecretAccessKey", ""), parsedInput.BucketName, Amazon.RegionEndpoint.USEast1, true);
// 1. check if there is any new file
files = GetFilesFromS3(s3, parsedInput).Where(f => !f.EndsWith("/")).ToList();
if (files.Any())
{
logger.Info("Files found: " + files.Count);
var sqlScript = s3.ReadFileAsText(parsedLoadScript.FilePath, false);
if (String.IsNullOrEmpty(sqlScript))
{
throw new Exception("Invalid sql script: " + parsedLoadScript.FilePath);
}
// Create a PostgeSQL connection string.
var connectionString = RedshiftHelper.GetConnectionString(context);
ExecuteRedshiftLoad(connectionString, logger, sqlScript, files, parsedInput);
logger.Debug("Moving files to backup folder");
// move files
// TODO: what happens if move fails?
foreach (var f in files)
{
var destName = System.IO.Path.Combine(parsedBackupLocation.FilePath, System.IO.Path.GetFileName(f));
if (s3.MoveFile(f, destName, false))
{
System.Threading.Thread.Sleep(250);
if (s3.MoveFile(f, destName, false))
{
logger.Error(String.Format("Error moving file {0}, {1}", f, s3.LastError));
}
}
}
logger.Success("Done");
return(true);
}
else
{
logger.Debug("No file found");
return(false);
}
}
catch (Exception ex)
{
context.Error = ex.Message;
logger.Error(ex);
try
{
if (files != null && s3 != null)
{
// move files
foreach (var f in files)
{
var destName = System.IO.Path.Combine(parsedErrorLocation.FilePath, System.IO.Path.GetFileName(f));
s3.MoveFile(f, destName, false);
}
}
} catch {}
return(false);
}
return(true);
}
public bool Execute(ISessionContext context)
{
var _logger = context.GetLogger();
var _options = context.Options;
try
{
var searchPath = _options.Get("searchPath", "");
if (String.IsNullOrEmpty(searchPath))
{
throw new ArgumentNullException("searchPath");
}
var destinationPath = _options.Get("destinationPath", "");
if (String.IsNullOrEmpty(destinationPath))
{
throw new ArgumentNullException("destinationPath");
}
var deleteSourceFile = _options.Get <bool> ("deleteSourceFile", false);
maxFilesCount = _options.Get("maxFileCount", maxFilesCount);
if (maxFilesCount <= 0)
{
maxFilesCount = Int32.MaxValue;
}
var defaultEncoding = Encoding.GetEncoding(_options.Get("encoding", "ISO-8859-1"));
// prepare paths
var parsedInput = FileTransferService.Create(searchPath, _options);
if (!parsedInput.HasConnectionString)
{
throw new Exception("Invalid searchPath: " + searchPath);
}
var parsedOutput = FileTransferService.Create(destinationPath, _options);
if (!parsedOutput.HasConnectionString)
{
throw new Exception("Invalid destinationPath: " + destinationPath);
}
var parsedBackupLocation = FileTransferService.Create(_options.Get("backupLocation", ""), _options);
parsedErrorLocation.Parse(_options.Get("errorLocation", ""), _options);
// open connection
Layout layout = new Layout();
input = parsedInput.OpenConnection();
output = parsedOutput.OpenConnection();
// try get files
foreach (var f in input.GetFileStreams())
{
lastFile = f.FileName;
_logger.Info("File found: " + lastFile);
filesCount++;
// upload file
var fileName = parsedOutput.GetDestinationPath(f.FileName);
output.SendFile(f.FileStream, fileName, true);
context.Emit(layout.Create()
.Set("FileName", fileName)
.Set("FileCount", filesCount)
.Set("FilePath", destinationPath)
.Set("SourcePath", searchPath)
.Set("SourceFileName", f.FileName));
// If backup folder exists, move file
if (parsedBackupLocation.HasConnectionString)
{
// TODO: implement move operation if location are the same!
var destName = parsedBackupLocation.GetDestinationPath(f.FileName);
using (var backup = parsedBackupLocation.OpenConnection())
backup.SendFile(input.GetFileStream(f.FileName).FileStream, destName, true);
}
// If DeleSource is set
if (deleteSourceFile)
{
input.RemoveFile(f.FileName);
}
// limit
if (filesCount >= maxFilesCount)
{
break;
}
}
if (filesCount > 0)
{
_logger.Success("Done");
return(true);
}
else
{
_logger.Debug("No Files Found on: " + searchPath);
return(true);
}
}
catch (Exception ex)
{
context.Error = ex.Message;
_logger.Error(ex);
try
{
if (lastFile != null && input != null && parsedErrorLocation.HasConnectionString)
{ // move files
var destName = parsedErrorLocation.GetDestinationPath(lastFile);
using (var backup = parsedErrorLocation.OpenConnection())
backup.SendFile(input.GetFileStream(lastFile).FileStream, destName, true);
}
}
catch { }
return(false);
}
finally
{
if (output != null)
{
output.Dispose();
}
if (input != null)
{
input.Dispose();
}
}
}
