private void ExecuteRedshiftLoad(string connectionString, IActionLogger logger, string script, List <string> files, FileTransferDetails filesDetails)
{
var dtStart = DateTime.UtcNow;
// prepare files full name
HashSet <string> fileMap = new HashSet <string> (files
.Select(f => System.IO.Path.Combine("s3://", filesDetails.BucketName, f.Trim()).Replace('\\', '/')),
StringComparer.OrdinalIgnoreCase);
// execute redshift load.
logger.Debug("SQL Script start");
RedshiftHelper.RunLoad(connectionString, script);
logger.Debug("SQL Script end");
int num_files = files.Count;
do
{
num_files = 0;
foreach (var i in stv_load_state.Read(connectionString))
{
if (fileMap.Contains(i.current_file))
{
num_files++;
}
}
// wait a while
if (num_files > 0)
{
// sanity check... 2 hours to run a load operation is too long...
if ((DateTime.UtcNow - dtStart) > TimeSpan.FromHours(3))
{
break;
}
// wait a while
System.Threading.Thread.Sleep(10000);
}
}while (num_files > 0);
// check load errors table
foreach (var i in stl_load_errors.Read(connectionString, dtStart))
{
if (fileMap.Contains(i.filename.Trim()))
{
throw new Exception("Load error detected <stl_load_errors>: " + Newtonsoft.Json.JsonConvert.SerializeObject(i));
}
}
// check commited files table
var filesLoaded = new HashSet <string> (stl_load_commits.ReadCommitedFiles(connectionString, dtStart).Select(f => f.Trim()), StringComparer.OrdinalIgnoreCase);
foreach (var f in fileMap)
{
if (!filesLoaded.Contains(f))
{
throw new Exception("Load error; file not commited <stl_load_commits>: " + f);
}
}
logger.Debug("Files loaded");
}
