private static string MoveToS3(AmazonS3Client s3Client, string bucket, int?chunkId, int?subChunkId,
IDataReader reader,
string tableName)
{
var folder = tableName;
string fileName;
if (chunkId.HasValue)
{
var indexOfSet = ServicesManager.GetIndexOfSet(chunkId.Value);
fileName = string.Format(@"{0}/{1}", indexOfSet, tableName + ".txt.gz." + chunkId + "." + subChunkId);
}
else
{
fileName = string.Format(@"{0}", folder + ".txt.gz");
}
if (!(AmazonS3Util.DoesS3BucketExist(s3Client, Settings.Current.Bucket)))
{
AmazonS3Helper.CreateBucket(s3Client, Settings.Current.Bucket);
}
AmazonS3Helper.CopyFile(s3Client, bucket, fileName, reader, tableName);
return(fileName);
}
private void LoadPregnancyDrug(bool storeToS3)
{
var sql = File.ReadAllText(Path.Combine(Settings.Settings.Current.Builder.Folder,
@"Core\Lookups\PregnancyDrug.sql"));
sql = sql.Replace("{sc}", Settings.Settings.Current.Building.VocabularySchemaName);
Console.WriteLine("PregnancyDrug - Loading...");
using (var connection =
SqlConnectionHelper.OpenOdbcConnection(Settings.Settings.Current.Building.VocabularyConnectionString))
using (var command = new OdbcCommand(sql, connection)
{
CommandTimeout = 0
})
using (var reader = command.ExecuteReader())
{
if (storeToS3)
{
var fileName =
$"{Settings.Settings.Current.Building.Vendor}/{Settings.Settings.Current.Building.Id}/Lookups/PregnancyDrug.txt.gz";
Console.WriteLine("PregnancyDrug - store to S3 | " + fileName);
using (var client = new AmazonS3Client(
Settings.Settings.Current.S3AwsAccessKeyId,
Settings.Settings.Current.S3AwsSecretAccessKey,
new AmazonS3Config
{
Timeout = TimeSpan.FromMinutes(60),
RegionEndpoint = Amazon.RegionEndpoint.USEast1,
MaxErrorRetry = 20,
}))
{
AmazonS3Helper.CopyFile(client, Settings.Settings.Current.Bucket,
fileName,
reader);
}
}
else
{
Console.WriteLine("PregnancyDrug - filling");
var lookup = new Lookup();
while (reader.Read())
{
lookup.Add(new LookupValue
{
ConceptId = int.Parse(reader[0].ToString()),
SourceCode = reader[0].ToString()
});
}
_lookups.Add("PregnancyDrug", lookup);
}
}
Console.WriteLine("PregnancyDrug - Done");
}
private static string SaveToS3CSV(AmazonS3Client s3Client, string bucket, int?chunkId, int?subChunkId,
IDataReader reader,
string tableName)
{
var folder = tableName;
string fileName;
if (chunkId.HasValue)
{
fileName = $@"{chunkId}/{tableName + ".txt.gz." + chunkId + "." + subChunkId}";
}
else
{
fileName = $@"{folder + ".txt.gz"}";
}
AmazonS3Helper.CopyFile(s3Client, bucket, fileName, reader);
return(fileName);
}
private void Load(IEnumerable <EntityDefinition> definitions, bool storeToS3)
{
if (definitions == null)
{
return;
}
foreach (var ed in definitions)
{
ed.Vocabulary = this;
if (ed.Concepts == null)
{
continue;
}
foreach (var c in ed.Concepts)
{
if (c.ConceptIdMappers == null)
{
continue;
}
foreach (var conceptIdMapper in c.ConceptIdMappers)
{
if (!string.IsNullOrEmpty(conceptIdMapper.Lookup))
{
if (!_lookups.ContainsKey(conceptIdMapper.Lookup))
{
string sql = string.Empty;
var vendorFolder = Settings.Settings.Current.Building.Vendor.GetAttribute <FolderAttribute>().Value;
var sqlFileDestination = Path.Combine(Settings.Settings.Current.Builder.Folder, "Core", "Transformation",
vendorFolder, "Lookups", conceptIdMapper.Lookup + ".sql");
var baseSql =
File.ReadAllText(Path.Combine(Settings.Settings.Current.Builder.Folder, @"Core\Lookups\Base.sql"));
sql = File.ReadAllText(sqlFileDestination);
sql = sql.Replace("{base}", baseSql);
sql = sql.Replace("{sc}", Settings.Settings.Current.Building.VocabularySchemaName);
try
{
Console.WriteLine(conceptIdMapper.Lookup + " - Loading...");
using (var connection = SqlConnectionHelper.OpenOdbcConnection(Settings.Settings.Current.Building.VocabularyConnectionString))
using (var command = new OdbcCommand(sql, connection)
{
CommandTimeout = 0
})
using (var reader = command.ExecuteReader())
{
if (storeToS3)
{
var fileName =
$"{Settings.Settings.Current.Building.Vendor}/{Settings.Settings.Current.Building.Id}/Lookups/{conceptIdMapper.Lookup}.txt.gz";
Console.WriteLine(conceptIdMapper.Lookup + " - store to S3 | " + fileName);
using (var client = new AmazonS3Client(
Settings.Settings.Current.S3AwsAccessKeyId,
Settings.Settings.Current.S3AwsSecretAccessKey,
new AmazonS3Config
{
Timeout = TimeSpan.FromMinutes(60),
RegionEndpoint = Amazon.RegionEndpoint.USEast1,
MaxErrorRetry = 20,
}))
{
AmazonS3Helper.CopyFile(client, Settings.Settings.Current.Bucket,
fileName,
reader);
}
}
else
{
Console.WriteLine(conceptIdMapper.Lookup + " - filling");
var lookup = new Lookup();
while (reader.Read())
{
var lv = CreateLookupValue(reader);
lookup.Add(lv);
}
_lookups.Add(conceptIdMapper.Lookup, lookup);
}
}
Console.WriteLine(conceptIdMapper.Lookup + " - Done");
}
catch (Exception)
{
Console.WriteLine("Lookup error [file]: " + sqlFileDestination);
Console.WriteLine("Lookup error [query]: " + sql);
Logger.WriteWarning("Lookup error [file]: " + sqlFileDestination);
Logger.WriteWarning("Lookup error [query]: " + sql);
throw;
}
}
}
}
}
}
}