public void Process()
{
try
{
var channelFactory = ServicesManager.GetChannelFactory <IVocabularyService>("net.pipe://localhost/VocabularyServiceEndpoint");
vocabularyService = channelFactory.CreateChannel();
var part = new ChunkPart(chunkId, createPersonBuilder, "0");
part.Load();
part.Build(vocabularyService);
part.Save();
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
dbChunk.ChunkComplete(chunkId);
channelFactory.Close();
}
catch (Exception e)
{
Logger.WriteError(chunkId, e);
throw;
}
}
private static void SetChunksAsComplete(int[] chunkIds, Stopwatch timer)
{
var attempt = 0;
var saved = false;
while (!saved)
{
try
{
attempt++;
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
foreach (var chunkId in chunkIds)
{
dbChunk.ChunkComplete(chunkId);
}
timer.Stop();
Logger.Write(null, LogMessageTypes.Info, string.Format("Save - {0} ms", timer.ElapsedMilliseconds));
saved = true;
}
catch (Exception)
{
if (attempt <= 10)
{
Thread.Sleep(1000);
}
else
{
throw;
}
}
}
}
public void Process()
{
try
{
Console.WriteLine("DatabaseChunkBuilder");
var dbChunk = new DbChunk(Settings.Settings.Current.Building.BuilderConnectionString);
var part = new DatabaseChunkPart(_chunkId, _createPersonBuilder, "0", 0);
var timer = new Stopwatch();
timer.Start();
part.Load();
Logger.Write(_chunkId, LogMessageTypes.Info,
$"Loaded - {timer.ElapsedMilliseconds} ms | {(GC.GetTotalMemory(false) / 1024f) / 1024f} Mb");
part.Build();
part.Save();
dbChunk.ChunkComplete(_chunkId, Settings.Settings.Current.Building.Id.Value);
Console.WriteLine($"ChunkId={_chunkId} was complete");
}
catch (Exception e)
{
Logger.WriteError(_chunkId, e);
throw;
}
}
public void Process()
{
try
{
var channelFactory = ServicesManager.GetChannelFactory <IVocabularyService>("net.pipe://localhost/VocabularyServiceEndpoint");
vocabularyService = channelFactory.CreateChannel();
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
var part = new ChunkPart(chunkId, createPersonBuilder, "0");
var timer = new Stopwatch();
timer.Start();
part.Load();
dbChunk.ChunkLoaded(chunkId);
Logger.Write(chunkId, LogMessageTypes.Info, string.Format("Loaded - {0} ms | {1} Mb", timer.ElapsedMilliseconds, (GC.GetTotalMemory(false) / 1024f) / 1024f));
part.Build(vocabularyService);
part.Save();
if (Settings.Current.Building.DestinationEngine.Database == Database.Redshift)
{
ServicesManager.AddToSaveQueue(chunkId);
Logger.Write(null, LogMessageTypes.Debug, "AddToSaveQueue " + chunkId);
}
else
{
dbChunk.ChunkComplete(chunkId);
}
channelFactory.Close();
}
catch (Exception e)
{
Logger.WriteError(chunkId, e);
throw;
}
}
public void Save(ChunkData chunk)
{
var timer = new Stopwatch();
timer.Start();
try
{
UpdateOffset(chunk.KeyMasterOffset);
Write(chunk.Id, new PersonDataReader(chunk.Persons.ToList()), "PERSON");
Write(chunk.Id, new ObservationPeriodDataReader(chunk.ObservationPeriods.ToList()), "OBSERVATION_PERIOD");
Write(chunk.Id, new PayerPlanPeriodDataReader(chunk.PayerPlanPeriods.ToList(), chunk.KeyMasterOffset), "PAYER_PLAN_PERIOD");
Write(chunk.Id, new ConditionOccurrenceDataReader(chunk.ConditionOccurrences.ToList(), chunk.KeyMasterOffset), "CONDITION_OCCURRENCE");
Write(chunk.Id, new DeathDataReader(chunk.Deaths.ToList()), "DEATH");
Write(chunk.Id, new DrugExposureDataReader(chunk.DrugExposures.ToList(), chunk.KeyMasterOffset), "DRUG_EXPOSURE");
Write(chunk.Id, new DrugCostDataReader(chunk.DrugCost.ToList(), chunk.KeyMasterOffset), "DRUG_COST");
Write(chunk.Id, new ObservationDataReader(chunk.Observations.ToList(), chunk.KeyMasterOffset), "OBSERVATION");
Write(chunk.Id, new VisitOccurrenceDataReader(chunk.VisitOccurrences.ToList(), chunk.KeyMasterOffset), "VISIT_OCCURRENCE");
Write(chunk.Id, new ProcedureOccurrenceDataReader(chunk.ProcedureOccurrences.ToList(), chunk.KeyMasterOffset), "PROCEDURE_OCCURRENCE");
Write(chunk.Id, new ProcedureCostDataReader(chunk.ProcedureCost.ToList(), chunk.KeyMasterOffset), "PROCEDURE_COST");
Write(chunk.Id, new DrugEraDataReader(chunk.DrugEra.ToList()), "DRUG_ERA");
Write(chunk.Id, new ConditionEraDataReader(chunk.ConditionEra.ToList()), "CONDITION_ERA");
Write(chunk.Id, new CohortDataReader(chunk.Cohort.ToList()), "COHORT");
Commit();
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
dbChunk.ChunkComplete(chunk.Id);
}
catch (Exception e)
{
Logger.WriteError(chunk.Id, e);
Rollback();
Logger.Write(chunk.Id, LogMessageTypes.Debug, "Rollback - Complete");
}
timer.Stop();
}
public void Save(ChunkData chunk)
{
var timer = new Stopwatch();
timer.Start();
try
{
UpdateOffset(chunk.KeyMasterOffset);
Write(chunk.Id, new PersonDataReader(chunk.Persons.ToList()), "PERSON");
Write(chunk.Id, new ObservationPeriodDataReader(chunk.ObservationPeriods.ToList()), "OBSERVATION_PERIOD");
Write(chunk.Id, new PayerPlanPeriodDataReader(chunk.PayerPlanPeriods.ToList(), chunk.KeyMasterOffset), "PAYER_PLAN_PERIOD");
Write(chunk.Id, new ConditionOccurrenceDataReader(chunk.ConditionOccurrences.ToList(), chunk.KeyMasterOffset), "CONDITION_OCCURRENCE");
Write(chunk.Id, new DeathDataReader(chunk.Deaths.ToList()), "DEATH");
Write(chunk.Id, new DrugExposureDataReader(chunk.DrugExposures.ToList(), chunk.KeyMasterOffset), "DRUG_EXPOSURE");
Write(chunk.Id, new DrugCostDataReader(chunk.DrugCost.ToList(), chunk.KeyMasterOffset), "DRUG_COST");
Write(chunk.Id, new ObservationDataReader(chunk.Observations.ToList(), chunk.KeyMasterOffset), "OBSERVATION");
Write(chunk.Id, new VisitOccurrenceDataReader(chunk.VisitOccurrences.ToList(), chunk.KeyMasterOffset), "VISIT_OCCURRENCE");
Write(chunk.Id, new ProcedureOccurrenceDataReader(chunk.ProcedureOccurrences.ToList(), chunk.KeyMasterOffset), "PROCEDURE_OCCURRENCE");
Write(chunk.Id, new ProcedureCostDataReader(chunk.ProcedureCost.ToList(), chunk.KeyMasterOffset), "PROCEDURE_COST");
Write(chunk.Id, new DrugEraDataReader(chunk.DrugEra.ToList()), "DRUG_ERA");
Write(chunk.Id, new ConditionEraDataReader(chunk.ConditionEra.ToList()), "CONDITION_ERA");
Write(chunk.Id, new CohortDataReader(chunk.Cohort.ToList()), "COHORT");
Commit();
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
dbChunk.ChunkComplete(chunk.Id);
}
catch (Exception e)
{
Logger.WriteError(chunk.Id, e);
Rollback();
Logger.Write(chunk.Id, LogMessageTypes.Debug, "Rollback - Complete");
}
timer.Stop();
}
public void Process()
{
try
{
var channelFactory = ServicesManager.GetChannelFactory <IVocabularyService>("net.pipe://localhost/VocabularyServiceEndpoint");
vocabularyService = channelFactory.CreateChannel();
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
var timer = new Stopwatch();
timer.Start();
var folder = string.Format("{0}/{1}/raw", Settings.Current.Building.Vendor,
Settings.Current.Building.Id);
Parallel.ForEach(Settings.Current.Building.SourceQueryDefinitions, qd =>
{
if (qd.Providers != null)
{
return;
}
if (qd.Locations != null)
{
return;
}
if (qd.CareSites != null)
{
return;
}
var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database,
Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchemaName);
if (string.IsNullOrEmpty(sql))
{
return;
}
qd.FieldHeaders = new Dictionary <string, int>(StringComparer.OrdinalIgnoreCase);
var metadataKey = string.Format("{0}/metadata/{1}", folder, qd.FileName + ".txt");
using (var client = new AmazonS3Client(Settings.Current.S3AwsAccessKeyId, Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var stream = new MemoryStream())
using (var sr = new StreamReader(stream))
{
var request = new GetObjectRequest {
BucketName = Settings.Current.Bucket, Key = metadataKey
};
using (var response = client.GetObject(request))
{
response.ResponseStream.CopyTo(stream);
}
stream.Position = 0;
var index = 0;
foreach (var fieldName in sr.ReadLine().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
try
{
qd.FieldHeaders.Add(fieldName, index);
index++;
}
catch (Exception)
{
throw new Exception("[RestoreMetadataFromS3] fieldName duplication: " + fieldName + " - " + qd.FileName);
}
}
}
});
Parallel.ForEach(GetParts(), new ParallelOptions {
MaxDegreeOfParallelism = 2
}, p =>
//Parallel.ForEach(GetParts(), p =>
{
Logger.Write(chunkId, LogMessageTypes.Info, "load part=" + p);
var part = new ChunkPart(chunkId, createPersonBuilder, p);
LoadPart(part, p);
part.Build(vocabularyService);
SavePart(part, p);
});
dbChunk.ChunkLoaded(chunkId);
Logger.Write(chunkId, LogMessageTypes.Info, string.Format("Loaded - {0} ms | {1} Mb", timer.ElapsedMilliseconds, (GC.GetTotalMemory(false) / 1024f) / 1024f));
if (Settings.Current.Building.DestinationEngine.Database == Database.Redshift)
{
ServicesManager.AddToSaveQueue(chunkId);
Logger.Write(null, LogMessageTypes.Debug, "AddToSaveQueue " + chunkId);
}
else
{
dbChunk.ChunkComplete(chunkId);
}
channelFactory.Close();
}
catch (Exception e)
{
Logger.WriteError(chunkId, e);
throw;
}
}
private void SaveSet(int setIndex)
{
var dbAvailableOnS3 = new DbAvailableOnS3(Settings.Current.Building.BuilderConnectionString);
Logger.Write(null, LogMessageTypes.Debug, "sss SaveSet BEGIN index - " + setIndex);
var chunkIds = dbAvailableOnS3.GetChunksId(Settings.Current.Building.Id.Value, setIndex).ToArray();
Logger.Write(null, LogMessageTypes.Debug, "sss SaveSet chunkIds: " + string.Join(",", chunkIds));
var timer = new Stopwatch();
timer.Start();
using (
var connection =
SqlConnectionHelper.OpenOdbcConnection(Settings.Current.Building.DestinationConnectionString))
using (var transaction = connection.BeginTransaction())
{
try
{
SaveTable(connection, transaction, setIndex, "PERSON");
SaveTable(connection, transaction, setIndex, "OBSERVATION_PERIOD");
SaveTable(connection, transaction, setIndex, "PAYER_PLAN_PERIOD");
SaveTable(connection, transaction, setIndex, "CONDITION_OCCURRENCE");
SaveTable(connection, transaction, setIndex, "DEATH");
SaveTable(connection, transaction, setIndex, "DRUG_EXPOSURE");
SaveTable(connection, transaction, setIndex, "OBSERVATION");
SaveTable(connection, transaction, setIndex, "VISIT_OCCURRENCE");
SaveTable(connection, transaction, setIndex, "PROCEDURE_OCCURRENCE");
SaveTable(connection, transaction, setIndex, "DRUG_ERA");
SaveTable(connection, transaction, setIndex, "CONDITION_ERA");
SaveTable(connection, transaction, setIndex, "DEVICE_EXPOSURE");
SaveTable(connection, transaction, setIndex, "MEASUREMENT");
SaveTable(connection, transaction, setIndex, "COHORT");
if (Settings.Current.Building.CDM == CDMVersions.v5)
{
SaveTable(connection, transaction, setIndex, "DRUG_COST");
SaveTable(connection, transaction, setIndex, "DEVICE_COST");
SaveTable(connection, transaction, setIndex, "VISIT_COST");
SaveTable(connection, transaction, setIndex, "PROCEDURE_COST");
}
else if (Settings.Current.Building.CDM == CDMVersions.v501)
{
SaveTable(connection, transaction, setIndex, "COST");
}
transaction.Commit();
}
catch (Exception e)
{
foreach (var chunkId in chunkIds)
{
Logger.WriteError(chunkId, e);
transaction.Rollback();
Logger.Write(chunkId, LogMessageTypes.Debug, "Rollback - Complete");
}
}
}
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
foreach (var chunkId in chunkIds)
{
dbChunk.ChunkComplete(chunkId);
}
timer.Stop();
Logger.Write(null, LogMessageTypes.Info, string.Format("Save - {0} ms", timer.ElapsedMilliseconds));
}
public void Process()
{
try
{
var dbChunk = new DbChunk(Settings.Settings.Current.Building.BuilderConnectionString);
var timer = new Stopwatch();
timer.Start();
var folder = $"{Settings.Settings.Current.Building.Vendor}/{Settings.Settings.Current.Building.Id}/raw";
Parallel.ForEach(Settings.Settings.Current.Building.SourceQueryDefinitions, qd =>
{
if (qd.Providers != null)
{
return;
}
if (qd.Locations != null)
{
return;
}
if (qd.CareSites != null)
{
return;
}
//var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database,
// Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchemaName);
var sql = GetSqlHelper.GetSql(Settings.Settings.Current.Building.SourceEngine.Database,
qd.GetSql(Settings.Settings.Current.Building.Vendor, Settings.Settings.Current.Building.SourceSchemaName), Settings.Settings.Current.Building.SourceSchemaName);
if (string.IsNullOrEmpty(sql))
{
return;
}
qd.FieldHeaders = new Dictionary <string, int>(StringComparer.OrdinalIgnoreCase);
var metadataKey = $"{folder}/metadata/{qd.FileName + ".txt"}";
using (var client = new AmazonS3Client(Settings.Settings.Current.S3AwsAccessKeyId, Settings.Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var stream = new MemoryStream())
using (var sr = new StreamReader(stream))
{
var request = new GetObjectRequest {
BucketName = Settings.Settings.Current.Bucket, Key = metadataKey
};
var getObject = client.GetObjectAsync(request);
getObject.Wait();
using (var response = getObject.Result.ResponseStream)
{
response.CopyTo(stream);
}
stream.Position = 0;
var index = 0;
foreach (var fieldName in sr.ReadLine().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
try
{
qd.FieldHeaders.Add(fieldName, index);
index++;
}
catch (Exception)
{
throw new Exception("[RestoreMetadataFromS3] fieldName duplication: " + fieldName + " - " + qd.FileName);
}
}
}
});
Parallel.ForEach(GetParts(), new ParallelOptions {
MaxDegreeOfParallelism = 2
}, p =>
{
Logger.Write(_chunkId, LogMessageTypes.Info, "load part=" + p);
var part = new DatabaseChunkPart(_chunkId, _createPersonBuilder, p, 0);
LoadPart(part, p);
part.Build();
SavePart(part, p);
});
Logger.Write(_chunkId, LogMessageTypes.Info,
$"Loaded - {timer.ElapsedMilliseconds} ms | {(GC.GetTotalMemory(false) / 1024f) / 1024f} Mb");
dbChunk.ChunkComplete(_chunkId, Settings.Settings.Current.Building.Id.Value);
}
catch (Exception e)
{
Logger.WriteError(_chunkId, e);
throw;
}
}