private static int[] GetChunkIds(int setIndex)
{
var attempt = 0;
while (true)
{
try
{
attempt++;
var dbAvailableOnS3 = new DbAvailableOnS3(Settings.Current.Building.BuilderConnectionString);
Logger.Write(null, LogMessageTypes.Debug, "sss SaveSet BEGIN index - " + setIndex);
var chunkIds = dbAvailableOnS3.GetChunksId(Settings.Current.Building.Id.Value, setIndex).ToArray();
Logger.Write(null, LogMessageTypes.Debug, "sss SaveSet chunkIds: " + string.Join(",", chunkIds));
return(chunkIds);
}
catch (Exception)
{
if (attempt <= 10)
{
Thread.Sleep(1000);
}
else
{
throw;
}
}
}
}
public void AddToSaveQueue(int chunkId)
{
Logger.Write(null, LogMessageTypes.Debug, "sss AddToSaveQueue " + chunkId);
var setIndex = chunksIndexes[chunkId];
var dbAvailableOnS3 = new DbAvailableOnS3(Settings.Current.Building.BuilderConnectionString);
dbAvailableOnS3.AddPreparedChunk(Settings.Current.Building.Id.Value, setIndex, chunkId);
var count = dbAvailableOnS3.GetChunksCount(Settings.Current.Building.Id.Value, setIndex);
Logger.Write(null, LogMessageTypes.Debug, string.Format("sss AddToSaveQueue index={0}; chunkId={1}; set[setIndex].Count={2}; indexSize={3}", setIndex, chunkId, count, indexSize[setIndex]));
}
private void saver_DoWork(object sender, DoWorkEventArgs e)
{
try
{
var worker = (BackgroundWorker)sender;
var dbAvailableOnS3 = new DbAvailableOnS3(Settings.Current.Building.BuilderConnectionString);
while (true)
{
if (worker.CancellationPending)
{
Logger.Write(null, LogMessageTypes.Debug, "sss CancellationPending = TRUE");
e.Cancel = true;
return;
}
var setIndex = dbAvailableOnS3.GetCompletedSetIndex(Settings.Current.Building.Id.Value, SizeOfSet);
if (!setIndex.HasValue)
{
Thread.Sleep(1000);
continue;
}
if (!Settings.Current.SkipSaveToRedshift)
{
Logger.Write(null, LogMessageTypes.Debug, "sss StartSaveChunkTask index - " + setIndex);
SaveSet(setIndex.Value);
dbAvailableOnS3.SetSaved(Settings.Current.Building.Id.Value, setIndex.Value);
Logger.Write(null, LogMessageTypes.Debug, "sss EndSaveChunkTask index - " + setIndex);
}
}
}
catch (Exception ex)
{
Logger.Write(null, LogMessageTypes.Debug, "sss StartSaveChunkTask ERROR");
Logger.WriteError(ex);
}
Logger.Write(null, LogMessageTypes.Debug, "sss StartSaveChunkTask index END");
}
private void SaveSet(int setIndex)
{
var dbAvailableOnS3 = new DbAvailableOnS3(Settings.Current.Building.BuilderConnectionString);
Logger.Write(null, LogMessageTypes.Debug, "sss SaveSet BEGIN index - " + setIndex);
var chunkIds = dbAvailableOnS3.GetChunksId(Settings.Current.Building.Id.Value, setIndex).ToArray();
Logger.Write(null, LogMessageTypes.Debug, "sss SaveSet chunkIds: " + string.Join(",", chunkIds));
var timer = new Stopwatch();
timer.Start();
using (
var connection =
SqlConnectionHelper.OpenOdbcConnection(Settings.Current.Building.DestinationConnectionString))
using (var transaction = connection.BeginTransaction())
{
try
{
SaveTable(connection, transaction, setIndex, "PERSON");
SaveTable(connection, transaction, setIndex, "OBSERVATION_PERIOD");
SaveTable(connection, transaction, setIndex, "PAYER_PLAN_PERIOD");
SaveTable(connection, transaction, setIndex, "CONDITION_OCCURRENCE");
SaveTable(connection, transaction, setIndex, "DEATH");
SaveTable(connection, transaction, setIndex, "DRUG_EXPOSURE");
SaveTable(connection, transaction, setIndex, "OBSERVATION");
SaveTable(connection, transaction, setIndex, "VISIT_OCCURRENCE");
SaveTable(connection, transaction, setIndex, "PROCEDURE_OCCURRENCE");
SaveTable(connection, transaction, setIndex, "DRUG_ERA");
SaveTable(connection, transaction, setIndex, "CONDITION_ERA");
SaveTable(connection, transaction, setIndex, "DEVICE_EXPOSURE");
SaveTable(connection, transaction, setIndex, "MEASUREMENT");
SaveTable(connection, transaction, setIndex, "COHORT");
if (Settings.Current.Building.CDM == CDMVersions.v5)
{
SaveTable(connection, transaction, setIndex, "DRUG_COST");
SaveTable(connection, transaction, setIndex, "DEVICE_COST");
SaveTable(connection, transaction, setIndex, "VISIT_COST");
SaveTable(connection, transaction, setIndex, "PROCEDURE_COST");
}
else if (Settings.Current.Building.CDM == CDMVersions.v501)
{
SaveTable(connection, transaction, setIndex, "COST");
}
transaction.Commit();
}
catch (Exception e)
{
foreach (var chunkId in chunkIds)
{
Logger.WriteError(chunkId, e);
transaction.Rollback();
Logger.Write(chunkId, LogMessageTypes.Debug, "Rollback - Complete");
}
}
}
var dbChunk = new DbChunk(Settings.Current.Building.BuilderConnectionString);
foreach (var chunkId in chunkIds)
{
dbChunk.ChunkComplete(chunkId);
}
timer.Stop();
Logger.Write(null, LogMessageTypes.Info, string.Format("Save - {0} ms", timer.ElapsedMilliseconds));
}
