private void SavePart(DatabaseChunkPart part, string p)
{
var saveAttempt = 0;
var saved = false;
while (!saved)
{
try
{
saveAttempt++;
part.Save();
saved = true;
}
catch (Exception ex)
{
if (saveAttempt <= 11)
{
Logger.Write(_chunkId, LogMessageTypes.Warning,
p + ") save attempt=" + saveAttempt + ") " + Logger.CreateExceptionString(ex));
}
else
{
throw;
}
}
}
part.Clean();
}
private void LoadPart(DatabaseChunkPart part, string p)
{
var loadAttempt = 0;
var loaded = false;
while (!loaded)
{
try
{
loadAttempt++;
part.Load();
loaded = true;
}
catch (Exception ex)
{
if (loadAttempt <= 11)
{
Logger.Write(_chunkId, LogMessageTypes.Warning,
p + ") load attempt=" + loadAttempt + ") " + Logger.CreateExceptionString(ex));
part.Reset();
}
else
{
throw;
}
}
}
}
public void Process()
{
try
{
Console.WriteLine("DatabaseChunkBuilder");
var dbChunk = new DbChunk(Settings.Settings.Current.Building.BuilderConnectionString);
var part = new DatabaseChunkPart(_chunkId, _createPersonBuilder, "0", 0);
var timer = new Stopwatch();
timer.Start();
part.Load();
Logger.Write(_chunkId, LogMessageTypes.Info,
$"Loaded - {timer.ElapsedMilliseconds} ms | {(GC.GetTotalMemory(false) / 1024f) / 1024f} Mb");
part.Build();
part.Save();
dbChunk.ChunkComplete(_chunkId, Settings.Settings.Current.Building.Id.Value);
Console.WriteLine($"ChunkId={_chunkId} was complete");
}
catch (Exception e)
{
Logger.WriteError(_chunkId, e);
throw;
}
}
public void Process()
{
try
{
var dbChunk = new DbChunk(Settings.Settings.Current.Building.BuilderConnectionString);
var timer = new Stopwatch();
timer.Start();
var folder = $"{Settings.Settings.Current.Building.Vendor}/{Settings.Settings.Current.Building.Id}/raw";
Parallel.ForEach(Settings.Settings.Current.Building.SourceQueryDefinitions, qd =>
{
if (qd.Providers != null)
{
return;
}
if (qd.Locations != null)
{
return;
}
if (qd.CareSites != null)
{
return;
}
//var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database,
// Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchemaName);
var sql = GetSqlHelper.GetSql(Settings.Settings.Current.Building.SourceEngine.Database,
qd.GetSql(Settings.Settings.Current.Building.Vendor, Settings.Settings.Current.Building.SourceSchemaName), Settings.Settings.Current.Building.SourceSchemaName);
if (string.IsNullOrEmpty(sql))
{
return;
}
qd.FieldHeaders = new Dictionary <string, int>(StringComparer.OrdinalIgnoreCase);
var metadataKey = $"{folder}/metadata/{qd.FileName + ".txt"}";
using (var client = new AmazonS3Client(Settings.Settings.Current.S3AwsAccessKeyId, Settings.Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var stream = new MemoryStream())
using (var sr = new StreamReader(stream))
{
var request = new GetObjectRequest {
BucketName = Settings.Settings.Current.Bucket, Key = metadataKey
};
var getObject = client.GetObjectAsync(request);
getObject.Wait();
using (var response = getObject.Result.ResponseStream)
{
response.CopyTo(stream);
}
stream.Position = 0;
var index = 0;
foreach (var fieldName in sr.ReadLine().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
try
{
qd.FieldHeaders.Add(fieldName, index);
index++;
}
catch (Exception)
{
throw new Exception("[RestoreMetadataFromS3] fieldName duplication: " + fieldName + " - " + qd.FileName);
}
}
}
});
Parallel.ForEach(GetParts(), new ParallelOptions {
MaxDegreeOfParallelism = 2
}, p =>
{
Logger.Write(_chunkId, LogMessageTypes.Info, "load part=" + p);
var part = new DatabaseChunkPart(_chunkId, _createPersonBuilder, p, 0);
LoadPart(part, p);
part.Build();
SavePart(part, p);
});
Logger.Write(_chunkId, LogMessageTypes.Info,
$"Loaded - {timer.ElapsedMilliseconds} ms | {(GC.GetTotalMemory(false) / 1024f) / 1024f} Mb");
dbChunk.ChunkComplete(_chunkId, Settings.Settings.Current.Building.Id.Value);
}
catch (Exception e)
{
Logger.WriteError(_chunkId, e);
throw;
}
}
