private static void ReadMetadata(QueryDefinition qd, string folder)
{
if (qd.Providers != null)
{
return;
}
if (qd.Locations != null)
{
return;
}
if (qd.CareSites != null)
{
return;
}
var sql = qd.GetSql(Settings.Current.Building.Vendor);
if (string.IsNullOrEmpty(sql))
{
return;
}
qd.FieldHeaders = new Dictionary <string, int>(StringComparer.OrdinalIgnoreCase);
var metadataKey = string.Format("{0}/metadata/{1}", folder, qd.FileName + ".txt");
using (var client = new AmazonS3Client(Settings.Current.S3AwsAccessKeyId, Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
using (var stream = new MemoryStream())
using (var sr = new StreamReader(stream))
{
var request = new GetObjectRequest {
BucketName = Settings.Current.Bucket, Key = metadataKey
};
Task <GetObjectResponse> getObject = client.GetObjectAsync(request);
getObject.Wait();
using (var response = getObject.Result)
{
response.ResponseStream.CopyTo(stream);
}
stream.Position = 0;
var index = 0;
foreach (var fieldName in sr.ReadLine().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
try
{
qd.FieldHeaders.Add(fieldName, index);
index++;
}
catch (Exception e)
{
Console.WriteLine("WARN_EXC - ReadMetadata - throw");
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
Console.WriteLine("[RestoreMetadataFromS3] fieldName duplication: " + fieldName + " - " + qd.FileName);
throw;
}
}
}
}
private void FillList <T>(ICollection <T> list, QueryDefinition qd, EntityDefinition ed) where T : IEntity
{
var sql = GetSqlHelper.GetSql(Settings.Current.Building.SourceEngine.Database,
qd.GetSql(Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchema), Settings.Current.Building.SourceSchema);
if (string.IsNullOrEmpty(sql))
{
return;
}
var keys = new Dictionary <string, bool>();
using (var connection = new OdbcConnection(Settings.Current.Building.SourceConnectionString))
{
connection.Open();
using (var c = new OdbcCommand(sql, connection))
{
c.CommandTimeout = 30000;
using (var reader = c.ExecuteReader())
{
while (reader.Read())
{
Concept conceptDef = null;
if (ed.Concepts != null && ed.Concepts.Any())
{
conceptDef = ed.Concepts[0];
}
var concept = (T)ed.GetConcepts(conceptDef, reader, null).ToList()[0];
var key = concept.GetKey();
if (key == null)
{
continue;
}
if (keys.ContainsKey(key))
{
continue;
}
keys.Add(key, false);
list.Add(concept);
if (CurrentState != BuilderState.Running)
{
break;
}
}
}
}
}
}
public void Load(QueryDefinition qd, EntityDefinition ed)
{
var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database, Settings.Current.Building.Vendor, schemaName);
if (string.IsNullOrEmpty(sql))
{
return;
}
using (var conn = SqlConnectionHelper.OpenOdbcConnection(connectionString))
{
using (var c = new OdbcCommand(sql, conn))
{
c.CommandTimeout = 30000;
using (var reader = c.ExecuteReader())
{
while (reader.Read())
{
Concept conceptDef = null;
if (ed.Concepts != null && ed.Concepts.Any())
{
conceptDef = ed.Concepts[0];
}
var concept = (T)ed.GetConcepts(conceptDef, reader, null).ToList()[0];
var key = concept.GetKey();
if (key == null)
{
continue;
}
if (!lookup.ContainsKey(key))
{
lookup.Add(key, concept);
}
}
}
}
}
}
