private static void ReadMetadata(QueryDefinition qd, string folder) { if (qd.Providers != null) { return; } if (qd.Locations != null) { return; } if (qd.CareSites != null) { return; } var sql = qd.GetSql(Settings.Current.Building.Vendor); if (string.IsNullOrEmpty(sql)) { return; } qd.FieldHeaders = new Dictionary <string, int>(StringComparer.OrdinalIgnoreCase); var metadataKey = string.Format("{0}/metadata/{1}", folder, qd.FileName + ".txt"); using (var client = new AmazonS3Client(Settings.Current.S3AwsAccessKeyId, Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1)) using (var stream = new MemoryStream()) using (var sr = new StreamReader(stream)) { var request = new GetObjectRequest { BucketName = Settings.Current.Bucket, Key = metadataKey }; Task <GetObjectResponse> getObject = client.GetObjectAsync(request); getObject.Wait(); using (var response = getObject.Result) { response.ResponseStream.CopyTo(stream); } stream.Position = 0; var index = 0; foreach (var fieldName in sr.ReadLine().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) { try { qd.FieldHeaders.Add(fieldName, index); index++; } catch (Exception e) { Console.WriteLine("WARN_EXC - ReadMetadata - throw"); Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); Console.WriteLine("[RestoreMetadataFromS3] fieldName duplication: " + fieldName + " - " + qd.FileName); throw; } } } }
private void FillList <T>(ICollection <T> list, QueryDefinition qd, EntityDefinition ed) where T : IEntity { var sql = GetSqlHelper.GetSql(Settings.Current.Building.SourceEngine.Database, qd.GetSql(Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchema), Settings.Current.Building.SourceSchema); if (string.IsNullOrEmpty(sql)) { return; } var keys = new Dictionary <string, bool>(); using (var connection = new OdbcConnection(Settings.Current.Building.SourceConnectionString)) { connection.Open(); using (var c = new OdbcCommand(sql, connection)) { c.CommandTimeout = 30000; using (var reader = c.ExecuteReader()) { while (reader.Read()) { Concept conceptDef = null; if (ed.Concepts != null && ed.Concepts.Any()) { conceptDef = ed.Concepts[0]; } var concept = (T)ed.GetConcepts(conceptDef, reader, null).ToList()[0]; var key = concept.GetKey(); if (key == null) { continue; } if (keys.ContainsKey(key)) { continue; } keys.Add(key, false); list.Add(concept); if (CurrentState != BuilderState.Running) { break; } } } } } }
public void Load(QueryDefinition qd, EntityDefinition ed) { var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database, Settings.Current.Building.Vendor, schemaName); if (string.IsNullOrEmpty(sql)) { return; } using (var conn = SqlConnectionHelper.OpenOdbcConnection(connectionString)) { using (var c = new OdbcCommand(sql, conn)) { c.CommandTimeout = 30000; using (var reader = c.ExecuteReader()) { while (reader.Read()) { Concept conceptDef = null; if (ed.Concepts != null && ed.Concepts.Any()) { conceptDef = ed.Concepts[0]; } var concept = (T)ed.GetConcepts(conceptDef, reader, null).ToList()[0]; var key = concept.GetKey(); if (key == null) { continue; } if (!lookup.ContainsKey(key)) { lookup.Add(key, concept); } } } } } }