public void Add(LookupValue lv) { if (!_lookup.ContainsKey(lv.SourceCode)) { _lookup.Add(lv.SourceCode, new Dictionary <int, LookupValue>()); } //if (!_lookupIgnoreCase.ContainsKey(lv.SourceCode)) // _lookupIgnoreCase.Add(lv.SourceCode, new HashSet<string>()); //_lookupIgnoreCase[lv.SourceCode].Add(lv.SourceCode); if (!_lookup[lv.SourceCode].ContainsKey(lv.ConceptId.Value)) { _lookup[lv.SourceCode].Add(lv.ConceptId.Value, lv); } if (lv.Ingredients != null && lv.Ingredients.Count > 0) { _lookup[lv.SourceCode][lv.ConceptId.Value].Ingredients.Add(lv.Ingredients.First()); } }
public void Fill(AmazonS3Client client, string bucket, string prefix) { //var prefix = $"Lookup/Truven/{fileName}.txt"; var spliter = new StringSplitter(5); var result = new List <string>(); var request = new ListObjectsV2Request { BucketName = bucket, Prefix = prefix }; ListObjectsV2Response response; do { var responseTask = client.ListObjectsV2Async(request); responseTask.Wait(); response = responseTask.Result; foreach (var entry in response.S3Objects) { if (entry.Size > 20) { result.Add(entry.Key); } } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); var getObjectRequest = new GetObjectRequest { BucketName = bucket, Key = result[0] }; var getObject = client.GetObjectAsync(getObjectRequest); getObject.Wait(); using (var responseStream = getObject.Result.ResponseStream) using (var bufferedStream = new BufferedStream(responseStream)) using (var gzipStream = new GZipStream(bufferedStream, CompressionMode.Decompress)) using (var reader = new StreamReader(gzipStream, Encoding.Default)) { string line; while ((line = reader.ReadLine()) != null) { if (!string.IsNullOrEmpty(line)) { spliter.SafeSplit(line, '\t'); var sourceCode = GetStringValue(spliter.Results[0]); if (!_lookup.ContainsKey(sourceCode)) { _lookup.Add(sourceCode, new Dictionary <int, LookupValue>()); } //if (!_lookupIgnoreCase.ContainsKey(sourceCode)) // _lookupIgnoreCase.Add(sourceCode, new HashSet<string>()); //_lookupIgnoreCase[sourceCode].Add(sourceCode); int conceptId = -1; if (int.TryParse(spliter.Results[1], out var cptId)) { conceptId = cptId; } if (!_lookup[sourceCode].ContainsKey(conceptId)) { if (!DateTime.TryParse(spliter.Results[3], out var validStartDate)) { validStartDate = DateTime.MinValue; } if (!DateTime.TryParse(spliter.Results[4], out var validEndDate)) { validEndDate = DateTime.MaxValue; } var lv = new LookupValue { ConceptId = conceptId, SourceCode = sourceCode, Domain = GetStringValue(spliter.Results[2]), ValidStartDate = validStartDate, ValidEndDate = validEndDate, //Ingredients = new HashSet<int>() }; if (spliter.Results.Length > 5) { //lv.SourceVocabularyId = GetStringValue(spliter.Results[5]); lv.SourceConceptId = IsNullOrEmpty(spliter.Results[6]) ? 0 : int.Parse(spliter.Results[6]); //if (!DateTime.TryParse(spliter.Results[7], out var validStartDate2)) // validStartDate2 = DateTime.MinValue; //if (!DateTime.TryParse(spliter.Results[8], out var validEndDate2)) // validEndDate2 = DateTime.MaxValue; //lv.SourceValidStartDate = validStartDate2; //lv.SourceValidEndDate = validEndDate2; } _lookup[sourceCode].Add(conceptId, lv); } if (spliter.Results.Length > 5) { if (!IsNullOrEmpty(spliter.Results[9]) && int.TryParse(spliter.Results[9], out var ingredient)) { if (_lookup[sourceCode][conceptId].Ingredients == null) { _lookup[sourceCode][conceptId].Ingredients = new HashSet <int>(); } _lookup[sourceCode][conceptId].Ingredients.Add(ingredient); } } } } foreach (var v1 in _lookup.Values) { foreach (var v2 in v1.Values) { v2.Ingredients?.TrimExcess(); } } GC.Collect(); } }