Ejemplo n.º 1
0
        public void Process()
        {
            try
            {
                Console.WriteLine("DatabaseChunkBuilder");
                var dbChunk = new DbChunk(Settings.Settings.Current.Building.BuilderConnectionString);
                var part    = new DatabaseChunkPart(_chunkId, _createPersonBuilder, "0", 0);

                var timer = new Stopwatch();
                timer.Start();

                part.Load();
                Logger.Write(_chunkId, LogMessageTypes.Info,
                             $"Loaded - {timer.ElapsedMilliseconds} ms | {(GC.GetTotalMemory(false) / 1024f) / 1024f} Mb");

                part.Build();
                part.Save();

                dbChunk.ChunkComplete(_chunkId, Settings.Settings.Current.Building.Id.Value);
                Console.WriteLine($"ChunkId={_chunkId} was complete");
            }
            catch (Exception e)
            {
                Logger.WriteError(_chunkId, e);

                throw;
            }
        }
        public void Process()
        {
            try
            {
                var dbChunk = new DbChunk(Settings.Settings.Current.Building.BuilderConnectionString);
                var timer   = new Stopwatch();
                timer.Start();

                var folder = $"{Settings.Settings.Current.Building.Vendor}/{Settings.Settings.Current.Building.Id}/raw";

                Parallel.ForEach(Settings.Settings.Current.Building.SourceQueryDefinitions, qd =>
                {
                    if (qd.Providers != null)
                    {
                        return;
                    }
                    if (qd.Locations != null)
                    {
                        return;
                    }
                    if (qd.CareSites != null)
                    {
                        return;
                    }

                    //var sql = qd.GetSql(Settings.Current.Building.SourceEngine.Database,
                    //   Settings.Current.Building.Vendor, Settings.Current.Building.SourceSchemaName);

                    var sql = GetSqlHelper.GetSql(Settings.Settings.Current.Building.SourceEngine.Database,
                                                  qd.GetSql(Settings.Settings.Current.Building.Vendor, Settings.Settings.Current.Building.SourceSchemaName), Settings.Settings.Current.Building.SourceSchemaName);


                    if (string.IsNullOrEmpty(sql))
                    {
                        return;
                    }

                    qd.FieldHeaders = new Dictionary <string, int>(StringComparer.OrdinalIgnoreCase);

                    var metadataKey = $"{folder}/metadata/{qd.FileName + ".txt"}";

                    using (var client = new AmazonS3Client(Settings.Settings.Current.S3AwsAccessKeyId, Settings.Settings.Current.S3AwsSecretAccessKey, Amazon.RegionEndpoint.USEast1))
                        using (var stream = new MemoryStream())
                            using (var sr = new StreamReader(stream))
                            {
                                var request = new GetObjectRequest {
                                    BucketName = Settings.Settings.Current.Bucket, Key = metadataKey
                                };
                                var getObject = client.GetObjectAsync(request);
                                getObject.Wait();

                                using (var response = getObject.Result.ResponseStream)
                                {
                                    response.CopyTo(stream);
                                }
                                stream.Position = 0;

                                var index = 0;
                                foreach (var fieldName in sr.ReadLine().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
                                {
                                    try
                                    {
                                        qd.FieldHeaders.Add(fieldName, index);
                                        index++;
                                    }
                                    catch (Exception)
                                    {
                                        throw new Exception("[RestoreMetadataFromS3] fieldName duplication: " + fieldName + " - " + qd.FileName);
                                    }
                                }
                            }
                });

                Parallel.ForEach(GetParts(), new ParallelOptions {
                    MaxDegreeOfParallelism = 2
                }, p =>
                {
                    Logger.Write(_chunkId, LogMessageTypes.Info, "load part=" + p);
                    var part = new DatabaseChunkPart(_chunkId, _createPersonBuilder, p, 0);

                    LoadPart(part, p);

                    part.Build();

                    SavePart(part, p);
                });


                Logger.Write(_chunkId, LogMessageTypes.Info,
                             $"Loaded - {timer.ElapsedMilliseconds} ms | {(GC.GetTotalMemory(false) / 1024f) / 1024f} Mb");

                dbChunk.ChunkComplete(_chunkId, Settings.Settings.Current.Building.Id.Value);
            }
            catch (Exception e)
            {
                Logger.WriteError(_chunkId, e);

                throw;
            }
        }