// TODO make async CancellationTokenSource IBackupService.Download(BackupEntry entry, Stream stream, Action <int> progress) { if (entry == null) { throw new ArgumentNullException(nameof(entry)); } if (stream == null) { throw new ArgumentNullException(nameof(stream)); } if (progress == null) { throw new ArgumentNullException(nameof(progress)); } var source = new CancellationTokenSource(); var key = GetKey(entry); var request = new GetObjectRequest { BucketName = _bucket, Key = key, }; var bytes = new byte[_bufferSize]; var readTotal = 0L; using (var response = _client.GetObject(request)) using (var responseStream = response.ResponseStream) { response.WriteObjectProgressEvent += (s, a) => progress(a.PercentDone); while (readTotal < response.ContentLength) { var read = responseStream.Read(bytes, 0, bytes.Length); if (read <= 0) { throw new InvalidOperationException(LocalizedStrings.NetworkConnectionError.Put(read)); } stream.Write(bytes, 0, read); readTotal += read; } } return(source); }
private static BackupEntry GetPath(string key) { var entities = key.Split('/').Select(p => new BackupEntry { Name = p }).ToArray(); BackupEntry parent = null; foreach (var entity in entities) { entity.Parent = parent; parent = entity; } return(entities.Last()); }
private static string GetKey(BackupEntry entry) { var key = entry.Name; if (key.IsEmpty()) { throw new ArgumentException("entry"); } if (entry.Parent != null) { key = GetKey(entry.Parent) + "/" + key; } return(key); }
IEnumerable <BackupEntry> IBackupService.Get(BackupEntry parent) { //if (parent != null && !parent.IsDirectory) // throw new ArgumentException("{0} should be directory.".Put(parent.Name), "parent"); var request = new ListObjectsRequest { BucketName = _bucket, Prefix = parent != null?GetKey(parent) : null, }; do { var response = _client.ListObjects(request); foreach (var entry in response.S3Objects) { var be = GetPath(entry.Key); be.Size = entry.Size; yield return(be); } foreach (var commonPrefix in response.CommonPrefixes) { yield return(new BackupEntry { Name = commonPrefix, Parent = parent, }); } if (response.IsTruncated) { request.Marker = response.NextMarker; } else { break; } }while (true); }
IEnumerable<BackupEntry> IBackupService.Get(BackupEntry parent) { //if (parent != null && !parent.IsDirectory) // throw new ArgumentException("{0} should be directory.".Put(parent.Name), "parent"); var request = new ListObjectsRequest { BucketName = _bucket, Prefix = parent != null ? GetKey(parent) : null, }; do { var response = _client.ListObjects(request); foreach (var entry in response.S3Objects) { var be = GetPath(entry.Key); be.Size = entry.Size; yield return be; } foreach (var commonPrefix in response.CommonPrefixes) { yield return new BackupEntry { Name = commonPrefix, Parent = parent, }; } if (response.IsTruncated) request.Marker = response.NextMarker; else break; } while (true); }
void IBackupService.Delete(BackupEntry entry) { throw new NotImplementedException(); }
private static string GetPath(BackupEntry entry) { if (entry == null) return null; return GetPath(entry.Parent) + "/" + entry.Name; }
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action<int> progress) { if (entry == null) throw new ArgumentNullException(nameof(entry)); if (stream == null) throw new ArgumentNullException(nameof(stream)); if (progress == null) throw new ArgumentNullException(nameof(progress)); var key = GetKey(entry); var initResponse = _client.InitiateMultipartUpload(new InitiateMultipartUploadRequest { BucketName = _bucket, Key = key, }); var filePosition = 0L; var nextProgress = 1; var etags = new List<PartETag>(); var partNum = 1; while (filePosition < stream.Length) { var response = _client.UploadPart(new UploadPartRequest { BucketName = _bucket, UploadId = initResponse.UploadId, PartNumber = partNum, PartSize = _bufferSize, //FilePosition = filePosition, InputStream = stream, Key = key }); etags.Add(new PartETag(partNum, response.ETag)); filePosition += _bufferSize; var currProgress = (int)(filePosition.Min(stream.Length) * 100 / stream.Length); if (currProgress >= nextProgress) { progress(currProgress); nextProgress = currProgress + 1; } partNum++; } _client.CompleteMultipartUpload(new CompleteMultipartUploadRequest { BucketName = _bucket, UploadId = initResponse.UploadId, Key = key, PartETags = etags }); var source = new CancellationTokenSource(); return source; }
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action <int> progress) { if (entry == null) { throw new ArgumentNullException(nameof(entry)); } if (stream == null) { throw new ArgumentNullException(nameof(stream)); } if (progress == null) { throw new ArgumentNullException(nameof(progress)); } var key = GetKey(entry); var initResponse = _client.InitiateMultipartUpload(new InitiateMultipartUploadRequest { BucketName = _bucket, Key = key, }); var filePosition = 0L; var nextProgress = 1; var etags = new List <PartETag>(); var partNum = 1; while (filePosition < stream.Length) { var response = _client.UploadPart(new UploadPartRequest { BucketName = _bucket, UploadId = initResponse.UploadId, PartNumber = partNum, PartSize = _bufferSize, //FilePosition = filePosition, InputStream = stream, Key = key }); etags.Add(new PartETag(partNum, response.ETag)); filePosition += _bufferSize; var currProgress = (int)(filePosition.Min(stream.Length) * 100 / stream.Length); if (currProgress >= nextProgress) { progress(currProgress); nextProgress = currProgress + 1; } partNum++; } _client.CompleteMultipartUpload(new CompleteMultipartUploadRequest { BucketName = _bucket, UploadId = initResponse.UploadId, Key = key, PartETags = etags }); var source = new CancellationTokenSource(); return(source); }
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action<int> progress) { throw new NotImplementedException(); }
IEnumerable<BackupEntry> IBackupService.Get(BackupEntry parent) { throw new NotImplementedException(); }
protected override TimeSpan OnProcess() { IBackupService service; switch (_settings.Service) { case BackupServices.AwsS3: service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>()); break; case BackupServices.AwsGlacier: service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>()); break; default: throw new ArgumentOutOfRangeException(); } var hasSecurities = false; this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom)); var startDate = _settings.StartFrom; var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset); var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray(); var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path)); IEnumerable<Tuple<Type, object>> dataTypes = new[] { Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Tick), Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.OrderLog), Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Order), Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Trade), Tuple.Create(typeof(QuoteChangeMessage), (object)null), Tuple.Create(typeof(Level1ChangeMessage), (object)null), Tuple.Create(typeof(NewsMessage), (object)null) }; var workingSecurities = GetWorkingSecurities().ToArray(); foreach (var date in allDates) { foreach (var security in workingSecurities) { hasSecurities = true; if (!CanProcess()) break; var dateEntry = new BackupEntry { Name = date.ToString("yyyy_MM_dd"), Parent = new BackupEntry { Parent = new BackupEntry { Name = security.Security.Id.Substring(0, 1), Parent = pathEntry }, Name = security.Security.Id, } }; var candleTypes = _settings.Drive.GetCandleTypes(security.Security.ToSecurityId(), _settings.StorageFormat); var secDataTypes = dataTypes.Concat(candleTypes.SelectMany(t => t.Item2.Select(a => Tuple.Create(t.Item1, a)))); foreach (var tuple in secDataTypes) { var storage = StorageRegistry.GetStorage(security.Security, tuple.Item1, tuple.Item2, _settings.Drive, _settings.StorageFormat); var drive = storage.Drive; var stream = drive.LoadStream(date); if (stream == Stream.Null) continue; var entry = new BackupEntry { Name = LocalMarketDataDrive.CreateFileName(tuple.Item1, tuple.Item2) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary), Parent = dateEntry, }; service.Upload(entry, stream, p => { }); this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry)); } } if (CanProcess()) { _settings.StartFrom += TimeSpan.FromDays(1); SaveSettings(); } } if (!hasSecurities) { this.AddWarningLog(LocalizedStrings.Str2292); return TimeSpan.MaxValue; } if (CanProcess()) this.AddInfoLog(LocalizedStrings.Str2300); return base.OnProcess(); }
void IBackupService.Delete(BackupEntry entry) { _client.DeleteObject(_bucket, GetKey(entry)); }
IEnumerable <BackupEntry> IBackupService.Get(BackupEntry parent) { throw new NotImplementedException(); }
// TODO make async CancellationTokenSource IBackupService.Download(BackupEntry entry, Stream stream, Action<int> progress) { if (entry == null) throw new ArgumentNullException(nameof(entry)); if (stream == null) throw new ArgumentNullException(nameof(stream)); if (progress == null) throw new ArgumentNullException(nameof(progress)); var source = new CancellationTokenSource(); var key = GetKey(entry); var request = new GetObjectRequest { BucketName = _bucket, Key = key, }; var bytes = new byte[_bufferSize]; var readTotal = 0L; using (var response = _client.GetObject(request)) using (var responseStream = response.ResponseStream) { response.WriteObjectProgressEvent += (s, a) => progress(a.PercentDone); while (readTotal < response.ContentLength) { var read = responseStream.Read(bytes, 0, bytes.Length); if (read <= 0) throw new InvalidOperationException(LocalizedStrings.NetworkConnectionError.Put(read)); stream.Write(bytes, 0, read); readTotal += read; } } return source; }
private static string GetKey(BackupEntry entry) { var key = entry.Name; if (key.IsEmpty()) throw new ArgumentException("entry"); if (entry.Parent != null) key = GetKey(entry.Parent) + "/" + key; return key; }
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action <int> progress) { throw new NotImplementedException(); }
protected override TimeSpan OnProcess() { IBackupService service; switch (_settings.Service) { case BackupServices.AwsS3: service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>()); break; case BackupServices.AwsGlacier: service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>()); break; default: throw new ArgumentOutOfRangeException(); } var hasSecurities = false; this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom)); var startDate = _settings.StartFrom; var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset); var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray(); var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path)); var workingSecurities = GetWorkingSecurities().ToArray(); foreach (var date in allDates) { foreach (var security in workingSecurities) { hasSecurities = true; if (!CanProcess()) break; var dateEntry = new BackupEntry { Name = date.ToString("yyyy_MM_dd"), Parent = new BackupEntry { Parent = new BackupEntry { Name = security.Security.Id.Substring(0, 1), Parent = pathEntry }, Name = security.Security.Id, } }; var dataTypes = _settings.Drive.GetAvailableDataTypes(security.Security.ToSecurityId(), _settings.StorageFormat); foreach (var dataType in dataTypes) { var storage = StorageRegistry.GetStorage(security.Security, dataType.MessageType, dataType.Arg, _settings.Drive, _settings.StorageFormat); var drive = storage.Drive; var stream = drive.LoadStream(date); if (stream == Stream.Null) continue; var entry = new BackupEntry { Name = LocalMarketDataDrive.GetFileName(dataType.MessageType, dataType.Arg) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary), Parent = dateEntry, }; service.Upload(entry, stream, p => { }); this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry)); } } if (CanProcess()) { _settings.StartFrom += TimeSpan.FromDays(1); SaveSettings(); } } if (!hasSecurities) { this.AddWarningLog(LocalizedStrings.Str2292); return TimeSpan.MaxValue; } if (CanProcess()) this.AddInfoLog(LocalizedStrings.Str2300); return base.OnProcess(); }