Beispiel #1
0
        // TODO make async

        CancellationTokenSource IBackupService.Download(BackupEntry entry, Stream stream, Action <int> progress)
        {
            if (entry == null)
            {
                throw new ArgumentNullException(nameof(entry));
            }

            if (stream == null)
            {
                throw new ArgumentNullException(nameof(stream));
            }

            if (progress == null)
            {
                throw new ArgumentNullException(nameof(progress));
            }

            var source = new CancellationTokenSource();

            var key = GetKey(entry);

            var request = new GetObjectRequest
            {
                BucketName = _bucket,
                Key        = key,
            };

            var bytes     = new byte[_bufferSize];
            var readTotal = 0L;

            using (var response = _client.GetObject(request))
                using (var responseStream = response.ResponseStream)
                {
                    response.WriteObjectProgressEvent += (s, a) => progress(a.PercentDone);

                    while (readTotal < response.ContentLength)
                    {
                        var read = responseStream.Read(bytes, 0, bytes.Length);

                        if (read <= 0)
                        {
                            throw new InvalidOperationException(LocalizedStrings.NetworkConnectionError.Put(read));
                        }

                        stream.Write(bytes, 0, read);

                        readTotal += read;
                    }
                }

            return(source);
        }
Beispiel #2
0
        private static BackupEntry GetPath(string key)
        {
            var entities = key.Split('/').Select(p => new BackupEntry {
                Name = p
            }).ToArray();

            BackupEntry parent = null;

            foreach (var entity in entities)
            {
                entity.Parent = parent;
                parent        = entity;
            }

            return(entities.Last());
        }
Beispiel #3
0
        private static string GetKey(BackupEntry entry)
        {
            var key = entry.Name;

            if (key.IsEmpty())
            {
                throw new ArgumentException("entry");
            }

            if (entry.Parent != null)
            {
                key = GetKey(entry.Parent) + "/" + key;
            }

            return(key);
        }
Beispiel #4
0
        IEnumerable <BackupEntry> IBackupService.Get(BackupEntry parent)
        {
            //if (parent != null && !parent.IsDirectory)
            //	throw new ArgumentException("{0} should be directory.".Put(parent.Name), "parent");

            var request = new ListObjectsRequest
            {
                BucketName = _bucket,
                Prefix     = parent != null?GetKey(parent) : null,
            };

            do
            {
                var response = _client.ListObjects(request);

                foreach (var entry in response.S3Objects)
                {
                    var be = GetPath(entry.Key);
                    be.Size = entry.Size;
                    yield return(be);
                }

                foreach (var commonPrefix in response.CommonPrefixes)
                {
                    yield return(new BackupEntry
                    {
                        Name = commonPrefix,
                        Parent = parent,
                    });
                }

                if (response.IsTruncated)
                {
                    request.Marker = response.NextMarker;
                }
                else
                {
                    break;
                }
            }while (true);
        }
		IEnumerable<BackupEntry> IBackupService.Get(BackupEntry parent)
		{
			//if (parent != null && !parent.IsDirectory)
			//	throw new ArgumentException("{0} should be directory.".Put(parent.Name), "parent");

			var request = new ListObjectsRequest
			{
				BucketName = _bucket,
				Prefix = parent != null ? GetKey(parent) : null,
			};

			do
			{
				var response = _client.ListObjects(request);

				foreach (var entry in response.S3Objects)
				{
					var be = GetPath(entry.Key);
					be.Size = entry.Size;
					yield return be;
				}

				foreach (var commonPrefix in response.CommonPrefixes)
				{
					yield return new BackupEntry
					{
						Name = commonPrefix,
						Parent = parent,
					};
				}

				if (response.IsTruncated)
					request.Marker = response.NextMarker;
				else
					break;
			}
			while (true);
		}
 void IBackupService.Delete(BackupEntry entry)
 {
     throw new NotImplementedException();
 }
Beispiel #7
0
		private static string GetPath(BackupEntry entry)
		{
			if (entry == null)
				return null;

			return GetPath(entry.Parent) + "/" + entry.Name;
		}
		CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action<int> progress)
		{
			if (entry == null)
				throw new ArgumentNullException(nameof(entry));

			if (stream == null)
				throw new ArgumentNullException(nameof(stream));

			if (progress == null)
				throw new ArgumentNullException(nameof(progress));

			var key = GetKey(entry);

			var initResponse = _client.InitiateMultipartUpload(new InitiateMultipartUploadRequest
			{
				BucketName = _bucket,
				Key = key,
			});

			var filePosition = 0L;
			var nextProgress = 1;

			var etags = new List<PartETag>();

			var partNum = 1;

			while (filePosition < stream.Length)
			{
				var response = _client.UploadPart(new UploadPartRequest
				{
					BucketName = _bucket,
					UploadId = initResponse.UploadId,
					PartNumber = partNum,
					PartSize = _bufferSize,
					//FilePosition = filePosition,
					InputStream = stream,
					Key = key
				});

				etags.Add(new PartETag(partNum, response.ETag));

				filePosition += _bufferSize;

				var currProgress = (int)(filePosition.Min(stream.Length) * 100 / stream.Length);

				if (currProgress >= nextProgress)
				{
					progress(currProgress);
					nextProgress = currProgress + 1;
				}

				partNum++;
			}

			_client.CompleteMultipartUpload(new CompleteMultipartUploadRequest
			{
				BucketName = _bucket,
				UploadId = initResponse.UploadId,
				Key = key,
				PartETags = etags
			});

			var source = new CancellationTokenSource();

			return source;
		}
Beispiel #9
0
        CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action <int> progress)
        {
            if (entry == null)
            {
                throw new ArgumentNullException(nameof(entry));
            }

            if (stream == null)
            {
                throw new ArgumentNullException(nameof(stream));
            }

            if (progress == null)
            {
                throw new ArgumentNullException(nameof(progress));
            }

            var key = GetKey(entry);

            var initResponse = _client.InitiateMultipartUpload(new InitiateMultipartUploadRequest
            {
                BucketName = _bucket,
                Key        = key,
            });

            var filePosition = 0L;
            var nextProgress = 1;

            var etags = new List <PartETag>();

            var partNum = 1;

            while (filePosition < stream.Length)
            {
                var response = _client.UploadPart(new UploadPartRequest
                {
                    BucketName = _bucket,
                    UploadId   = initResponse.UploadId,
                    PartNumber = partNum,
                    PartSize   = _bufferSize,
                    //FilePosition = filePosition,
                    InputStream = stream,
                    Key         = key
                });

                etags.Add(new PartETag(partNum, response.ETag));

                filePosition += _bufferSize;

                var currProgress = (int)(filePosition.Min(stream.Length) * 100 / stream.Length);

                if (currProgress >= nextProgress)
                {
                    progress(currProgress);
                    nextProgress = currProgress + 1;
                }

                partNum++;
            }

            _client.CompleteMultipartUpload(new CompleteMultipartUploadRequest
            {
                BucketName = _bucket,
                UploadId   = initResponse.UploadId,
                Key        = key,
                PartETags  = etags
            });

            var source = new CancellationTokenSource();

            return(source);
        }
		CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action<int> progress)
		{
			throw new NotImplementedException();
		}
		void IBackupService.Delete(BackupEntry entry)
		{
			throw new NotImplementedException();
		}
		IEnumerable<BackupEntry> IBackupService.Get(BackupEntry parent)
		{
			throw new NotImplementedException();
		}
Beispiel #13
0
		protected override TimeSpan OnProcess()
		{
			IBackupService service;

			switch (_settings.Service)
			{
				case BackupServices.AwsS3:
					service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
					break;
				case BackupServices.AwsGlacier:
					service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
					break;
				default:
					throw new ArgumentOutOfRangeException();
			}

			var hasSecurities = false;

			this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));

			var startDate = _settings.StartFrom;
			var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);

			var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray();

			var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path));

			IEnumerable<Tuple<Type, object>> dataTypes = new[]
			{
				Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Tick),
				Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.OrderLog),
				Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Order),
				Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Trade),
				Tuple.Create(typeof(QuoteChangeMessage), (object)null),
				Tuple.Create(typeof(Level1ChangeMessage), (object)null),
				Tuple.Create(typeof(NewsMessage), (object)null)
			};

			var workingSecurities = GetWorkingSecurities().ToArray();

			foreach (var date in allDates)
			{
				foreach (var security in workingSecurities)
				{
					hasSecurities = true;

					if (!CanProcess())
						break;

					var dateEntry = new BackupEntry
					{
						Name = date.ToString("yyyy_MM_dd"),
						Parent = new BackupEntry
						{
							Parent = new BackupEntry
							{
								Name = security.Security.Id.Substring(0, 1),
								Parent = pathEntry
							},
							Name = security.Security.Id,
						}
					};

					var candleTypes = _settings.Drive.GetCandleTypes(security.Security.ToSecurityId(), _settings.StorageFormat);

					var secDataTypes = dataTypes.Concat(candleTypes.SelectMany(t => t.Item2.Select(a => Tuple.Create(t.Item1, a))));

					foreach (var tuple in secDataTypes)
					{
						var storage = StorageRegistry.GetStorage(security.Security, tuple.Item1, tuple.Item2, _settings.Drive, _settings.StorageFormat);

						var drive = storage.Drive;

						var stream = drive.LoadStream(date);

						if (stream == Stream.Null)
							continue;

						var entry = new BackupEntry
						{
							Name = LocalMarketDataDrive.CreateFileName(tuple.Item1, tuple.Item2) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary),
							Parent = dateEntry,
						};

						service.Upload(entry, stream, p => { });

						this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry));
					}
				}

				if (CanProcess())
				{
					_settings.StartFrom += TimeSpan.FromDays(1);
					SaveSettings();
				}
			}

			if (!hasSecurities)
			{
				this.AddWarningLog(LocalizedStrings.Str2292);
				return TimeSpan.MaxValue;
			}

			if (CanProcess())
				this.AddInfoLog(LocalizedStrings.Str2300);

			return base.OnProcess();
		}
Beispiel #14
0
 void IBackupService.Delete(BackupEntry entry)
 {
     _client.DeleteObject(_bucket, GetKey(entry));
 }
 IEnumerable <BackupEntry> IBackupService.Get(BackupEntry parent)
 {
     throw new NotImplementedException();
 }
Beispiel #16
0
		// TODO make async

		CancellationTokenSource IBackupService.Download(BackupEntry entry, Stream stream, Action<int> progress)
		{
			if (entry == null)
				throw new ArgumentNullException(nameof(entry));

			if (stream == null)
				throw new ArgumentNullException(nameof(stream));

			if (progress == null)
				throw new ArgumentNullException(nameof(progress));

			var source = new CancellationTokenSource();

			var key = GetKey(entry);

			var request = new GetObjectRequest
			{
				BucketName = _bucket,
				Key = key,
			};

			var bytes = new byte[_bufferSize];
			var readTotal = 0L;

			using (var response = _client.GetObject(request))
			using (var responseStream = response.ResponseStream)
			{
				response.WriteObjectProgressEvent += (s, a) => progress(a.PercentDone);

				while (readTotal < response.ContentLength)
				{
					var read = responseStream.Read(bytes, 0, bytes.Length);

					if (read <= 0)
						throw new InvalidOperationException(LocalizedStrings.NetworkConnectionError.Put(read));

					stream.Write(bytes, 0, read);

					readTotal += read;
				}
			}

			return source;
		}
Beispiel #17
0
		void IBackupService.Delete(BackupEntry entry)
		{
			_client.DeleteObject(_bucket, GetKey(entry));
		}
Beispiel #18
0
		private static string GetKey(BackupEntry entry)
		{
			var key = entry.Name;

			if (key.IsEmpty())
				throw new ArgumentException("entry");

			if (entry.Parent != null)
				key = GetKey(entry.Parent) + "/" + key;

			return key;
		}
 CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action <int> progress)
 {
     throw new NotImplementedException();
 }
Beispiel #20
0
		protected override TimeSpan OnProcess()
		{
			IBackupService service;

			switch (_settings.Service)
			{
				case BackupServices.AwsS3:
					service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
					break;
				case BackupServices.AwsGlacier:
					service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
					break;
				default:
					throw new ArgumentOutOfRangeException();
			}

			var hasSecurities = false;

			this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));

			var startDate = _settings.StartFrom;
			var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);

			var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray();

			var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path));

			var workingSecurities = GetWorkingSecurities().ToArray();

			foreach (var date in allDates)
			{
				foreach (var security in workingSecurities)
				{
					hasSecurities = true;

					if (!CanProcess())
						break;

					var dateEntry = new BackupEntry
					{
						Name = date.ToString("yyyy_MM_dd"),
						Parent = new BackupEntry
						{
							Parent = new BackupEntry
							{
								Name = security.Security.Id.Substring(0, 1),
								Parent = pathEntry
							},
							Name = security.Security.Id,
						}
					};

					var dataTypes = _settings.Drive.GetAvailableDataTypes(security.Security.ToSecurityId(), _settings.StorageFormat);

					foreach (var dataType in dataTypes)
					{
						var storage = StorageRegistry.GetStorage(security.Security, dataType.MessageType, dataType.Arg, _settings.Drive, _settings.StorageFormat);

						var drive = storage.Drive;

						var stream = drive.LoadStream(date);

						if (stream == Stream.Null)
							continue;

						var entry = new BackupEntry
						{
							Name = LocalMarketDataDrive.GetFileName(dataType.MessageType, dataType.Arg) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary),
							Parent = dateEntry,
						};

						service.Upload(entry, stream, p => { });

						this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry));
					}
				}

				if (CanProcess())
				{
					_settings.StartFrom += TimeSpan.FromDays(1);
					SaveSettings();
				}
			}

			if (!hasSecurities)
			{
				this.AddWarningLog(LocalizedStrings.Str2292);
				return TimeSpan.MaxValue;
			}

			if (CanProcess())
				this.AddInfoLog(LocalizedStrings.Str2300);

			return base.OnProcess();
		}