// TODO make async
CancellationTokenSource IBackupService.Download(BackupEntry entry, Stream stream, Action <int> progress)
{
if (entry == null)
{
throw new ArgumentNullException(nameof(entry));
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (progress == null)
{
throw new ArgumentNullException(nameof(progress));
}
var source = new CancellationTokenSource();
var key = GetKey(entry);
var request = new GetObjectRequest
{
BucketName = _bucket,
Key = key,
};
var bytes = new byte[_bufferSize];
var readTotal = 0L;
using (var response = _client.GetObject(request))
using (var responseStream = response.ResponseStream)
{
response.WriteObjectProgressEvent += (s, a) => progress(a.PercentDone);
while (readTotal < response.ContentLength)
{
var read = responseStream.Read(bytes, 0, bytes.Length);
if (read <= 0)
{
throw new InvalidOperationException(LocalizedStrings.NetworkConnectionError.Put(read));
}
stream.Write(bytes, 0, read);
readTotal += read;
}
}
return(source);
}
private static BackupEntry GetPath(string key)
{
var entities = key.Split('/').Select(p => new BackupEntry {
Name = p
}).ToArray();
BackupEntry parent = null;
foreach (var entity in entities)
{
entity.Parent = parent;
parent = entity;
}
return(entities.Last());
}
private static string GetKey(BackupEntry entry)
{
var key = entry.Name;
if (key.IsEmpty())
{
throw new ArgumentException("entry");
}
if (entry.Parent != null)
{
key = GetKey(entry.Parent) + "/" + key;
}
return(key);
}
IEnumerable <BackupEntry> IBackupService.Get(BackupEntry parent)
{
//if (parent != null && !parent.IsDirectory)
// throw new ArgumentException("{0} should be directory.".Put(parent.Name), "parent");
var request = new ListObjectsRequest
{
BucketName = _bucket,
Prefix = parent != null?GetKey(parent) : null,
};
do
{
var response = _client.ListObjects(request);
foreach (var entry in response.S3Objects)
{
var be = GetPath(entry.Key);
be.Size = entry.Size;
yield return(be);
}
foreach (var commonPrefix in response.CommonPrefixes)
{
yield return(new BackupEntry
{
Name = commonPrefix,
Parent = parent,
});
}
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
break;
}
}while (true);
}
IEnumerable<BackupEntry> IBackupService.Get(BackupEntry parent)
{
//if (parent != null && !parent.IsDirectory)
// throw new ArgumentException("{0} should be directory.".Put(parent.Name), "parent");
var request = new ListObjectsRequest
{
BucketName = _bucket,
Prefix = parent != null ? GetKey(parent) : null,
};
do
{
var response = _client.ListObjects(request);
foreach (var entry in response.S3Objects)
{
var be = GetPath(entry.Key);
be.Size = entry.Size;
yield return be;
}
foreach (var commonPrefix in response.CommonPrefixes)
{
yield return new BackupEntry
{
Name = commonPrefix,
Parent = parent,
};
}
if (response.IsTruncated)
request.Marker = response.NextMarker;
else
break;
}
while (true);
}
void IBackupService.Delete(BackupEntry entry)
{
throw new NotImplementedException();
}
private static string GetPath(BackupEntry entry)
{
if (entry == null)
return null;
return GetPath(entry.Parent) + "/" + entry.Name;
}
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action<int> progress)
{
if (entry == null)
throw new ArgumentNullException(nameof(entry));
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (progress == null)
throw new ArgumentNullException(nameof(progress));
var key = GetKey(entry);
var initResponse = _client.InitiateMultipartUpload(new InitiateMultipartUploadRequest
{
BucketName = _bucket,
Key = key,
});
var filePosition = 0L;
var nextProgress = 1;
var etags = new List<PartETag>();
var partNum = 1;
while (filePosition < stream.Length)
{
var response = _client.UploadPart(new UploadPartRequest
{
BucketName = _bucket,
UploadId = initResponse.UploadId,
PartNumber = partNum,
PartSize = _bufferSize,
//FilePosition = filePosition,
InputStream = stream,
Key = key
});
etags.Add(new PartETag(partNum, response.ETag));
filePosition += _bufferSize;
var currProgress = (int)(filePosition.Min(stream.Length) * 100 / stream.Length);
if (currProgress >= nextProgress)
{
progress(currProgress);
nextProgress = currProgress + 1;
}
partNum++;
}
_client.CompleteMultipartUpload(new CompleteMultipartUploadRequest
{
BucketName = _bucket,
UploadId = initResponse.UploadId,
Key = key,
PartETags = etags
});
var source = new CancellationTokenSource();
return source;
}
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action <int> progress)
{
if (entry == null)
{
throw new ArgumentNullException(nameof(entry));
}
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (progress == null)
{
throw new ArgumentNullException(nameof(progress));
}
var key = GetKey(entry);
var initResponse = _client.InitiateMultipartUpload(new InitiateMultipartUploadRequest
{
BucketName = _bucket,
Key = key,
});
var filePosition = 0L;
var nextProgress = 1;
var etags = new List <PartETag>();
var partNum = 1;
while (filePosition < stream.Length)
{
var response = _client.UploadPart(new UploadPartRequest
{
BucketName = _bucket,
UploadId = initResponse.UploadId,
PartNumber = partNum,
PartSize = _bufferSize,
//FilePosition = filePosition,
InputStream = stream,
Key = key
});
etags.Add(new PartETag(partNum, response.ETag));
filePosition += _bufferSize;
var currProgress = (int)(filePosition.Min(stream.Length) * 100 / stream.Length);
if (currProgress >= nextProgress)
{
progress(currProgress);
nextProgress = currProgress + 1;
}
partNum++;
}
_client.CompleteMultipartUpload(new CompleteMultipartUploadRequest
{
BucketName = _bucket,
UploadId = initResponse.UploadId,
Key = key,
PartETags = etags
});
var source = new CancellationTokenSource();
return(source);
}
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action<int> progress)
{
throw new NotImplementedException();
}
void IBackupService.Delete(BackupEntry entry)
{
throw new NotImplementedException();
}
IEnumerable<BackupEntry> IBackupService.Get(BackupEntry parent)
{
throw new NotImplementedException();
}
protected override TimeSpan OnProcess()
{
IBackupService service;
switch (_settings.Service)
{
case BackupServices.AwsS3:
service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
break;
case BackupServices.AwsGlacier:
service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
break;
default:
throw new ArgumentOutOfRangeException();
}
var hasSecurities = false;
this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));
var startDate = _settings.StartFrom;
var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);
var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray();
var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path));
IEnumerable<Tuple<Type, object>> dataTypes = new[]
{
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Tick),
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.OrderLog),
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Order),
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Trade),
Tuple.Create(typeof(QuoteChangeMessage), (object)null),
Tuple.Create(typeof(Level1ChangeMessage), (object)null),
Tuple.Create(typeof(NewsMessage), (object)null)
};
var workingSecurities = GetWorkingSecurities().ToArray();
foreach (var date in allDates)
{
foreach (var security in workingSecurities)
{
hasSecurities = true;
if (!CanProcess())
break;
var dateEntry = new BackupEntry
{
Name = date.ToString("yyyy_MM_dd"),
Parent = new BackupEntry
{
Parent = new BackupEntry
{
Name = security.Security.Id.Substring(0, 1),
Parent = pathEntry
},
Name = security.Security.Id,
}
};
var candleTypes = _settings.Drive.GetCandleTypes(security.Security.ToSecurityId(), _settings.StorageFormat);
var secDataTypes = dataTypes.Concat(candleTypes.SelectMany(t => t.Item2.Select(a => Tuple.Create(t.Item1, a))));
foreach (var tuple in secDataTypes)
{
var storage = StorageRegistry.GetStorage(security.Security, tuple.Item1, tuple.Item2, _settings.Drive, _settings.StorageFormat);
var drive = storage.Drive;
var stream = drive.LoadStream(date);
if (stream == Stream.Null)
continue;
var entry = new BackupEntry
{
Name = LocalMarketDataDrive.CreateFileName(tuple.Item1, tuple.Item2) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary),
Parent = dateEntry,
};
service.Upload(entry, stream, p => { });
this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry));
}
}
if (CanProcess())
{
_settings.StartFrom += TimeSpan.FromDays(1);
SaveSettings();
}
}
if (!hasSecurities)
{
this.AddWarningLog(LocalizedStrings.Str2292);
return TimeSpan.MaxValue;
}
if (CanProcess())
this.AddInfoLog(LocalizedStrings.Str2300);
return base.OnProcess();
}
void IBackupService.Delete(BackupEntry entry)
{
_client.DeleteObject(_bucket, GetKey(entry));
}
IEnumerable <BackupEntry> IBackupService.Get(BackupEntry parent)
{
throw new NotImplementedException();
}
// TODO make async
CancellationTokenSource IBackupService.Download(BackupEntry entry, Stream stream, Action<int> progress)
{
if (entry == null)
throw new ArgumentNullException(nameof(entry));
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (progress == null)
throw new ArgumentNullException(nameof(progress));
var source = new CancellationTokenSource();
var key = GetKey(entry);
var request = new GetObjectRequest
{
BucketName = _bucket,
Key = key,
};
var bytes = new byte[_bufferSize];
var readTotal = 0L;
using (var response = _client.GetObject(request))
using (var responseStream = response.ResponseStream)
{
response.WriteObjectProgressEvent += (s, a) => progress(a.PercentDone);
while (readTotal < response.ContentLength)
{
var read = responseStream.Read(bytes, 0, bytes.Length);
if (read <= 0)
throw new InvalidOperationException(LocalizedStrings.NetworkConnectionError.Put(read));
stream.Write(bytes, 0, read);
readTotal += read;
}
}
return source;
}
void IBackupService.Delete(BackupEntry entry)
{
_client.DeleteObject(_bucket, GetKey(entry));
}
private static string GetKey(BackupEntry entry)
{
var key = entry.Name;
if (key.IsEmpty())
throw new ArgumentException("entry");
if (entry.Parent != null)
key = GetKey(entry.Parent) + "/" + key;
return key;
}
CancellationTokenSource IBackupService.Upload(BackupEntry entry, Stream stream, Action <int> progress)
{
throw new NotImplementedException();
}
protected override TimeSpan OnProcess()
{
IBackupService service;
switch (_settings.Service)
{
case BackupServices.AwsS3:
service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
break;
case BackupServices.AwsGlacier:
service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To<string>());
break;
default:
throw new ArgumentOutOfRangeException();
}
var hasSecurities = false;
this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));
var startDate = _settings.StartFrom;
var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);
var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray();
var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path));
var workingSecurities = GetWorkingSecurities().ToArray();
foreach (var date in allDates)
{
foreach (var security in workingSecurities)
{
hasSecurities = true;
if (!CanProcess())
break;
var dateEntry = new BackupEntry
{
Name = date.ToString("yyyy_MM_dd"),
Parent = new BackupEntry
{
Parent = new BackupEntry
{
Name = security.Security.Id.Substring(0, 1),
Parent = pathEntry
},
Name = security.Security.Id,
}
};
var dataTypes = _settings.Drive.GetAvailableDataTypes(security.Security.ToSecurityId(), _settings.StorageFormat);
foreach (var dataType in dataTypes)
{
var storage = StorageRegistry.GetStorage(security.Security, dataType.MessageType, dataType.Arg, _settings.Drive, _settings.StorageFormat);
var drive = storage.Drive;
var stream = drive.LoadStream(date);
if (stream == Stream.Null)
continue;
var entry = new BackupEntry
{
Name = LocalMarketDataDrive.GetFileName(dataType.MessageType, dataType.Arg) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary),
Parent = dateEntry,
};
service.Upload(entry, stream, p => { });
this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry));
}
}
if (CanProcess())
{
_settings.StartFrom += TimeSpan.FromDays(1);
SaveSettings();
}
}
if (!hasSecurities)
{
this.AddWarningLog(LocalizedStrings.Str2292);
return TimeSpan.MaxValue;
}
if (CanProcess())
this.AddInfoLog(LocalizedStrings.Str2300);
return base.OnProcess();
}