protected override TimeSpan OnProcess()
{
IBackupService service;
switch (_settings.Service)
{
case BackupServices.AwsS3:
service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>());
break;
case BackupServices.AwsGlacier:
service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>());
break;
default:
throw new ArgumentOutOfRangeException();
}
var hasSecurities = false;
this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));
var startDate = _settings.StartFrom;
var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);
var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray();
var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path));
var workingSecurities = GetWorkingSecurities().ToArray();
foreach (var date in allDates)
{
foreach (var security in workingSecurities)
{
hasSecurities = true;
if (!CanProcess())
{
break;
}
var dateEntry = new BackupEntry
{
Name = date.ToString("yyyy_MM_dd"),
Parent = new BackupEntry
{
Parent = new BackupEntry
{
Name = security.Security.Id.Substring(0, 1),
Parent = pathEntry
},
Name = security.Security.Id,
}
};
var dataTypes = _settings.Drive.GetAvailableDataTypes(security.Security.ToSecurityId(), _settings.StorageFormat);
foreach (var dataType in dataTypes)
{
var storage = StorageRegistry.GetStorage(security.Security, dataType.MessageType, dataType.Arg, _settings.Drive, _settings.StorageFormat);
var drive = storage.Drive;
var stream = drive.LoadStream(date);
if (stream == Stream.Null)
{
continue;
}
var entry = new BackupEntry
{
Name = LocalMarketDataDrive.GetFileName(dataType.MessageType, dataType.Arg) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary),
Parent = dateEntry,
};
service.Upload(entry, stream, p => { });
this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry));
}
}
if (CanProcess())
{
_settings.StartFrom += TimeSpan.FromDays(1);
SaveSettings();
}
}
if (!hasSecurities)
{
this.AddWarningLog(LocalizedStrings.Str2292);
return(TimeSpan.MaxValue);
}
if (CanProcess())
{
this.AddInfoLog(LocalizedStrings.Str2300);
}
return(base.OnProcess());
}
protected override TimeSpan OnProcess()
{
if (_settings.ExportType == ExportTypes.Sql && _settings.Connection == null)
{
this.AddErrorLog(LocalizedStrings.Str3768);
return(TimeSpan.MaxValue);
}
var allSecurity = this.GetAllSecurity();
var supportedDataTypes = (allSecurity == null
? Enumerable.Empty <Type>()
: SupportedMarketDataTypes.Intersect(allSecurity.MarketDataTypes)
).ToArray();
this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));
Func <int, bool> isCancelled = count => !CanProcess();
var hasSecurities = false;
foreach (var security in GetWorkingSecurities())
{
hasSecurities = true;
if (!CanProcess())
{
break;
}
var path = _settings.ExportFolder;
if (path.IsEmpty())
{
path = DriveCache.Instance.DefaultDrive.Path;
}
foreach (var t in (allSecurity == null ? security.MarketDataTypes : supportedDataTypes))
{
if (!CanProcess())
{
break;
}
var arg = _settings.CandleSettings.Arg;
var dataType = t.ToMessageType(ref arg);
this.AddInfoLog(LocalizedStrings.Str3769Params.Put(security.Security.Id, dataType.Name, _settings.ExportType));
var fromStorage = StorageRegistry.GetStorage(security.Security, dataType, arg, _settings.Drive, _settings.StorageFormat);
var from = fromStorage.GetFromDate();
var to = fromStorage.GetToDate();
if (from == null || to == null)
{
this.AddWarningLog(LocalizedStrings.Str3770);
continue;
}
from = _settings.StartFrom.Max(from.Value);
to = (DateTime.Today - TimeSpan.FromDays(_settings.Offset)).Min(to.Value);
if (from > to)
{
continue;
}
BaseExporter exporter;
if (_settings.ExportType == ExportTypes.Sql)
{
exporter = new DatabaseExporter(security.Security, arg, isCancelled, _settings.Connection)
{
BatchSize = _settings.BatchSize,
CheckUnique = _settings.CheckUnique,
};
}
else
{
var fileName = Path.Combine(path, security.Security.GetFileName(
dataType, arg, from.Value, to.Value, _settings.ExportType));
switch (_settings.ExportType)
{
case ExportTypes.Excel:
exporter = new ExcelExporter(security.Security, arg, isCancelled, fileName, () => this.AddErrorLog(LocalizedStrings.Str3771));
break;
case ExportTypes.Xml:
exporter = new XmlExporter(security.Security, arg, isCancelled, fileName);
break;
case ExportTypes.Txt:
exporter = new TextExporter(security.Security, arg, isCancelled, fileName, GetTxtTemplate(dataType, arg), _settings.Header);
break;
case ExportTypes.Bin:
exporter = new BinExporter(security.Security, arg, isCancelled, DriveCache.Instance.GetDrive(path));
break;
default:
throw new ArgumentOutOfRangeException();
}
}
foreach (var date in from.Value.Range(to.Value, TimeSpan.FromDays(1)))
{
if (!CanProcess())
{
break;
}
try
{
this.AddInfoLog(LocalizedStrings.Str3772Params.Put(security.Security.Id, dataType.Name, _settings.ExportType, date));
exporter.Export(dataType, fromStorage.Load(date));
}
catch (Exception ex)
{
HandleError(ex);
}
}
}
}
if (!hasSecurities)
{
this.AddWarningLog(LocalizedStrings.Str2292);
return(TimeSpan.MaxValue);
}
if (CanProcess())
{
this.AddInfoLog(LocalizedStrings.Str2300);
_settings.StartFrom = DateTime.Today - TimeSpan.FromDays(_settings.Offset);
SaveSettings();
}
return(base.OnProcess());
}
private void Erase_Click(object sender, RoutedEventArgs e)
{
if (_token != null)
{
StopSync();
return;
}
var securities = AllSecurities.IsChecked == true
? null
: SelectSecurityBtn.SelectedSecurities.ToArray();
var from = From.Value ?? DateTime.MinValue;
var to = To.Value ?? DateTime.MaxValue;
if (from > to)
{
new MessageBoxBuilder()
.Caption(LocalizedStrings.Str2879)
.Text(LocalizedStrings.Str1119Params.Put(from, to))
.Warning()
.Owner(this)
.Show();
return;
}
if (securities != null && securities.IsEmpty())
{
new MessageBoxBuilder()
.Caption(LocalizedStrings.Str2879)
.Text(LocalizedStrings.Str2881)
.Warning()
.Owner(this)
.Show();
return;
}
var msg = string.Empty;
if (from != DateTime.MinValue || to != DateTime.MaxValue)
{
msg += LocalizedStrings.Str3846;
if (from != DateTime.MinValue)
{
msg += " " + LocalizedStrings.XamlStr624.ToLowerInvariant() + " " + from.ToString("d");
}
if (to != DateTime.MaxValue)
{
msg += " " + LocalizedStrings.XamlStr132 + " " + to.ToString("d");
}
}
//msg += AllSecurities.IsChecked == true
// ? LocalizedStrings.Str2882
// : LocalizedStrings.Str2883Params.Put(securities.Take(100).Select(sec => sec.Id).Join(", "));
if (new MessageBoxBuilder()
.Caption(LocalizedStrings.Str2879)
.Text(LocalizedStrings.Str2884Params.Put(msg))
.Warning()
.YesNo()
.Owner(this)
.Show() != MessageBoxResult.Yes)
{
return;
}
Erase.Content = LocalizedStrings.Str2890;
_token = new CancellationTokenSource();
var drives = Drive.IsAllDrive
? DriveCache.Instance.AllDrives.ToArray()
: new[] { Drive.SelectedDrive };
if (to != DateTime.MaxValue && to.TimeOfDay == TimeSpan.Zero)
{
to = to.EndOfDay();
}
Task.Factory.StartNew(() =>
{
var formats = Enumerator.GetValues <StorageFormats>().ToArray();
var iterCount = drives.Length * (securities?.Length ?? ((IStorageEntityList <Security>)EntityRegistry.Securities).Count) * 5 /* message types count */ * formats.Length;
this.GuiSync(() => Progress.Maximum = iterCount);
foreach (var drive in drives)
{
if (_token.IsCancellationRequested)
{
break;
}
foreach (var securityId in drive.AvailableSecurities)
{
if (_token.IsCancellationRequested)
{
break;
}
var id = securityId.ToStringId();
var security = securities == null
? EntityRegistry.Securities.ReadById(id)
: securities.FirstOrDefault(s => s.Id.CompareIgnoreCase(id));
if (security == null)
{
continue;
}
foreach (var format in formats)
{
if (_token.IsCancellationRequested)
{
break;
}
foreach (var dataType in drive.GetAvailableDataTypes(securityId, format))
{
if (_token.IsCancellationRequested)
{
break;
}
StorageRegistry
.GetStorage(security, dataType.MessageType, dataType.Arg, drive, format)
.Delete(from, to);
this.GuiSync(() =>
{
Progress.Value++;
});
}
}
}
}
}, _token.Token)
.ContinueWithExceptionHandling(this, res =>
{
Erase.Content = LocalizedStrings.Str2060;
Erase.IsEnabled = true;
Progress.Value = 0;
_token = null;
});
}
private bool DownloadData(HydraTaskSecurity security, Type dataType, object arg, RemoteStorageClient client)
{
var localStorage = StorageRegistry.GetStorage(security.Security, dataType, arg, _settings.Drive, _settings.StorageFormat);
var remoteStorage = client.GetRemoteStorage(security.Security.ToSecurityId(), dataType, arg, _settings.StorageFormat);
var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);
var dates = remoteStorage.Dates.Where(date => date >= _settings.StartFrom && date <= endDate).Except(localStorage.Dates).ToArray();
if (dates.IsEmpty())
{
if (!CanProcess())
{
return(false);
}
}
else
{
this.AddInfoLog(LocalizedStrings.Str2308Params.Put(dataType.Name));
foreach (var date in dates)
{
if (!CanProcess())
{
return(false);
}
if (_settings.IgnoreWeekends && !security.IsTradeDate(date))
{
this.AddDebugLog(LocalizedStrings.WeekEndDate, date);
continue;
}
this.AddDebugLog(LocalizedStrings.StartDownloding, dataType, arg, date, security.Security.Id);
using (var stream = remoteStorage.LoadStream(date))
{
if (stream == Stream.Null)
{
this.AddDebugLog(LocalizedStrings.NoData);
continue;
}
this.AddInfoLog(LocalizedStrings.Str2309Params.Put(date));
localStorage.Drive.SaveStream(date, stream);
var info = localStorage.Serializer.CreateMetaInfo(date);
stream.Position = 0;
info.Read(stream);
if (dataType == typeof(Trade))
{
dataType = typeof(ExecutionMessage);
arg = ExecutionTypes.Tick;
}
else if (dataType == typeof(OrderLogItem))
{
dataType = typeof(ExecutionMessage);
arg = ExecutionTypes.OrderLog;
}
else if (dataType.IsCandle())
{
dataType = dataType.ToCandleMessageType();
}
RaiseDataLoaded(security.Security, dataType, arg, date, info.Count);
}
}
}
return(true);
}
private void Erase_Click(object sender, RoutedEventArgs e)
{
if (_token != null)
{
StopSync();
return;
}
var securities = (AllSecurities.IsChecked == true
? EntityRegistry.Securities.Where(s => !s.IsAllSecurity())
: SelectSecurityBtn.SelectedSecurities
).ToArray();
var from = From.Value ?? DateTime.MinValue;
var to = To.Value ?? DateTime.MaxValue;
if (from > to)
{
new MessageBoxBuilder()
.Caption(LocalizedStrings.Str2879)
.Text(LocalizedStrings.Str1119Params.Put(from, to))
.Warning()
.Owner(this)
.Show();
return;
}
if (securities.IsEmpty())
{
new MessageBoxBuilder()
.Caption(LocalizedStrings.Str2879)
.Text(LocalizedStrings.Str2881)
.Warning()
.Owner(this)
.Show();
return;
}
var msg = string.Empty;
if (from != DateTime.MinValue || to != DateTime.MaxValue)
{
msg += LocalizedStrings.Str3846;
if (from != DateTime.MinValue)
{
msg += " " + LocalizedStrings.XamlStr624.ToLowerInvariant() + " " + from.ToString("d");
}
if (to != DateTime.MaxValue)
{
msg += " " + LocalizedStrings.XamlStr132 + " " + to.ToString("d");
}
}
//msg += AllSecurities.IsChecked == true
// ? LocalizedStrings.Str2882
// : LocalizedStrings.Str2883Params.Put(securities.Take(100).Select(sec => sec.Id).Join(", "));
if (new MessageBoxBuilder()
.Caption(LocalizedStrings.Str2879)
.Text(LocalizedStrings.Str2884Params.Put(msg))
.Warning()
.YesNo()
.Owner(this)
.Show() != MessageBoxResult.Yes)
{
return;
}
Erase.Content = LocalizedStrings.Str2890;
_token = new CancellationTokenSource();
var drives = Drive.SelectedDrive == null
? DriveCache.Instance.AllDrives.ToArray()
: new[] { Drive.SelectedDrive };
if (to != DateTime.MaxValue && to.TimeOfDay == TimeSpan.Zero)
{
to = to.EndOfDay();
}
Task.Factory.StartNew(() =>
{
var dataTypes = new[] { typeof(Trade), typeof(QuoteChangeMessage), typeof(Level1ChangeMessage), typeof(OrderLogItem), typeof(CandleMessage) };
var formats = Enumerator.GetValues <StorageFormats>().ToArray();
var iterCount = drives.Length * securities.Length * dataTypes.Length * formats.Length;
this.GuiSync(() => Progress.Maximum = iterCount);
foreach (var drive in drives)
{
if (_token.IsCancellationRequested)
{
break;
}
foreach (var security in securities)
{
if (_token.IsCancellationRequested)
{
break;
}
foreach (var format in formats)
{
if (_token.IsCancellationRequested)
{
break;
}
foreach (var dataType in dataTypes)
{
if (_token.IsCancellationRequested)
{
break;
}
if (dataType == typeof(CandleMessage))
{
foreach (var candleType in drive.GetCandleTypes(security.ToSecurityId(), format))
{
StorageRegistry
.GetStorage(security, candleType.Item1, candleType.Item2, drive, format)
.Delete(from, to);
}
}
else
{
StorageRegistry
.GetStorage(security, dataType, null, drive, format)
.Delete(from, to);
}
this.GuiSync(() =>
{
Progress.Value++;
});
}
}
}
}
}, _token.Token)
.ContinueWithExceptionHandling(this, res =>
{
Erase.Content = LocalizedStrings.Str2060;
Erase.IsEnabled = true;
Progress.Value = 0;
_token = null;
});
}
private void SafeSave <T>(Security security, Type messageType, object arg, IEnumerable <T> values, Func <T, DateTimeOffset> getTime, IEnumerable <Func <T, string> > getErrors)
{
SafeSave(security, messageType, arg, values, getTime, getErrors, (s, d, f) => (IMarketDataStorage <T>)StorageRegistry.GetStorage(s, typeof(T), arg, d, f));
}
protected override TimeSpan OnProcess()
{
IBackupService service;
switch (_settings.Service)
{
case BackupServices.AwsS3:
service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>());
break;
case BackupServices.AwsGlacier:
service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>());
break;
default:
throw new ArgumentOutOfRangeException();
}
var hasSecurities = false;
this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom));
var startDate = _settings.StartFrom;
var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset);
var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray();
var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path));
IEnumerable <Tuple <Type, object> > dataTypes = new[]
{
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Tick),
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.OrderLog),
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Order),
Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Trade),
Tuple.Create(typeof(QuoteChangeMessage), (object)null),
Tuple.Create(typeof(Level1ChangeMessage), (object)null),
Tuple.Create(typeof(NewsMessage), (object)null)
};
var workingSecurities = GetWorkingSecurities().ToArray();
foreach (var date in allDates)
{
foreach (var security in workingSecurities)
{
hasSecurities = true;
if (!CanProcess())
{
break;
}
var dateEntry = new BackupEntry
{
Name = date.ToString("yyyy_MM_dd"),
Parent = new BackupEntry
{
Parent = new BackupEntry
{
Name = security.Security.Id.Substring(0, 1),
Parent = pathEntry
},
Name = security.Security.Id,
}
};
var candleTypes = _settings.Drive.GetCandleTypes(security.Security.ToSecurityId(), _settings.StorageFormat);
var secDataTypes = dataTypes.Concat(candleTypes.SelectMany(t => t.Item2.Select(a => Tuple.Create(t.Item1, a))));
foreach (var tuple in secDataTypes)
{
var storage = StorageRegistry.GetStorage(security.Security, tuple.Item1, tuple.Item2, _settings.Drive, _settings.StorageFormat);
var drive = storage.Drive;
var stream = drive.LoadStream(date);
if (stream == Stream.Null)
{
continue;
}
var entry = new BackupEntry
{
Name = LocalMarketDataDrive.CreateFileName(tuple.Item1, tuple.Item2) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary),
Parent = dateEntry,
};
service.Upload(entry, stream, p => { });
this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry));
}
}
if (CanProcess())
{
_settings.StartFrom += TimeSpan.FromDays(1);
SaveSettings();
}
}
if (!hasSecurities)
{
this.AddWarningLog(LocalizedStrings.Str2292);
return(TimeSpan.MaxValue);
}
if (CanProcess())
{
this.AddInfoLog(LocalizedStrings.Str2300);
}
return(base.OnProcess());
}