protected override TimeSpan OnProcess() { IBackupService service; switch (_settings.Service) { case BackupServices.AwsS3: service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>()); break; case BackupServices.AwsGlacier: service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>()); break; default: throw new ArgumentOutOfRangeException(); } var hasSecurities = false; this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom)); var startDate = _settings.StartFrom; var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset); var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray(); var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path)); var workingSecurities = GetWorkingSecurities().ToArray(); foreach (var date in allDates) { foreach (var security in workingSecurities) { hasSecurities = true; if (!CanProcess()) { break; } var dateEntry = new BackupEntry { Name = date.ToString("yyyy_MM_dd"), Parent = new BackupEntry { Parent = new BackupEntry { Name = security.Security.Id.Substring(0, 1), Parent = pathEntry }, Name = security.Security.Id, } }; var dataTypes = _settings.Drive.GetAvailableDataTypes(security.Security.ToSecurityId(), _settings.StorageFormat); foreach (var dataType in dataTypes) { var storage = StorageRegistry.GetStorage(security.Security, dataType.MessageType, dataType.Arg, _settings.Drive, _settings.StorageFormat); var drive = storage.Drive; var stream = drive.LoadStream(date); if (stream == Stream.Null) { continue; } var entry = new BackupEntry { Name = LocalMarketDataDrive.GetFileName(dataType.MessageType, dataType.Arg) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary), Parent = dateEntry, }; service.Upload(entry, stream, p => { }); this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry)); } } if (CanProcess()) { _settings.StartFrom += TimeSpan.FromDays(1); SaveSettings(); } } if (!hasSecurities) { this.AddWarningLog(LocalizedStrings.Str2292); return(TimeSpan.MaxValue); } if (CanProcess()) { this.AddInfoLog(LocalizedStrings.Str2300); } return(base.OnProcess()); }
protected override TimeSpan OnProcess() { if (_settings.ExportType == ExportTypes.Sql && _settings.Connection == null) { this.AddErrorLog(LocalizedStrings.Str3768); return(TimeSpan.MaxValue); } var allSecurity = this.GetAllSecurity(); var supportedDataTypes = (allSecurity == null ? Enumerable.Empty <Type>() : SupportedMarketDataTypes.Intersect(allSecurity.MarketDataTypes) ).ToArray(); this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom)); Func <int, bool> isCancelled = count => !CanProcess(); var hasSecurities = false; foreach (var security in GetWorkingSecurities()) { hasSecurities = true; if (!CanProcess()) { break; } var path = _settings.ExportFolder; if (path.IsEmpty()) { path = DriveCache.Instance.DefaultDrive.Path; } foreach (var t in (allSecurity == null ? security.MarketDataTypes : supportedDataTypes)) { if (!CanProcess()) { break; } var arg = _settings.CandleSettings.Arg; var dataType = t.ToMessageType(ref arg); this.AddInfoLog(LocalizedStrings.Str3769Params.Put(security.Security.Id, dataType.Name, _settings.ExportType)); var fromStorage = StorageRegistry.GetStorage(security.Security, dataType, arg, _settings.Drive, _settings.StorageFormat); var from = fromStorage.GetFromDate(); var to = fromStorage.GetToDate(); if (from == null || to == null) { this.AddWarningLog(LocalizedStrings.Str3770); continue; } from = _settings.StartFrom.Max(from.Value); to = (DateTime.Today - TimeSpan.FromDays(_settings.Offset)).Min(to.Value); if (from > to) { continue; } BaseExporter exporter; if (_settings.ExportType == ExportTypes.Sql) { exporter = new DatabaseExporter(security.Security, arg, isCancelled, _settings.Connection) { BatchSize = _settings.BatchSize, CheckUnique = _settings.CheckUnique, }; } else { var fileName = Path.Combine(path, security.Security.GetFileName( dataType, arg, from.Value, to.Value, _settings.ExportType)); switch (_settings.ExportType) { case ExportTypes.Excel: exporter = new ExcelExporter(security.Security, arg, isCancelled, fileName, () => this.AddErrorLog(LocalizedStrings.Str3771)); break; case ExportTypes.Xml: exporter = new XmlExporter(security.Security, arg, isCancelled, fileName); break; case ExportTypes.Txt: exporter = new TextExporter(security.Security, arg, isCancelled, fileName, GetTxtTemplate(dataType, arg), _settings.Header); break; case ExportTypes.Bin: exporter = new BinExporter(security.Security, arg, isCancelled, DriveCache.Instance.GetDrive(path)); break; default: throw new ArgumentOutOfRangeException(); } } foreach (var date in from.Value.Range(to.Value, TimeSpan.FromDays(1))) { if (!CanProcess()) { break; } try { this.AddInfoLog(LocalizedStrings.Str3772Params.Put(security.Security.Id, dataType.Name, _settings.ExportType, date)); exporter.Export(dataType, fromStorage.Load(date)); } catch (Exception ex) { HandleError(ex); } } } } if (!hasSecurities) { this.AddWarningLog(LocalizedStrings.Str2292); return(TimeSpan.MaxValue); } if (CanProcess()) { this.AddInfoLog(LocalizedStrings.Str2300); _settings.StartFrom = DateTime.Today - TimeSpan.FromDays(_settings.Offset); SaveSettings(); } return(base.OnProcess()); }
private void Erase_Click(object sender, RoutedEventArgs e) { if (_token != null) { StopSync(); return; } var securities = AllSecurities.IsChecked == true ? null : SelectSecurityBtn.SelectedSecurities.ToArray(); var from = From.Value ?? DateTime.MinValue; var to = To.Value ?? DateTime.MaxValue; if (from > to) { new MessageBoxBuilder() .Caption(LocalizedStrings.Str2879) .Text(LocalizedStrings.Str1119Params.Put(from, to)) .Warning() .Owner(this) .Show(); return; } if (securities != null && securities.IsEmpty()) { new MessageBoxBuilder() .Caption(LocalizedStrings.Str2879) .Text(LocalizedStrings.Str2881) .Warning() .Owner(this) .Show(); return; } var msg = string.Empty; if (from != DateTime.MinValue || to != DateTime.MaxValue) { msg += LocalizedStrings.Str3846; if (from != DateTime.MinValue) { msg += " " + LocalizedStrings.XamlStr624.ToLowerInvariant() + " " + from.ToString("d"); } if (to != DateTime.MaxValue) { msg += " " + LocalizedStrings.XamlStr132 + " " + to.ToString("d"); } } //msg += AllSecurities.IsChecked == true // ? LocalizedStrings.Str2882 // : LocalizedStrings.Str2883Params.Put(securities.Take(100).Select(sec => sec.Id).Join(", ")); if (new MessageBoxBuilder() .Caption(LocalizedStrings.Str2879) .Text(LocalizedStrings.Str2884Params.Put(msg)) .Warning() .YesNo() .Owner(this) .Show() != MessageBoxResult.Yes) { return; } Erase.Content = LocalizedStrings.Str2890; _token = new CancellationTokenSource(); var drives = Drive.IsAllDrive ? DriveCache.Instance.AllDrives.ToArray() : new[] { Drive.SelectedDrive }; if (to != DateTime.MaxValue && to.TimeOfDay == TimeSpan.Zero) { to = to.EndOfDay(); } Task.Factory.StartNew(() => { var formats = Enumerator.GetValues <StorageFormats>().ToArray(); var iterCount = drives.Length * (securities?.Length ?? ((IStorageEntityList <Security>)EntityRegistry.Securities).Count) * 5 /* message types count */ * formats.Length; this.GuiSync(() => Progress.Maximum = iterCount); foreach (var drive in drives) { if (_token.IsCancellationRequested) { break; } foreach (var securityId in drive.AvailableSecurities) { if (_token.IsCancellationRequested) { break; } var id = securityId.ToStringId(); var security = securities == null ? EntityRegistry.Securities.ReadById(id) : securities.FirstOrDefault(s => s.Id.CompareIgnoreCase(id)); if (security == null) { continue; } foreach (var format in formats) { if (_token.IsCancellationRequested) { break; } foreach (var dataType in drive.GetAvailableDataTypes(securityId, format)) { if (_token.IsCancellationRequested) { break; } StorageRegistry .GetStorage(security, dataType.MessageType, dataType.Arg, drive, format) .Delete(from, to); this.GuiSync(() => { Progress.Value++; }); } } } } }, _token.Token) .ContinueWithExceptionHandling(this, res => { Erase.Content = LocalizedStrings.Str2060; Erase.IsEnabled = true; Progress.Value = 0; _token = null; }); }
private bool DownloadData(HydraTaskSecurity security, Type dataType, object arg, RemoteStorageClient client) { var localStorage = StorageRegistry.GetStorage(security.Security, dataType, arg, _settings.Drive, _settings.StorageFormat); var remoteStorage = client.GetRemoteStorage(security.Security.ToSecurityId(), dataType, arg, _settings.StorageFormat); var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset); var dates = remoteStorage.Dates.Where(date => date >= _settings.StartFrom && date <= endDate).Except(localStorage.Dates).ToArray(); if (dates.IsEmpty()) { if (!CanProcess()) { return(false); } } else { this.AddInfoLog(LocalizedStrings.Str2308Params.Put(dataType.Name)); foreach (var date in dates) { if (!CanProcess()) { return(false); } if (_settings.IgnoreWeekends && !security.IsTradeDate(date)) { this.AddDebugLog(LocalizedStrings.WeekEndDate, date); continue; } this.AddDebugLog(LocalizedStrings.StartDownloding, dataType, arg, date, security.Security.Id); using (var stream = remoteStorage.LoadStream(date)) { if (stream == Stream.Null) { this.AddDebugLog(LocalizedStrings.NoData); continue; } this.AddInfoLog(LocalizedStrings.Str2309Params.Put(date)); localStorage.Drive.SaveStream(date, stream); var info = localStorage.Serializer.CreateMetaInfo(date); stream.Position = 0; info.Read(stream); if (dataType == typeof(Trade)) { dataType = typeof(ExecutionMessage); arg = ExecutionTypes.Tick; } else if (dataType == typeof(OrderLogItem)) { dataType = typeof(ExecutionMessage); arg = ExecutionTypes.OrderLog; } else if (dataType.IsCandle()) { dataType = dataType.ToCandleMessageType(); } RaiseDataLoaded(security.Security, dataType, arg, date, info.Count); } } } return(true); }
private void Erase_Click(object sender, RoutedEventArgs e) { if (_token != null) { StopSync(); return; } var securities = (AllSecurities.IsChecked == true ? EntityRegistry.Securities.Where(s => !s.IsAllSecurity()) : SelectSecurityBtn.SelectedSecurities ).ToArray(); var from = From.Value ?? DateTime.MinValue; var to = To.Value ?? DateTime.MaxValue; if (from > to) { new MessageBoxBuilder() .Caption(LocalizedStrings.Str2879) .Text(LocalizedStrings.Str1119Params.Put(from, to)) .Warning() .Owner(this) .Show(); return; } if (securities.IsEmpty()) { new MessageBoxBuilder() .Caption(LocalizedStrings.Str2879) .Text(LocalizedStrings.Str2881) .Warning() .Owner(this) .Show(); return; } var msg = string.Empty; if (from != DateTime.MinValue || to != DateTime.MaxValue) { msg += LocalizedStrings.Str3846; if (from != DateTime.MinValue) { msg += " " + LocalizedStrings.XamlStr624.ToLowerInvariant() + " " + from.ToString("d"); } if (to != DateTime.MaxValue) { msg += " " + LocalizedStrings.XamlStr132 + " " + to.ToString("d"); } } //msg += AllSecurities.IsChecked == true // ? LocalizedStrings.Str2882 // : LocalizedStrings.Str2883Params.Put(securities.Take(100).Select(sec => sec.Id).Join(", ")); if (new MessageBoxBuilder() .Caption(LocalizedStrings.Str2879) .Text(LocalizedStrings.Str2884Params.Put(msg)) .Warning() .YesNo() .Owner(this) .Show() != MessageBoxResult.Yes) { return; } Erase.Content = LocalizedStrings.Str2890; _token = new CancellationTokenSource(); var drives = Drive.SelectedDrive == null ? DriveCache.Instance.AllDrives.ToArray() : new[] { Drive.SelectedDrive }; if (to != DateTime.MaxValue && to.TimeOfDay == TimeSpan.Zero) { to = to.EndOfDay(); } Task.Factory.StartNew(() => { var dataTypes = new[] { typeof(Trade), typeof(QuoteChangeMessage), typeof(Level1ChangeMessage), typeof(OrderLogItem), typeof(CandleMessage) }; var formats = Enumerator.GetValues <StorageFormats>().ToArray(); var iterCount = drives.Length * securities.Length * dataTypes.Length * formats.Length; this.GuiSync(() => Progress.Maximum = iterCount); foreach (var drive in drives) { if (_token.IsCancellationRequested) { break; } foreach (var security in securities) { if (_token.IsCancellationRequested) { break; } foreach (var format in formats) { if (_token.IsCancellationRequested) { break; } foreach (var dataType in dataTypes) { if (_token.IsCancellationRequested) { break; } if (dataType == typeof(CandleMessage)) { foreach (var candleType in drive.GetCandleTypes(security.ToSecurityId(), format)) { StorageRegistry .GetStorage(security, candleType.Item1, candleType.Item2, drive, format) .Delete(from, to); } } else { StorageRegistry .GetStorage(security, dataType, null, drive, format) .Delete(from, to); } this.GuiSync(() => { Progress.Value++; }); } } } } }, _token.Token) .ContinueWithExceptionHandling(this, res => { Erase.Content = LocalizedStrings.Str2060; Erase.IsEnabled = true; Progress.Value = 0; _token = null; }); }
private void SafeSave <T>(Security security, Type messageType, object arg, IEnumerable <T> values, Func <T, DateTimeOffset> getTime, IEnumerable <Func <T, string> > getErrors) { SafeSave(security, messageType, arg, values, getTime, getErrors, (s, d, f) => (IMarketDataStorage <T>)StorageRegistry.GetStorage(s, typeof(T), arg, d, f)); }
protected override TimeSpan OnProcess() { IBackupService service; switch (_settings.Service) { case BackupServices.AwsS3: service = new AmazonS3Service(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>()); break; case BackupServices.AwsGlacier: service = new AmazonGlacierService(AmazonExtensions.GetEndpoint(_settings.Address), _settings.ServiceRepo, _settings.Login, _settings.Password.To <string>()); break; default: throw new ArgumentOutOfRangeException(); } var hasSecurities = false; this.AddInfoLog(LocalizedStrings.Str2306Params.Put(_settings.StartFrom)); var startDate = _settings.StartFrom; var endDate = DateTime.Today - TimeSpan.FromDays(_settings.Offset); var allDates = startDate.Range(endDate, TimeSpan.FromDays(1)).ToArray(); var pathEntry = ToEntry(new DirectoryInfo(_settings.Drive.Path)); IEnumerable <Tuple <Type, object> > dataTypes = new[] { Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Tick), Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.OrderLog), Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Order), Tuple.Create(typeof(ExecutionMessage), (object)ExecutionTypes.Trade), Tuple.Create(typeof(QuoteChangeMessage), (object)null), Tuple.Create(typeof(Level1ChangeMessage), (object)null), Tuple.Create(typeof(NewsMessage), (object)null) }; var workingSecurities = GetWorkingSecurities().ToArray(); foreach (var date in allDates) { foreach (var security in workingSecurities) { hasSecurities = true; if (!CanProcess()) { break; } var dateEntry = new BackupEntry { Name = date.ToString("yyyy_MM_dd"), Parent = new BackupEntry { Parent = new BackupEntry { Name = security.Security.Id.Substring(0, 1), Parent = pathEntry }, Name = security.Security.Id, } }; var candleTypes = _settings.Drive.GetCandleTypes(security.Security.ToSecurityId(), _settings.StorageFormat); var secDataTypes = dataTypes.Concat(candleTypes.SelectMany(t => t.Item2.Select(a => Tuple.Create(t.Item1, a)))); foreach (var tuple in secDataTypes) { var storage = StorageRegistry.GetStorage(security.Security, tuple.Item1, tuple.Item2, _settings.Drive, _settings.StorageFormat); var drive = storage.Drive; var stream = drive.LoadStream(date); if (stream == Stream.Null) { continue; } var entry = new BackupEntry { Name = LocalMarketDataDrive.CreateFileName(tuple.Item1, tuple.Item2) + LocalMarketDataDrive.GetExtension(StorageFormats.Binary), Parent = dateEntry, }; service.Upload(entry, stream, p => { }); this.AddInfoLog(LocalizedStrings.Str1580Params, GetPath(entry)); } } if (CanProcess()) { _settings.StartFrom += TimeSpan.FromDays(1); SaveSettings(); } } if (!hasSecurities) { this.AddWarningLog(LocalizedStrings.Str2292); return(TimeSpan.MaxValue); } if (CanProcess()) { this.AddInfoLog(LocalizedStrings.Str2300); } return(base.OnProcess()); }