public Task SetArchiveOptions(ArchiveOptions archiveOptions) { this.ArchiveOptions.MinIntervalSeconds = archiveOptions.MinIntervalSeconds; this.ArchiveOptions.EventPageSize = archiveOptions.EventPageSize; this.ArchiveOptions.RetainSeconds = archiveOptions.RetainSeconds; return(Task.CompletedTask); }
public SqlGeneratorArchiveData() { Tables = new List <ArchiveTable>(); Columns = new List <ArchiveColumn>(); Options = new ArchiveOptions(); PdfDesign = new PdfDesign(); }
public Watcher(DoRecord log, ConfigManager config) { MakeRecord += log; var paths = config.GetConfiguration <PathOptions>() as PathOptions; watcher = new FileSystemWatcher(paths.SourceDirectory); watcher.Deleted += ElementDeleted; watcher.Created += ElementCreated; //watcher.Changed += ElementChanged; watcher.Renamed += ElementRenamed; watcher.Filter = "*.xml"; targetDir = paths.TargetDirectory; enableArchiveDir = paths.EnableArchivation; archivedFilesDir = paths.ArchiveDirectory; if (!Directory.Exists(archivedFilesDir) && enableArchiveDir) { try { Directory.CreateDirectory(archivedFilesDir); } catch { archivedFilesDir = targetDir + "\\archive"; Directory.CreateDirectory(archivedFilesDir); MakeRecord($"Couldn't find directory for archive files, so created this one {archivedFilesDir}.\n"); } } enableEncrypting = (config.GetConfiguration <EncryptionOptions>() as EncryptionOptions).EnableEncryption; archiveOptions = config.GetConfiguration <ArchiveOptions>() as ArchiveOptions; }
public bool IsCompletedArchive <GrainState>(ArchiveOptions <GrainState> archiveOptions, ArchiveBrief preArchive = default) { var intervalMilliseconds = preArchive == default ? EndTimestamp - StartTimestamp : EndTimestamp - preArchive.EndTimestamp; var intervalVersiion = EndVersion - StartVersion; return((intervalMilliseconds > archiveOptions.IntervalMilliSeconds && intervalVersiion > archiveOptions.IntervalVersion) || intervalMilliseconds > archiveOptions.MaxIntervalMilliSeconds || intervalVersiion > archiveOptions.MaxIntervalVersion); }
public EtlOptions(ArchiveOptions archiveOptions, EncryptionOptions encryptionOptions, LoggingOptions loggingOptions, MoveOptions moveOptions, MoveDbOptions moveDbOptions, WatcherOptions watcherOptions) { ArchiveOptions = archiveOptions; EncryptionOptions = encryptionOptions; LoggingOptions = loggingOptions; MoveOptions = moveOptions; MoveDbOptions = moveDbOptions; WatcherOptions = watcherOptions; }
public bool IsCompletedArchive(ArchiveOptions archiveOptions, ArchiveBrief preArchive = default) { var intervalMilliseconds = (preArchive == default ? EndTimestamp - StartTimestamp : EndTimestamp - preArchive.EndTimestamp) / 1000; var intervalVersiion = EndVersion - StartVersion; return((intervalMilliseconds > archiveOptions.SecondsInterval && intervalVersiion > archiveOptions.VersionInterval) || intervalMilliseconds > archiveOptions.MaxSecondsInterval || intervalVersiion > archiveOptions.MaxVersionInterval); }
private void RecordEntry(string filePath, string fileEvent) { ArchiveOptions archiveOptions = optionsManager.GetOptions <ArchiveOptions>(this) as ArchiveOptions; CipherOptions cipherOptions = optionsManager.GetOptions <CipherOptions>(this) as CipherOptions; string encrypted; FileInfo file = new FileInfo(filePath); string td = $"{targetDirectory}\\{file.LastWriteTime:yyyy\\\\MM\\\\dd}"; string newName = $"Sales_{file.LastWriteTime:yyyy_MM_dd_HH_mm_ss}"; int i = 0; while (File.Exists($"{td}\\{newName}.txt")) { i++; newName = $"{Path.GetFileNameWithoutExtension(filePath)}({i})_{file.LastWriteTime:yyyy_MM_dd_HH_mm_ss}"; } if (!Directory.Exists(td)) { Directory.CreateDirectory(td); } Log($"File {newName} {fileEvent}", true, logFile); using (StreamReader sr = new StreamReader(filePath)) { encrypted = Cypher.Encrypt(sr.ReadToEnd(), cipherOptions.EncrypterKey); } using (StreamWriter sw = new StreamWriter(filePath)) { sw.Write(encrypted); } string newPath = $"{td}\\{newName}"; try { Archivate.Compress(filePath, newPath + ".gz", archiveOptions.CompressionLevel); Archivate.Compress(filePath, $"{archiveDirectory}\\{newName}.gz", archiveOptions.CompressionLevel); Archivate.Decompress(newPath + ".gz", newPath + ".txt"); } catch (Exception ex) { Log($"Fatal error ocured while compressing - {ex}", true, logFile); return; } File.Delete(filePath); File.Delete(newPath + ".gz"); using (StreamReader sr = new StreamReader(newPath + ".txt")) { encrypted = sr.ReadToEnd(); } using (StreamWriter sw = new StreamWriter(newPath + ".txt")) { sw.Write(Cypher.Decrypt(encrypted, cipherOptions.EncrypterKey)); } Log($"File {newName} sent successfully", true, logFile); }
public EtlXmlOptions(ArchiveOptions archiveOptions, CryptingOptions cryptingOptions, LoggerOptions loggerOptions, WatcherOptions watcherOptions, Options defaultOptions) { ArchiveOptions = archiveOptions; CryptingOptions = cryptingOptions; LoggerOptions = loggerOptions; WatcherOptions = watcherOptions; DefaultOptions = defaultOptions; SourceDirectory = DefaultOptions.SourceDirectory; TargetDirectory = DefaultOptions.TargetDirectory; IsLoggerEnable = DefaultOptions.IsLoggerEnable; }
public string Compress(string place, ArchiveOptions archiveOptions) { if (!Info.Exists) { return(null); } string compressedName = PathHelper.GetNameOfNewFile( place + "\\" + Info.Name, COMPRESSED_TYPE); compresser.Compress(Info.FullName, compressedName, archiveOptions); return(compressedName); }
public async Task CompressAsync(string pathToFile, string compressedName, ArchiveOptions archiveOptions) { using (var sourceStream = new FileStream(pathToFile, FileMode.OpenOrCreate)) { using (var targetStream = File.Open(compressedName, FileMode.Create)) { using (var compressionStream = new GZipStream(targetStream, archiveOptions.CompressionLevel)) { await sourceStream.CopyToAsync(compressionStream); } } } }
public PerfionXml Query(string query, Action <QueryArchiveOptions> archive = null) { return(Client(client => { string xml = client.ExecuteQuery(new ExecuteQueryRequest { Body = new ExecuteQueryRequestBody(query) }).Body.ExecuteQueryResult; ArchiveCreated archiveCreated = null; ArchiveOptions globalArchiveOptions = _configuration.ArchiveOptions; QueryArchiveOptions localArchiveOptions = GetLocalArchiveOptions(archive, globalArchiveOptions); ArchiveOptions finalArchiveOptions = localArchiveOptions ?? globalArchiveOptions; if (localArchiveOptions != null && localArchiveOptions.Disabled) { finalArchiveOptions = null; } if (finalArchiveOptions != null) { archiveCreated = _kernel.Resolve <IArchiveService>().Archive(finalArchiveOptions.Name, newArchive => { newArchive.Options.GroupedBy(finalArchiveOptions.GroupName); if (finalArchiveOptions.Expires.HasValue) { newArchive.Options.ExpiresOn(finalArchiveOptions.Expires.Value); } if (finalArchiveOptions.CompressionLevel.HasValue) { newArchive.Options.Compression(finalArchiveOptions.CompressionLevel.Value); } newArchive .IncludeContent("Query.xml", query) .IncludeContent("Data.xml", xml); }); } return new PerfionXml(this, XDocument.Parse(xml)) { Archive = archiveCreated }; })); }
internal static void Validate(ETLOptions options, Logger logger) { DirectoryOptions directoryOptions = options.DirectoryOptions; if (!MakeValidDir(directoryOptions.SourceDirectory)) { directoryOptions.SourceDirectory = @"C:\FileWatcher\SourceDirectory"; MakeValidDir(directoryOptions.SourceDirectory); logger.Log("Using default directory. Error in creating sourseDirectory ", true); } if (!MakeValidDir(directoryOptions.TargetDirectory)) { directoryOptions.TargetDirectory = @"C:\FileWatcher\TargetDirectory"; MakeValidDir(directoryOptions.TargetDirectory); logger.Log("Using default directory. Error in creating targetDirectory", true); } if (!MakeValidDir(directoryOptions.ArchiveDirectory)) { directoryOptions.ArchiveDirectory = @"C:\FileWatcher\TargetDirectory\archive"; MakeValidDir(directoryOptions.ArchiveDirectory); logger.Log("The access to archive directory is denied, using default directory.", true); } if (!MakeValidFile(directoryOptions.LogFile)) { directoryOptions.LogFile = @"C:\FileWtcher\TargetDirectory\Logfile.txt"; MakeValidFile(directoryOptions.LogFile); logger.Log("The access to log file is denied, using default log file.", true); } ArchiveOptions archivationOptions = options.ArchiveOptions; if ((int)archivationOptions.CompressionLevel < 0 || (int)archivationOptions.CompressionLevel > 2) { archivationOptions.CompressionLevel = System.IO.Compression.CompressionLevel.Optimal; logger.Log("Incorrect value of compression level. Default value is set.", true); } }
private PerfionClientImpl CreateSubject(ArchiveOptions globalArchiveOptions = null) { _archiveService = Substitute.For <IArchiveService>(); _kernel = Substitute.For <IKernel>(); _kernel.Resolve <IArchiveService>().Returns(_archiveService); _proxy = Substitute.For <GetDataSoap>(); _connection = new ConnectionStub(_proxy); _configuration = new ConfigurationStub(globalArchiveOptions); var beginArchive = new BeginArchive("name", (options, stream) => { }); _archiveService.Create(Arg.Any <string>(), Arg.Any <Action <ArchiveCreated> >()).Returns(beginArchive); var subject = new PerfionClientImpl(_connection, _configuration, _kernel); return(subject); }
public string[] Upload(IEnumerable <FtpFile> files, Action <ArchiveCreated> onArchived = null) { if (files == null) { throw new ArgumentNullException(nameof(files)); } ArchiveOptions archiveOptions = _configuration.ArchiveOptions; if (archiveOptions != null) { var archive = _archive.Archive(archiveOptions.Name, newArchive => { newArchive.Options.GroupedBy(archiveOptions.GroupName); if (archiveOptions.Expires.HasValue) { newArchive.Options.ExpiresOn(archiveOptions.Expires.Value); } foreach (FtpFile file in files) { newArchive.IncludeContent(file.FileName, file.Contents); } }); onArchived?.Invoke(archive); } IFtpClient ftpClient = _ftpClientFactory.Create(_configuration.FtpConnectionString); var encoding = new UTF8Encoding(false /* to ensure that Globase can read the file, this parameter is necessary */); return(files .Select(file => ftpClient.UploadFromString(file.FileName, file.Contents, encoding)) .ToArray()); }
public static IEnumerable <string> Archive(string treeish, string output, ArchiveOptions options = default) { if (string.IsNullOrWhiteSpace(treeish)) { throw new ArgumentNullException(nameof(treeish)); } if (string.IsNullOrWhiteSpace(output)) { throw new ArgumentNullException(nameof(output)); } yield return("archive"); if (!string.IsNullOrEmpty(options.Format)) { yield return($"--format={QuoteEscape(options.Format)}"); } if (!string.IsNullOrEmpty(options.Prefix)) { yield return($"--prefix={QuoteEscape(options.Prefix)}"); } yield return($"--output={QuoteEscape(output)}"); if (options.WorktreeAttributes.GetValueOrDefault()) { yield return("--worktree-attributes"); } if (options.Extra != null) { foreach (var extraArg in options.Extra) { yield return($"-{extraArg}"); } } if (!string.IsNullOrEmpty(options.Remote)) { yield return($"--remote={QuoteEscape(options.Remote)}"); } if (!string.IsNullOrEmpty(options.Exec)) { yield return($"--exec={QuoteEscape(options.Exec)}"); } yield return(treeish); if (options.Paths != null) { yield return("--"); foreach (var path in options.Paths.Select(QuoteEscape)) { yield return(path); } } }
private static QueryArchiveOptions GetLocalArchiveOptions(Action <QueryArchiveOptions> archive, ArchiveOptions archiveOptions) { if (archive == null) { return(null); } var result = new QueryArchiveOptions(archiveOptions?.Name ?? "Data"); result.GroupedBy(archiveOptions?.GroupName ?? "Perfion"); if (archiveOptions != null) { if (archiveOptions.Expires.HasValue) { result.ExpiresOn(archiveOptions.Expires.Value); } } else { result.ExpiresAfterMonths(1); } archive(result); return(result); }
/// <summary> /// /// </summary> /// <param name="archiveOptionConfigFileName"></param> public Collector(string archiveOptionConfigFileName) { ArchOptions = ArchiveOptions.ReadParameters(archiveOptionConfigFileName); NetComm = new NetworkCommunicator(); }
public static void CreateArchive(ArchiveOptions archiveOptions, ProcessingEntry processingEntry, CancellationTokenEx cancellationToken, ReportCompressionStatus reportCompressionStatus) { var folderName = new Lazy <string>(() => { var firstEntry = archiveOptions.Entries[0]; return((firstEntry.IsDirectory ? new DirectoryInfo(firstEntry.Path).Parent.FullName : new FileInfo(firstEntry.Path).DirectoryName).TrimEnd('\\')); }); Stream outputStream = new FileStream(archiveOptions.ArchivePath, FileMode.Create, FileAccess.ReadWrite); switch (archiveOptions.CompressionMethod) { case CompressionMethod.None: //dont wrap the stream break; case CompressionMethod.Zip: using (var zipStream = new ZipOutputStream(outputStream) { IsStreamOwner = true }) { zipStream.SetLevel(archiveOptions.CompressionLevel); zipStream.Password = archiveOptions.Password; var folderOffset = folderName.Value.Length; var fileList = new List <FileInfo>(); foreach (var entry in archiveOptions.Entries) { if (entry.IsDirectory) { CollectFiles(fileList, new DirectoryInfo(entry.Path)); } else { fileList.Add(new FileInfo(entry.Path)); } } double totalLength = fileList.Sum(x => x.Length); long currentLength = 0; var updateStopwatch = Stopwatch.StartNew(); void UpdateProgress(float progress) { //important for a lot of small files if (updateStopwatch.ElapsedMilliseconds > 1000) { updateStopwatch.Reset(); processingEntry.Progress = progress; processingEntry.Size = zipStream.Length; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); updateStopwatch.Start(); } } foreach (var fileInfo in fileList) { var entryName = ZipEntry.CleanName(fileInfo.FullName.Substring(folderOffset)); var zipEntry = new ZipEntry(entryName) { DateTime = fileInfo.LastWriteTime, AESKeySize = string.IsNullOrEmpty(archiveOptions.Password) ? 0 : 256, Size = fileInfo.Length }; byte[] buffer = new byte[4096]; FileStream zipEntryStream; try { zipEntryStream = fileInfo.OpenRead(); } catch (Exception) { continue; //access denied } zipStream.PutNextEntry(zipEntry); using (zipEntryStream) { StreamUtils.Copy(zipEntryStream, zipStream, buffer, (sender, args) => { UpdateProgress((float)((currentLength + args.Processed) / totalLength)); args.ContinueRunning = !cancellationToken.IsCanceled; }, TimeSpan.FromSeconds(1), null, null); } if (cancellationToken.IsCanceled) { //force update processingEntry.Progress = -1; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); return; } currentLength += fileInfo.Length; zipStream.CloseEntry(); UpdateProgress((float)(currentLength / totalLength)); } //force update processingEntry.Size = zipStream.Length; processingEntry.Progress = 1; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); } return; case CompressionMethod.Gzip: var gzipStream = new GZipOutputStream(outputStream) { IsStreamOwner = true }; gzipStream.SetLevel(archiveOptions.CompressionLevel); gzipStream.Password = archiveOptions.Password; outputStream = gzipStream; break; case CompressionMethod.Bzip2: outputStream = new BZip2OutputStream(outputStream) { IsStreamOwner = true }; break; default: throw new ArgumentException("Unknown compression method: " + archiveOptions.CompressionMethod); } using (outputStream) { if (archiveOptions.UseTarPacker) { using (var tarOutputStream = new TarOutputStream(outputStream)) { var rootPath = Path.GetDirectoryName(archiveOptions.Entries[0].Path).Replace('\\', '/').TrimEnd('/'); var fileList = new List <FileInfo>(); foreach (var entry in archiveOptions.Entries) { if (entry.IsDirectory) { CollectFiles(fileList, new DirectoryInfo(entry.Path)); } else { fileList.Add(new FileInfo(entry.Path)); } } var buffer = new byte[4096]; double totalLength = fileList.Sum(x => x.Length); long currentLength = 0; var updateStopwatch = Stopwatch.StartNew(); foreach (var fileInfo in fileList) { Stream fileStream; try { fileStream = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read); } catch (Exception) { continue; } using (fileStream) { var tarEntry = TarEntry.CreateEntryFromFile(fileInfo.FullName); tarEntry.Name = fileInfo.FullName.Substring(rootPath.Length + 1); tarOutputStream.PutNextEntry(tarEntry); StreamUtils.Copy(fileStream, tarOutputStream, buffer, (sender, args) => { args.ContinueRunning = !cancellationToken.IsCanceled; if (updateStopwatch.ElapsedMilliseconds > 1000) { updateStopwatch.Reset(); processingEntry.Progress = (float)((currentLength + args.Processed) / totalLength); processingEntry.Size = tarOutputStream.Length; ThreadPool.QueueUserWorkItem( state => reportCompressionStatus.Invoke(processingEntry)); updateStopwatch.Start(); } }, TimeSpan.FromSeconds(1), null, null); tarOutputStream.CloseEntry(); } currentLength += fileInfo.Length; if (cancellationToken.IsCanceled) { processingEntry.Progress = -1; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); return; } } } } else { var entry = archiveOptions.Entries[0]; if (entry.IsDirectory) { throw new ArgumentException("Cannot pack directory without tar/zip"); } byte[] dataBuffer = new byte[4096]; using (var sourceStream = new FileStream(entry.Path, FileMode.Open, FileAccess.Read)) StreamUtils.Copy(sourceStream, outputStream, dataBuffer, (sender, args) => { //no stopwatch needed because it is only one entry processingEntry.Progress = args.PercentComplete / 100; processingEntry.Size = outputStream.Length; args.ContinueRunning = !cancellationToken.IsCanceled; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); }, TimeSpan.FromSeconds(1), null, null); if (cancellationToken.IsCanceled) { //force update processingEntry.Progress = -1; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); return; } } processingEntry.Size = outputStream.Length; processingEntry.Progress = 1; ThreadPool.QueueUserWorkItem(state => reportCompressionStatus.Invoke(processingEntry)); } }
private void SaveSettings() { if (EnableWindowsIntegration) { ShellRegister("Directory"); ShellRegister("Drive"); ShellRegister("*"); ShellRegister("here"); } else { ShellUnregister("Directory"); ShellUnregister("Drive"); ShellUnregister("*"); ShellUnregister("here"); } if (EnableStartupAcceleration) { StartupRegister(); } else { StartupUnregister(); } ApplicationFontFamily = EditApplicationFontFamily; MainFormFontSize = EditMainFormFontSize; ReplaceFormFontSize = EditReplaceFormFontSize; DialogFontSize = EditDialogFontSize; Settings.Set(GrepSettings.Key.EnableUpdateChecking, EnableCheckForUpdates); Settings.Set(GrepSettings.Key.UpdateCheckInterval, CheckForUpdatesInterval); Settings.Set(GrepSettings.Key.CustomEditor, CustomEditorPath); Settings.Set(GrepSettings.Key.CustomEditorArgs, CustomEditorArgs); Settings.Set(GrepSettings.Key.CompareApplication, CompareApplicationPath); Settings.Set(GrepSettings.Key.CompareApplicationArgs, CompareApplicationArgs); Settings.Set(GrepSettings.Key.ShowFilePathInResults, ShowFilePathInResults); Settings.Set(GrepSettings.Key.AllowSearchingForFileNamePattern, AllowSearchWithEmptyPattern); Settings.Set(GrepSettings.Key.DetectEncodingForFileNamePattern, DetectEncodingForFileNamePattern); Settings.Set(GrepSettings.Key.ExpandResults, AutoExpandSearchTree); Settings.Set(GrepSettings.Key.ShowVerboseMatchCount, ShowVerboseMatchCount); Settings.Set(GrepSettings.Key.ShowFileInfoTooltips, ShowFileInfoTooltips); Settings.Set(GrepSettings.Key.MatchTimeout, MatchTimeout); Settings.Set(GrepSettings.Key.FuzzyMatchThreshold, MatchThreshold); Settings.Set(GrepSettings.Key.ShowLinesInContext, ShowLinesInContext); Settings.Set(GrepSettings.Key.ContextLinesBefore, ContextLinesBefore); Settings.Set(GrepSettings.Key.ContextLinesAfter, ContextLinesAfter); Settings.Set(GrepSettings.Key.MaxSearchBookmarks, MaxSearchBookmarks); Settings.Set(GrepSettings.Key.MaxPathBookmarks, MaxPathBookmarks); Settings.Set(GrepSettings.Key.MaxExtensionBookmarks, MaxExtensionBookmarks); Settings.Set(GrepSettings.Key.OptionsOnMainPanel, OptionsLocation == PanelSelection.MainPanel); Settings.Set(GrepSettings.Key.FollowWindowsTheme, FollowWindowsTheme); Settings.Set(GrepSettings.Key.CurrentTheme, CurrentTheme); Settings.Set(GrepSettings.Key.UseDefaultFont, UseDefaultFont); Settings.Set(GrepSettings.Key.ApplicationFontFamily, ApplicationFontFamily); Settings.Set(GrepSettings.Key.MainFormFontSize, MainFormFontSize); Settings.Set(GrepSettings.Key.ReplaceFormFontSize, ReplaceFormFontSize); Settings.Set(GrepSettings.Key.DialogFontSize, DialogFontSize); Settings.Set(GrepSettings.Key.PdfToTextOptions, PdfToTextOptions); if (ArchiveOptions.IsChanged) { string nameKey = "Archive"; string addKey = "Add" + nameKey + "Extensions"; string remKey = "Rem" + nameKey + "Extensions"; Settings.Set(addKey, CleanExtensions(ArchiveOptions.AddExtensions)); Settings.Set(remKey, CleanExtensions(ArchiveOptions.RemExtensions)); ArchiveOptions.SetUnchanged(); ArchiveDirectory.Reinitialize(); } bool pluginsChanged = Plugins.Any(p => p.IsChanged); foreach (var plugin in Plugins) { string nameKey = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(plugin.Name); string enabledkey = nameKey + "Enabled"; string addKey = "Add" + nameKey + "Extensions"; string remKey = "Rem" + nameKey + "Extensions"; Settings.Set(enabledkey, plugin.IsEnabled); Settings.Set(addKey, CleanExtensions(plugin.AddExtensions)); Settings.Set(remKey, CleanExtensions(plugin.RemExtensions)); plugin.SetUnchanged(); } Settings.Save(); if (pluginsChanged) { GrepEngineFactory.ReloadPlugins(); } }
public ConfigurationStub(ArchiveOptions globalArchiveOptions) { ArchiveOptions = globalArchiveOptions; }
public void CreateArchive(ArchiveOptions archiveOptions) { _dtpFactory.ExecuteProcedure("CreateArchive", archiveOptions); }