public Attempt <int> Deploy(string folder, XElement configNode, SyncUpdateCallback callback) { var config = LoadSettings(configNode); try { var cleanFolder = $"{folder.Replace("/", "\\")}\\".Replace("\\\\", "\\"); var connectionInfo = new ConnectionInfo(config.Server, config.Username, new PasswordAuthenticationMethod(config.Username, config.Password), new PrivateKeyAuthenticationMethod("rsa.key")); int count = 0; using (var client = new SftpClient(connectionInfo)) { callback?.Invoke("Connecting to Server", 1, 100); client.Connect(); count = UploadFolder(config.Folder, cleanFolder, "", client, callback); callback?.Invoke("Complete", 100, 100); client.Disconnect(); } return(Attempt.Succeed(count)); } catch (Exception ex) { return(Attempt.Fail(0, ex)); } }
virtual public IEnumerable <uSyncAction> ExportAll(int parent, string folder, HandlerSettings config, SyncUpdateCallback callback) { int count = 0; var actions = new List <uSyncAction>(); if (itemContainerType != UmbracoObjectTypes.Unknown) { var containers = entityService.GetChildren(parent, this.itemContainerType); foreach (var container in containers) { actions.AddRange(ExportAll(container.Id, folder, config, callback)); } } var items = GetExportItems(parent, itemObjectType).ToList(); foreach (var item in items) { count++; var concreateType = GetFromService(item.Id); callback?.Invoke(GetItemName(concreateType), count, items.Count); actions.Add(Export(concreateType, folder, config)); actions.AddRange(ExportAll(item.Id, folder, config, callback)); } callback?.Invoke("Done", 1, 1); return(actions); }
protected virtual IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback) { List <uSyncAction> actions = new List <uSyncAction>(); var files = syncFileService.GetFiles(folder, "*.config"); var flags = SerializerFlags.None; if (force) { flags |= SerializerFlags.Force; } if (config.BatchSave) { flags |= SerializerFlags.DoNotSave; } int count = 0; int total = files.Count(); foreach (string file in files) { count++; callback?.Invoke($"Importing {Path.GetFileNameWithoutExtension(file)}", count, total); var attempt = Import(file, config, flags); if (attempt.Success && attempt.Item != null) { updates.Add(file, attempt.Item); } var action = uSyncActionHelper <TObject> .SetAction(attempt, file, IsTwoPass); if (attempt.Details != null && attempt.Details.Any()) { action.Details = attempt.Details; } actions.Add(action); } // bulk save .. if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) { callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); serializer.Save(updates.Select(x => x.Value)); } var folders = syncFileService.GetDirectories(folder); foreach (var children in folders) { actions.AddRange(ImportFolder(children, config, updates, force, callback)); } callback?.Invoke("", 1, 1); return(actions); }
public IEnumerable <uSyncAction> Report(string folder, HandlerSettings config, SyncUpdateCallback callback) { var actions = new List <uSyncAction>(); callback?.Invoke("Checking Actions", 0, 1); actions.AddRange(ReportFolder(folder, config, callback)); callback?.Invoke("Done", 1, 1); return(actions); }
private void ProcessSecondPasses(IDictionary <string, TObject> updates, List <uSyncAction> actions, HandlerSettings config, SyncUpdateCallback callback = null) { List <TObject> updatedItems = new List <TObject>(); foreach (var item in updates.Select((update, Index) => new { update, Index })) { callback?.Invoke($"Second Pass {Path.GetFileName(item.update.Key)}", item.Index, updates.Count); var attempt = ImportSecondPass(item.update.Key, item.update.Value, config, callback); if (attempt.Success) { // if the second attempt has a message on it, add it to the first attempt. if (!string.IsNullOrWhiteSpace(attempt.Message)) { if (actions.Any(x => x.FileName == item.update.Key)) { var action = actions.FirstOrDefault(x => x.FileName == item.update.Key); actions.Remove(action); action.Message += attempt.Message; actions.Add(action); } } if (attempt.Change > ChangeType.NoChange) { updatedItems.Add(attempt.Item); } } else { // the second attempt failed - update the action. if (actions.Any(x => x.FileName == item.update.Key)) { var action = actions.FirstOrDefault(x => x.FileName == item.update.Key); actions.Remove(action); action.Success = attempt.Success; action.Message = $"Second Pass Fail: {attempt.Message}"; action.Exception = attempt.Exception; actions.Add(action); } } } if (config.BatchSave) { callback?.Invoke($"Saving {updatedItems.Count} Second Pass Items", 2, 3); serializer.Save(updatedItems); } }
public IEnumerable <uSyncAction> ReportFolder(string folder, HandlerSettings config, SyncUpdateCallback callback) { List <uSyncAction> actions = new List <uSyncAction>(); var files = syncFileService.GetFiles(folder, "*.config"); int count = 0; int total = files.Count(); foreach (string file in files) { count++; callback?.Invoke(Path.GetFileNameWithoutExtension(file), count, total); actions.Add(ReportItem(file)); } foreach (var children in syncFileService.GetDirectories(folder)) { actions.AddRange(ReportFolder(children, config, callback)); } return(actions); }
/// <summary> /// don't think you can get dictionary items via the entity service :( /// </summary> public IEnumerable <uSyncAction> ExportAll(Guid parent, string folder, HandlerSettings config, SyncUpdateCallback callback) { var actions = new List <uSyncAction>(); var items = new List <IDictionaryItem>(); if (parent == Guid.Empty) { items = localizationService.GetRootDictionaryItems().ToList(); } else { items = localizationService.GetDictionaryItemChildren(parent).ToList(); } int count = 0; foreach (var item in items) { count++; callback?.Invoke(item.ItemKey, count, items.Count); actions.AddRange(Export(item, folder, config)); actions.AddRange(ExportAll(item.Key, folder, config, callback)); } return(actions); }
public IEnumerable <uSyncAction> ImportAll(string folder, HandlerSettings config, bool force, SyncUpdateCallback callback = null) { var sw = Stopwatch.StartNew(); logger.Debug(handlerType, "{alias} ImportAll: {fileName}", this.Alias, Path.GetFileName(folder)); var actions = new List <uSyncAction>(); var updates = new Dictionary <string, TObject>(); runtimeCache.ClearByKey($"keycache_{this.Alias}"); actions.AddRange(ImportFolder(folder, config, updates, force, callback)); if (updates.Any()) { ProcessSecondPasses(updates, actions, config, callback); } runtimeCache.ClearByKey($"keycache_{this.Alias}"); callback?.Invoke("Done", 3, 3); sw.Stop(); logger.Debug(handlerType, "{alias} Import Complete {elapsedMilliseconds}ms", this.Alias, sw.ElapsedMilliseconds); return(actions); }
private int UploadFolder(string serverRoot, string localRoot, string folder, FtpClient client, SyncUpdateCallback update) { var serverPath = $"{serverRoot}/{folder}".Replace("\\", "/"); if (!client.DirectoryExists(serverPath)) { client.CreateDirectory(serverPath); } var localPath = Path.Combine(localRoot, folder); var localDir = new DirectoryInfo(localPath); var files = localDir.GetFiles(); // update?.Invoke($"Updating Folder {Path.GetFileName(folder)}", 5, 10); // var count = client.UploadFiles(files, serverPath); int count = 0; foreach (var file in files) { update?.Invoke($"Uploading {folder}/{file.Name}", 5, 10); client.UploadFile(file.FullName, serverPath + "/" + file.Name); count++; } foreach (var childFolder in localDir.GetDirectories()) { var relativeFolder = childFolder.FullName.Substring(localRoot.Length + 1); count += UploadFolder(serverRoot, localRoot, relativeFolder, client, update); } return(count); }
public Attempt <int> Deploy(string folder, XElement config, SyncUpdateCallback update) { var settings = LoadSettings(config); update?.Invoke($"Copying site to {Path.GetFileName(folder)}", 1, 2); Directory.CreateDirectory(settings.Folder); templateFileService.CopyFolder(folder, settings.Folder); return(Attempt.Succeed(1)); }
/// <inheritdoc/> public override IEnumerable <uSyncAction> ExportAll(string folder, HandlerSettings config, SyncUpdateCallback callback) { var actions = new List <uSyncAction>(); var domains = domainService.GetAll(true).ToList(); int count = 0; foreach (var domain in domains) { count++; if (domain != null) { callback?.Invoke(domain.DomainName, count, domains.Count); actions.AddRange(Export(domain, folder, config)); } } callback?.Invoke("done", 1, 1); return(actions); }
/// <inheritdoc/> public override IEnumerable <uSyncAction> ExportAll(string folder, HandlerSettings config, SyncUpdateCallback callback) { var actions = new List <uSyncAction>(); var items = relationService.GetAllRelationTypes().ToList(); foreach (var item in items.Select((relationType, index) => new { relationType, index })) { callback?.Invoke(item.relationType.Name, item.index, items.Count); actions.AddRange(Export(item.relationType, folder, config)); } return(actions); }
public override IEnumerable <uSyncAction> ExportAll(string folder, HandlerSettings config, SyncUpdateCallback callback) { var items = syncFormService.GetAllForms(); var actions = new List <uSyncAction>(); foreach (var item in items) { callback?.Invoke(GetItemName(item), 2, 4); actions.AddRange(Export(item, folder, config)); } return(actions); }
public IEnumerable <uSyncAction> ImportAll(string folder, HandlerSettings config, bool force, SyncUpdateCallback callback = null) { logger.Info <uSync8BackOffice>("Running Import: {0}", Path.GetFileName(folder)); var actions = new List <uSyncAction>(); var updates = new Dictionary <string, TObject>(); actions.AddRange(ImportFolder(folder, config, updates, force, callback)); if (updates.Any()) { int count = 0; foreach (var update in updates) { count++; callback?.Invoke($"Second Pass {Path.GetFileName(update.Key)}", count, updates.Count); ImportSecondPass(update.Key, update.Value, config, callback); } } callback?.Invoke("Done", 1, 1); return(actions); }
/// <summary> /// overrider the default export, because macros, don't exist as an object type??? /// </summary> public override IEnumerable <uSyncAction> ExportAll(int parent, string folder, HandlerSettings config, SyncUpdateCallback callback) { // we clean the folder out on an export all. syncFileService.CleanFolder(folder); var actions = new List <uSyncAction>(); var items = macroService.GetAll().ToList(); int count = 0; foreach (var item in items) { count++; callback?.Invoke(item.Name, count, items.Count); actions.AddRange(Export(item, folder, config)); } return(actions); }
public Attempt <int> Deploy(string folder, XElement config, SyncUpdateCallback update) { var settings = LoadSettings(config); var client = new FtpClient(settings.Server); client.Credentials = new NetworkCredential(settings.Username, settings.Password); client.EncryptionMode = FtpEncryptionMode.Explicit; client.SslProtocols = SslProtocols.Default | SslProtocols.Tls11 | SslProtocols.Tls12; client.ValidateCertificate += new FtpSslValidation(OnValidateCertificate); update?.Invoke("connecting to server", 1, 100); client.Connect(); var count = UploadFolder(settings.Folder, folder, "", client, update); client.Disconnect(); return(Attempt.Succeed(count)); }
private int UploadFolder(string siteRoot, string localRoot, string folder, SftpClient client, SyncUpdateCallback callback) { if (!client.IsConnected) { throw new FtpException("Not connected"); } var path = Path.Combine(localRoot, folder); if (!Directory.Exists(path)) { throw new DirectoryNotFoundException("Path not found: " + path); } var dest = $"/{siteRoot}/{folder}"; try { client.CreateDirectory(dest); } catch { } callback?.Invoke($"Syncing {folder}", 5, 10); var files = client.SynchronizeDirectories(path, dest, "*.*"); var total = files.Count(); foreach (var child in Directory.GetDirectories(path)) { var childFolder = child.Substring(localRoot.Length) .Replace("\\", "/"); total += UploadFolder(siteRoot, localRoot, childFolder, client, callback); } return(total); }
/// <summary> /// this is the simple interface, based purely on level, /// we could get clever (like dependency trees for content types) /// /// but that would have to be implimented lower down (and it doesn't /// really matter for things in containers only things that parent others). /// </summary> /// <param name="folder"></param> /// <param name="force"></param> /// <param name="updates"></param> /// <returns></returns> protected override IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback) { // if not using flat, then directory structure is doing // this for us. if (config.UseFlatStructure == false) { return(base.ImportFolder(folder, config, updates, force, callback)); } List <uSyncAction> actions = new List <uSyncAction>(); var files = syncFileService.GetFiles(folder, "*.config"); List <LeveledFile> nodes = new List <LeveledFile>(); callback?.Invoke("Calculating import order", 0, 1); logger.Verbose(handlerType, "Calculating import order"); foreach (var file in files) { try { var node = LoadNode(file); if (node != null) { nodes.Add(new LeveledFile { Level = node.GetLevel(), File = file }); } } catch (XmlException ex) { // one of the files is wrong. (do we stop or carry on) logger.Warn(handlerType, $"Error loading file: {file} [{ex.Message}]"); actions.Add(uSyncActionHelper <TObject> .SetAction( SyncAttempt <TObject> .Fail(Path.GetFileName(file), ChangeType.Fail, $"Failed to Load: {ex.Message}"), file, false)); } } // loaded - now process. var flags = SerializerFlags.None; if (force) { flags |= SerializerFlags.Force; } if (config.BatchSave) { flags |= SerializerFlags.DoNotSave; } var cleanMarkers = new List <string>(); foreach (var item in nodes.OrderBy(x => x.Level).Select((Node, Index) => new { Node, Index })) { var filename = Path.GetFileNameWithoutExtension(item.Node.File); callback?.Invoke($"{filename}", item.Index, nodes.Count); logger.Verbose(handlerType, "{Index} Importing: {File}, [Level {Level}]", item.Index, filename, item.Node.Level); var attempt = Import(item.Node.File, config, flags); if (attempt.Success) { if (attempt.Change == ChangeType.Clean) { cleanMarkers.Add(item.Node.File); } else if (attempt.Item != null) { updates.Add(item.Node.File, attempt.Item); } } actions.Add(uSyncActionHelper <TObject> .SetAction(attempt, item.Node.File, IsTwoPass)); } if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) { // bulk save - should be the fastest way to do this callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); serializer.Save(updates.Select(x => x.Value)); } var folders = syncFileService.GetDirectories(folder); foreach (var children in folders) { actions.AddRange(ImportFolder(children, config, updates, force, callback)); } if (actions.All(x => x.Success)) { // LINQ // actions.AddRange(cleanMarkers.Select(x => CleanFolder(x)).SelectMany(a => a)); // only if there are no fails. // then we consider the folder safe to clean foreach (var cleanfile in cleanMarkers) { actions.AddRange(CleanFolder(cleanfile, false, config.UseFlatStructure)); } // remove the actual cleans (they will have been replaced by the deletes actions.RemoveAll(x => x.Change == ChangeType.Clean); } callback?.Invoke("", 1, 1); return(actions); }
protected virtual IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback) { List <uSyncAction> actions = new List <uSyncAction>(); var files = GetImportFiles(folder); var flags = SerializerFlags.None; if (force) { flags |= SerializerFlags.Force; } if (config.BatchSave) { flags |= SerializerFlags.DoNotSave; } var cleanMarkers = new List <string>(); int count = 0; int total = files.Count(); foreach (string file in files) { count++; callback?.Invoke($"Importing {Path.GetFileNameWithoutExtension(file)}", count, total); var attempt = Import(file, config, flags); if (attempt.Success) { if (attempt.Change == ChangeType.Clean) { cleanMarkers.Add(file); } else if (attempt.Item != null) { updates.Add(file, attempt.Item); } } var action = uSyncActionHelper <TObject> .SetAction(attempt, file, this.Alias, IsTwoPass); if (attempt.Details != null && attempt.Details.Any()) { action.Details = attempt.Details; } if (attempt.Change != ChangeType.Clean) { actions.Add(action); } } // bulk save .. if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) { // callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); serializer.Save(updates.Select(x => x.Value)); } var folders = syncFileService.GetDirectories(folder); foreach (var children in folders) { actions.AddRange(ImportFolder(children, config, updates, force, callback)); } if (actions.All(x => x.Success) && cleanMarkers.Count > 0) { // this is just extra messaging, given how quickly the next message will be sent. // callback?.Invoke("Cleaning Folders", 1, cleanMarkers.Count); foreach (var item in cleanMarkers.Select((filePath, Index) => new { filePath, Index })) { var folderName = Path.GetFileName(item.filePath); callback?.Invoke($"Cleaning {folderName}", item.Index, cleanMarkers.Count); var cleanActions = CleanFolder(item.filePath, false, config.UseFlatStructure); if (cleanActions.Any()) { actions.AddRange(cleanActions); } else { // nothing to delete, we report this as a no change actions.Add(uSyncAction.SetAction(true, $"Folder {Path.GetFileName(item.filePath)}", change: ChangeType.NoChange, filename: item.filePath)); } } // remove the actual cleans (they will have been replaced by the deletes actions.RemoveAll(x => x.Change == ChangeType.Clean); } return(actions); }
/// <inheritdoc/> protected override IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback) { // if not using flat then directory structure is sorting them for us. if (config.UseFlatStructure == false) { return(base.ImportFolder(folder, config, updates, force, callback)); } List <uSyncAction> actions = new List <uSyncAction>(); callback?.Invoke("Calculating import order", 0, 1); logger.LogDebug("Calculating import order"); var orderedFiles = GetLevelOrderedFiles(folder, actions); // process. var flags = SerializerFlags.None; if (force) { flags |= SerializerFlags.Force; } var cleanMarkers = new List <string>(); foreach (var item in orderedFiles.Select((Node, Index) => new { Node, Index })) { var filename = Path.GetFileNameWithoutExtension(item.Node.File); callback?.Invoke($"{filename}", item.Index, orderedFiles.Count); logger.LogTrace("{Index} Importing: {File}, [Level {Level}]", item.Index, filename, item.Node.Level); var result = Import(item.Node.File, config, flags); foreach (var attempt in result) { if (attempt.Success) { if (attempt.Change == ChangeType.Clean) { cleanMarkers.Add(item.Node.File); } else if (attempt.Item != null && attempt.Item is TObject attemptItem) { updates.Add(item.Node.File, attemptItem); } } if (attempt.Change != ChangeType.Clean) { actions.Add(attempt); } } } if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) { // bulk save - should be the fastest way to do this callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); serializer.Save(updates.Select(x => x.Value)); } var folders = syncFileService.GetDirectories(folder); foreach (var children in folders) { actions.AddRange(ImportFolder(children, config, updates, force, callback)); } if (actions.All(x => x.Success)) { // LINQ // actions.AddRange(cleanMarkers.Select(x => CleanFolder(x)).SelectMany(a => a)); // only if there are no fails. // then we consider the folder safe to clean foreach (var cleanfile in cleanMarkers) { actions.AddRange(CleanFolder(cleanfile, false, config.UseFlatStructure)); } // remove the actual cleans (they will have been replaced by the deletes actions.RemoveAll(x => x.Change == ChangeType.Clean); } callback?.Invoke("", 1, 1); return(actions); }
/// <summary> /// this is the simple interface, based purely on level, /// we could get clever (like dependency trees for content types) /// /// but that would have to be implimented lower down (and it doesn't /// really matter for things in containers only things that parent others). /// </summary> /// <param name="folder"></param> /// <param name="force"></param> /// <param name="updates"></param> /// <returns></returns> protected override IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback) { // if not using flat, then directory structure is doing // this for us. if (config.UseFlatStructure == false) { return(base.ImportFolder(folder, config, updates, force, callback)); } List <uSyncAction> actions = new List <uSyncAction>(); var files = syncFileService.GetFiles(folder, "*.config"); List <LeveledFile> nodes = new List <LeveledFile>(); callback?.Invoke("Calculating import order", 0, 1); foreach (var file in files) { try { var node = LoadNode(file); if (node != null) { nodes.Add(new LeveledFile { Level = node.GetLevel(), File = file }); } } catch (XmlException ex) { // one of the files is wrong. (do we stop or carry on) logger.Warn <TObject>($"Error loading file: {file} [{ex.Message}]"); actions.Add(uSyncActionHelper <TObject> .SetAction( SyncAttempt <TObject> .Fail(Path.GetFileName(file), ChangeType.Fail, $"Failed to Load: {ex.Message}"), file, false)); } } // loaded - now process. var flags = SerializerFlags.None; if (force) { flags |= SerializerFlags.Force; } if (config.BatchSave) { flags |= SerializerFlags.DoNotSave; } var count = 0; foreach (var node in nodes.OrderBy(x => x.Level)) { count++; callback?.Invoke($"{Path.GetFileName(node.File)}", count, nodes.Count); var attempt = Import(node.File, config, flags); if (attempt.Success && attempt.Item != null) { updates.Add(node.File, attempt.Item); } actions.Add(uSyncActionHelper <TObject> .SetAction(attempt, node.File, IsTwoPass)); } if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) { // bulk save - should be the fastest way to do this callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); serializer.Save(updates.Select(x => x.Value)); } var folders = syncFileService.GetDirectories(folder); foreach (var children in folders) { actions.AddRange(ImportFolder(children, config, updates, force, callback)); } callback?.Invoke("", 1, 1); return(actions); }