public Attempt <int> Deploy(string folder, XElement configNode, SyncUpdateCallback callback)
{
var config = LoadSettings(configNode);
try
{
var cleanFolder = $"{folder.Replace("/", "\\")}\\".Replace("\\\\", "\\");
var connectionInfo = new ConnectionInfo(config.Server,
config.Username,
new PasswordAuthenticationMethod(config.Username, config.Password),
new PrivateKeyAuthenticationMethod("rsa.key"));
int count = 0;
using (var client = new SftpClient(connectionInfo))
{
callback?.Invoke("Connecting to Server", 1, 100);
client.Connect();
count = UploadFolder(config.Folder, cleanFolder, "", client, callback);
callback?.Invoke("Complete", 100, 100);
client.Disconnect();
}
return(Attempt.Succeed(count));
}
catch (Exception ex)
{
return(Attempt.Fail(0, ex));
}
}
virtual public IEnumerable <uSyncAction> ExportAll(int parent, string folder, HandlerSettings config, SyncUpdateCallback callback)
{
int count = 0;
var actions = new List <uSyncAction>();
if (itemContainerType != UmbracoObjectTypes.Unknown)
{
var containers = entityService.GetChildren(parent, this.itemContainerType);
foreach (var container in containers)
{
actions.AddRange(ExportAll(container.Id, folder, config, callback));
}
}
var items = GetExportItems(parent, itemObjectType).ToList();
foreach (var item in items)
{
count++;
var concreateType = GetFromService(item.Id);
callback?.Invoke(GetItemName(concreateType), count, items.Count);
actions.Add(Export(concreateType, folder, config));
actions.AddRange(ExportAll(item.Id, folder, config, callback));
}
callback?.Invoke("Done", 1, 1);
return(actions);
}
protected virtual IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback)
{
List <uSyncAction> actions = new List <uSyncAction>();
var files = syncFileService.GetFiles(folder, "*.config");
var flags = SerializerFlags.None;
if (force)
{
flags |= SerializerFlags.Force;
}
if (config.BatchSave)
{
flags |= SerializerFlags.DoNotSave;
}
int count = 0;
int total = files.Count();
foreach (string file in files)
{
count++;
callback?.Invoke($"Importing {Path.GetFileNameWithoutExtension(file)}", count, total);
var attempt = Import(file, config, flags);
if (attempt.Success && attempt.Item != null)
{
updates.Add(file, attempt.Item);
}
var action = uSyncActionHelper <TObject> .SetAction(attempt, file, IsTwoPass);
if (attempt.Details != null && attempt.Details.Any())
{
action.Details = attempt.Details;
}
actions.Add(action);
}
// bulk save ..
if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}
var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
actions.AddRange(ImportFolder(children, config, updates, force, callback));
}
callback?.Invoke("", 1, 1);
return(actions);
}
public IEnumerable <uSyncAction> Report(string folder, HandlerSettings config, SyncUpdateCallback callback)
{
var actions = new List <uSyncAction>();
callback?.Invoke("Checking Actions", 0, 1);
actions.AddRange(ReportFolder(folder, config, callback));
callback?.Invoke("Done", 1, 1);
return(actions);
}
private void ProcessSecondPasses(IDictionary <string, TObject> updates, List <uSyncAction> actions, HandlerSettings config, SyncUpdateCallback callback = null)
{
List <TObject> updatedItems = new List <TObject>();
foreach (var item in updates.Select((update, Index) => new { update, Index }))
{
callback?.Invoke($"Second Pass {Path.GetFileName(item.update.Key)}", item.Index, updates.Count);
var attempt = ImportSecondPass(item.update.Key, item.update.Value, config, callback);
if (attempt.Success)
{
// if the second attempt has a message on it, add it to the first attempt.
if (!string.IsNullOrWhiteSpace(attempt.Message))
{
if (actions.Any(x => x.FileName == item.update.Key))
{
var action = actions.FirstOrDefault(x => x.FileName == item.update.Key);
actions.Remove(action);
action.Message += attempt.Message;
actions.Add(action);
}
}
if (attempt.Change > ChangeType.NoChange)
{
updatedItems.Add(attempt.Item);
}
}
else
{
// the second attempt failed - update the action.
if (actions.Any(x => x.FileName == item.update.Key))
{
var action = actions.FirstOrDefault(x => x.FileName == item.update.Key);
actions.Remove(action);
action.Success = attempt.Success;
action.Message = $"Second Pass Fail: {attempt.Message}";
action.Exception = attempt.Exception;
actions.Add(action);
}
}
}
if (config.BatchSave)
{
callback?.Invoke($"Saving {updatedItems.Count} Second Pass Items", 2, 3);
serializer.Save(updatedItems);
}
}
public IEnumerable <uSyncAction> ReportFolder(string folder, HandlerSettings config, SyncUpdateCallback callback)
{
List <uSyncAction> actions = new List <uSyncAction>();
var files = syncFileService.GetFiles(folder, "*.config");
int count = 0;
int total = files.Count();
foreach (string file in files)
{
count++;
callback?.Invoke(Path.GetFileNameWithoutExtension(file), count, total);
actions.Add(ReportItem(file));
}
foreach (var children in syncFileService.GetDirectories(folder))
{
actions.AddRange(ReportFolder(children, config, callback));
}
return(actions);
}
/// <summary>
/// don't think you can get dictionary items via the entity service :(
/// </summary>
public IEnumerable <uSyncAction> ExportAll(Guid parent, string folder, HandlerSettings config, SyncUpdateCallback callback)
{
var actions = new List <uSyncAction>();
var items = new List <IDictionaryItem>();
if (parent == Guid.Empty)
{
items = localizationService.GetRootDictionaryItems().ToList();
}
else
{
items = localizationService.GetDictionaryItemChildren(parent).ToList();
}
int count = 0;
foreach (var item in items)
{
count++;
callback?.Invoke(item.ItemKey, count, items.Count);
actions.AddRange(Export(item, folder, config));
actions.AddRange(ExportAll(item.Key, folder, config, callback));
}
return(actions);
}
public IEnumerable <uSyncAction> ImportAll(string folder, HandlerSettings config, bool force, SyncUpdateCallback callback = null)
{
var sw = Stopwatch.StartNew();
logger.Debug(handlerType, "{alias} ImportAll: {fileName}", this.Alias, Path.GetFileName(folder));
var actions = new List <uSyncAction>();
var updates = new Dictionary <string, TObject>();
runtimeCache.ClearByKey($"keycache_{this.Alias}");
actions.AddRange(ImportFolder(folder, config, updates, force, callback));
if (updates.Any())
{
ProcessSecondPasses(updates, actions, config, callback);
}
runtimeCache.ClearByKey($"keycache_{this.Alias}");
callback?.Invoke("Done", 3, 3);
sw.Stop();
logger.Debug(handlerType, "{alias} Import Complete {elapsedMilliseconds}ms", this.Alias, sw.ElapsedMilliseconds);
return(actions);
}
private int UploadFolder(string serverRoot, string localRoot, string folder, FtpClient client, SyncUpdateCallback update)
{
var serverPath = $"{serverRoot}/{folder}".Replace("\\", "/");
if (!client.DirectoryExists(serverPath))
{
client.CreateDirectory(serverPath);
}
var localPath = Path.Combine(localRoot, folder);
var localDir = new DirectoryInfo(localPath);
var files = localDir.GetFiles();
// update?.Invoke($"Updating Folder {Path.GetFileName(folder)}", 5, 10);
// var count = client.UploadFiles(files, serverPath);
int count = 0;
foreach (var file in files)
{
update?.Invoke($"Uploading {folder}/{file.Name}", 5, 10);
client.UploadFile(file.FullName, serverPath + "/" + file.Name);
count++;
}
foreach (var childFolder in localDir.GetDirectories())
{
var relativeFolder = childFolder.FullName.Substring(localRoot.Length + 1);
count += UploadFolder(serverRoot, localRoot, relativeFolder, client, update);
}
return(count);
}
public Attempt <int> Deploy(string folder, XElement config, SyncUpdateCallback update)
{
var settings = LoadSettings(config);
update?.Invoke($"Copying site to {Path.GetFileName(folder)}", 1, 2);
Directory.CreateDirectory(settings.Folder);
templateFileService.CopyFolder(folder, settings.Folder);
return(Attempt.Succeed(1));
}
/// <inheritdoc/>
public override IEnumerable <uSyncAction> ExportAll(string folder, HandlerSettings config, SyncUpdateCallback callback)
{
var actions = new List <uSyncAction>();
var domains = domainService.GetAll(true).ToList();
int count = 0;
foreach (var domain in domains)
{
count++;
if (domain != null)
{
callback?.Invoke(domain.DomainName, count, domains.Count);
actions.AddRange(Export(domain, folder, config));
}
}
callback?.Invoke("done", 1, 1);
return(actions);
}
/// <inheritdoc/>
public override IEnumerable <uSyncAction> ExportAll(string folder, HandlerSettings config, SyncUpdateCallback callback)
{
var actions = new List <uSyncAction>();
var items = relationService.GetAllRelationTypes().ToList();
foreach (var item in items.Select((relationType, index) => new { relationType, index }))
{
callback?.Invoke(item.relationType.Name, item.index, items.Count);
actions.AddRange(Export(item.relationType, folder, config));
}
return(actions);
}
public override IEnumerable <uSyncAction> ExportAll(string folder, HandlerSettings config, SyncUpdateCallback callback)
{
var items = syncFormService.GetAllForms();
var actions = new List <uSyncAction>();
foreach (var item in items)
{
callback?.Invoke(GetItemName(item), 2, 4);
actions.AddRange(Export(item, folder, config));
}
return(actions);
}
public IEnumerable <uSyncAction> ImportAll(string folder, HandlerSettings config, bool force, SyncUpdateCallback callback = null)
{
logger.Info <uSync8BackOffice>("Running Import: {0}", Path.GetFileName(folder));
var actions = new List <uSyncAction>();
var updates = new Dictionary <string, TObject>();
actions.AddRange(ImportFolder(folder, config, updates, force, callback));
if (updates.Any())
{
int count = 0;
foreach (var update in updates)
{
count++;
callback?.Invoke($"Second Pass {Path.GetFileName(update.Key)}", count, updates.Count);
ImportSecondPass(update.Key, update.Value, config, callback);
}
}
callback?.Invoke("Done", 1, 1);
return(actions);
}
/// <summary>
/// overrider the default export, because macros, don't exist as an object type???
/// </summary>
public override IEnumerable <uSyncAction> ExportAll(int parent, string folder, HandlerSettings config, SyncUpdateCallback callback)
{
// we clean the folder out on an export all.
syncFileService.CleanFolder(folder);
var actions = new List <uSyncAction>();
var items = macroService.GetAll().ToList();
int count = 0;
foreach (var item in items)
{
count++;
callback?.Invoke(item.Name, count, items.Count);
actions.AddRange(Export(item, folder, config));
}
return(actions);
}
public Attempt <int> Deploy(string folder, XElement config, SyncUpdateCallback update)
{
var settings = LoadSettings(config);
var client = new FtpClient(settings.Server);
client.Credentials = new NetworkCredential(settings.Username, settings.Password);
client.EncryptionMode = FtpEncryptionMode.Explicit;
client.SslProtocols = SslProtocols.Default | SslProtocols.Tls11 | SslProtocols.Tls12;
client.ValidateCertificate += new FtpSslValidation(OnValidateCertificate);
update?.Invoke("connecting to server", 1, 100);
client.Connect();
var count = UploadFolder(settings.Folder, folder, "", client, update);
client.Disconnect();
return(Attempt.Succeed(count));
}
private int UploadFolder(string siteRoot, string localRoot, string folder, SftpClient client, SyncUpdateCallback callback)
{
if (!client.IsConnected)
{
throw new FtpException("Not connected");
}
var path = Path.Combine(localRoot, folder);
if (!Directory.Exists(path))
{
throw new DirectoryNotFoundException("Path not found: " + path);
}
var dest = $"/{siteRoot}/{folder}";
try
{
client.CreateDirectory(dest);
}
catch { }
callback?.Invoke($"Syncing {folder}", 5, 10);
var files = client.SynchronizeDirectories(path, dest, "*.*");
var total = files.Count();
foreach (var child in Directory.GetDirectories(path))
{
var childFolder = child.Substring(localRoot.Length)
.Replace("\\", "/");
total += UploadFolder(siteRoot, localRoot, childFolder, client, callback);
}
return(total);
}
/// <summary>
/// this is the simple interface, based purely on level,
/// we could get clever (like dependency trees for content types)
///
/// but that would have to be implimented lower down (and it doesn't
/// really matter for things in containers only things that parent others).
/// </summary>
/// <param name="folder"></param>
/// <param name="force"></param>
/// <param name="updates"></param>
/// <returns></returns>
protected override IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback)
{
// if not using flat, then directory structure is doing
// this for us.
if (config.UseFlatStructure == false)
{
return(base.ImportFolder(folder, config, updates, force, callback));
}
List <uSyncAction> actions = new List <uSyncAction>();
var files = syncFileService.GetFiles(folder, "*.config");
List <LeveledFile> nodes = new List <LeveledFile>();
callback?.Invoke("Calculating import order", 0, 1);
logger.Verbose(handlerType, "Calculating import order");
foreach (var file in files)
{
try
{
var node = LoadNode(file);
if (node != null)
{
nodes.Add(new LeveledFile
{
Level = node.GetLevel(),
File = file
});
}
}
catch (XmlException ex)
{
// one of the files is wrong. (do we stop or carry on)
logger.Warn(handlerType, $"Error loading file: {file} [{ex.Message}]");
actions.Add(uSyncActionHelper <TObject> .SetAction(
SyncAttempt <TObject> .Fail(Path.GetFileName(file), ChangeType.Fail, $"Failed to Load: {ex.Message}"), file, false));
}
}
// loaded - now process.
var flags = SerializerFlags.None;
if (force)
{
flags |= SerializerFlags.Force;
}
if (config.BatchSave)
{
flags |= SerializerFlags.DoNotSave;
}
var cleanMarkers = new List <string>();
foreach (var item in nodes.OrderBy(x => x.Level).Select((Node, Index) => new { Node, Index }))
{
var filename = Path.GetFileNameWithoutExtension(item.Node.File);
callback?.Invoke($"{filename}", item.Index, nodes.Count);
logger.Verbose(handlerType, "{Index} Importing: {File}, [Level {Level}]", item.Index, filename, item.Node.Level);
var attempt = Import(item.Node.File, config, flags);
if (attempt.Success)
{
if (attempt.Change == ChangeType.Clean)
{
cleanMarkers.Add(item.Node.File);
}
else if (attempt.Item != null)
{
updates.Add(item.Node.File, attempt.Item);
}
}
actions.Add(uSyncActionHelper <TObject> .SetAction(attempt, item.Node.File, IsTwoPass));
}
if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
// bulk save - should be the fastest way to do this
callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}
var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
actions.AddRange(ImportFolder(children, config, updates, force, callback));
}
if (actions.All(x => x.Success))
{
// LINQ
// actions.AddRange(cleanMarkers.Select(x => CleanFolder(x)).SelectMany(a => a));
// only if there are no fails.
// then we consider the folder safe to clean
foreach (var cleanfile in cleanMarkers)
{
actions.AddRange(CleanFolder(cleanfile, false, config.UseFlatStructure));
}
// remove the actual cleans (they will have been replaced by the deletes
actions.RemoveAll(x => x.Change == ChangeType.Clean);
}
callback?.Invoke("", 1, 1);
return(actions);
}
protected virtual IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback)
{
List <uSyncAction> actions = new List <uSyncAction>();
var files = GetImportFiles(folder);
var flags = SerializerFlags.None;
if (force)
{
flags |= SerializerFlags.Force;
}
if (config.BatchSave)
{
flags |= SerializerFlags.DoNotSave;
}
var cleanMarkers = new List <string>();
int count = 0;
int total = files.Count();
foreach (string file in files)
{
count++;
callback?.Invoke($"Importing {Path.GetFileNameWithoutExtension(file)}", count, total);
var attempt = Import(file, config, flags);
if (attempt.Success)
{
if (attempt.Change == ChangeType.Clean)
{
cleanMarkers.Add(file);
}
else if (attempt.Item != null)
{
updates.Add(file, attempt.Item);
}
}
var action = uSyncActionHelper <TObject> .SetAction(attempt, file, this.Alias, IsTwoPass);
if (attempt.Details != null && attempt.Details.Any())
{
action.Details = attempt.Details;
}
if (attempt.Change != ChangeType.Clean)
{
actions.Add(action);
}
}
// bulk save ..
if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
// callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}
var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
actions.AddRange(ImportFolder(children, config, updates, force, callback));
}
if (actions.All(x => x.Success) && cleanMarkers.Count > 0)
{
// this is just extra messaging, given how quickly the next message will be sent.
// callback?.Invoke("Cleaning Folders", 1, cleanMarkers.Count);
foreach (var item in cleanMarkers.Select((filePath, Index) => new { filePath, Index }))
{
var folderName = Path.GetFileName(item.filePath);
callback?.Invoke($"Cleaning {folderName}", item.Index, cleanMarkers.Count);
var cleanActions = CleanFolder(item.filePath, false, config.UseFlatStructure);
if (cleanActions.Any())
{
actions.AddRange(cleanActions);
}
else
{
// nothing to delete, we report this as a no change
actions.Add(uSyncAction.SetAction(true, $"Folder {Path.GetFileName(item.filePath)}", change: ChangeType.NoChange, filename: item.filePath));
}
}
// remove the actual cleans (they will have been replaced by the deletes
actions.RemoveAll(x => x.Change == ChangeType.Clean);
}
return(actions);
}
/// <inheritdoc/>
protected override IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback)
{
// if not using flat then directory structure is sorting them for us.
if (config.UseFlatStructure == false)
{
return(base.ImportFolder(folder, config, updates, force, callback));
}
List <uSyncAction> actions = new List <uSyncAction>();
callback?.Invoke("Calculating import order", 0, 1);
logger.LogDebug("Calculating import order");
var orderedFiles = GetLevelOrderedFiles(folder, actions);
// process.
var flags = SerializerFlags.None;
if (force)
{
flags |= SerializerFlags.Force;
}
var cleanMarkers = new List <string>();
foreach (var item in orderedFiles.Select((Node, Index) => new { Node, Index }))
{
var filename = Path.GetFileNameWithoutExtension(item.Node.File);
callback?.Invoke($"{filename}", item.Index, orderedFiles.Count);
logger.LogTrace("{Index} Importing: {File}, [Level {Level}]", item.Index, filename, item.Node.Level);
var result = Import(item.Node.File, config, flags);
foreach (var attempt in result)
{
if (attempt.Success)
{
if (attempt.Change == ChangeType.Clean)
{
cleanMarkers.Add(item.Node.File);
}
else if (attempt.Item != null && attempt.Item is TObject attemptItem)
{
updates.Add(item.Node.File, attemptItem);
}
}
if (attempt.Change != ChangeType.Clean)
{
actions.Add(attempt);
}
}
}
if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
// bulk save - should be the fastest way to do this
callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}
var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
actions.AddRange(ImportFolder(children, config, updates, force, callback));
}
if (actions.All(x => x.Success))
{
// LINQ
// actions.AddRange(cleanMarkers.Select(x => CleanFolder(x)).SelectMany(a => a));
// only if there are no fails.
// then we consider the folder safe to clean
foreach (var cleanfile in cleanMarkers)
{
actions.AddRange(CleanFolder(cleanfile, false, config.UseFlatStructure));
}
// remove the actual cleans (they will have been replaced by the deletes
actions.RemoveAll(x => x.Change == ChangeType.Clean);
}
callback?.Invoke("", 1, 1);
return(actions);
}
/// <summary>
/// this is the simple interface, based purely on level,
/// we could get clever (like dependency trees for content types)
///
/// but that would have to be implimented lower down (and it doesn't
/// really matter for things in containers only things that parent others).
/// </summary>
/// <param name="folder"></param>
/// <param name="force"></param>
/// <param name="updates"></param>
/// <returns></returns>
protected override IEnumerable <uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary <string, TObject> updates, bool force, SyncUpdateCallback callback)
{
// if not using flat, then directory structure is doing
// this for us.
if (config.UseFlatStructure == false)
{
return(base.ImportFolder(folder, config, updates, force, callback));
}
List <uSyncAction> actions = new List <uSyncAction>();
var files = syncFileService.GetFiles(folder, "*.config");
List <LeveledFile> nodes = new List <LeveledFile>();
callback?.Invoke("Calculating import order", 0, 1);
foreach (var file in files)
{
try
{
var node = LoadNode(file);
if (node != null)
{
nodes.Add(new LeveledFile
{
Level = node.GetLevel(),
File = file
});
}
}
catch (XmlException ex)
{
// one of the files is wrong. (do we stop or carry on)
logger.Warn <TObject>($"Error loading file: {file} [{ex.Message}]");
actions.Add(uSyncActionHelper <TObject> .SetAction(
SyncAttempt <TObject> .Fail(Path.GetFileName(file), ChangeType.Fail, $"Failed to Load: {ex.Message}"), file, false));
}
}
// loaded - now process.
var flags = SerializerFlags.None;
if (force)
{
flags |= SerializerFlags.Force;
}
if (config.BatchSave)
{
flags |= SerializerFlags.DoNotSave;
}
var count = 0;
foreach (var node in nodes.OrderBy(x => x.Level))
{
count++;
callback?.Invoke($"{Path.GetFileName(node.File)}", count, nodes.Count);
var attempt = Import(node.File, config, flags);
if (attempt.Success && attempt.Item != null)
{
updates.Add(node.File, attempt.Item);
}
actions.Add(uSyncActionHelper <TObject> .SetAction(attempt, node.File, IsTwoPass));
}
if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
// bulk save - should be the fastest way to do this
callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}
var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
actions.AddRange(ImportFolder(children, config, updates, force, callback));
}
callback?.Invoke("", 1, 1);
return(actions);
}