public string GetDestPath(CopyItem ci, CopyType ct)
{
string path = "";
string dir = "";
if (!String.IsNullOrEmpty(ci.ToDir))
{
path = ci.ToDir;
dir = ci.ToDir;
}
else if (!String.IsNullOrEmpty(ci.ToFile))
{
path = ci.ToFile;
dir = Path.GetDirectoryName(path);
}
Utils.Exec("mkdir", String.Format("-p {0}/{1}", wipDir, dir));
string dstDir = String.Format("{0}/{1}", wipDir, path);
string dstPath = dstDir;
if (ct.Equals(CopyType.Http))
{
dstPath = String.Format("-o {0}", dstDir);
}
return(dstPath);
}
public void CopyFile(CopyItem copyItem, string source, string dest, string outputPath)
{
source = copyItem.FormatSourcePath(source);
source = Path.GetFullPath(source);
dest = copyItem.FormatDestinationPath(dest, outputPath);
// If the output path is a directory assume they want to just drop the file in here
if (Directory.Exists(dest))
{
Debug.Log($"[{nameof(CopyProjectFilesOnBuild)}] {dest} is directory so automatically appending the filename");
dest += $"{Path.GetFileName(source)}";
}
// Create the directory structure to the output path, if the output path is a file make sure to get the parent directory (To avoid creating a folder with the file name)
Directory.CreateDirectory(!Directory.Exists(dest) ? Directory.GetParent(dest).ToString() : dest);
// Follow overwrite rule
if (!copyItem.OverwriteIfExists && File.Exists(dest))
{
Debug.Log($"[{nameof(CopyProjectFilesOnBuild)}] File {copyItem.Name} already exists and set to no overwrite, skipping...");
return;
}
Debug.Log($"[{nameof(CopyProjectFilesOnBuild)}] Copying {source} => {dest}");
/*if (EditorUtility.DisplayDialog("ExecuteCopy?", $"Copy {source} => {dest}", "Yes", "No"))*/
File.Copy(source, dest, copyItem.OverwriteIfExists);
}
public void CopyFiles(CopyItem copyItem, string source, string dest, string outputPath, string searchPattern = "*")
{
dest = copyItem.FormatDestinationPath(dest, outputPath);
Directory.CreateDirectory(dest);
// If the output is a file then get the parent
//if (File.Exists(dest)) dest = Directory.GetParent(dest).ToString();
foreach (var file in Directory.GetFiles(source, searchPattern))
{
var skip = false;
// If the file ends with any of the extensions then skip
foreach (var excludeExtension in copyItem.DirectoryExcludeExtensions)
{
if (file.EndsWith(excludeExtension, StringComparison.InvariantCultureIgnoreCase))
{
skip = true;
}
}
if (skip)
{
continue;
}
CopyFile(copyItem, file, dest, outputPath);
}
}
private void GenerateCopyItems(string sourceDir, string sourceRootDir, string outputRootDir,
string filter, List <CopyItem> copyItems)
{
Debug.Assert(sourceDir.StartsWith(sourceRootDir));
int rootDirLen = sourceRootDir.Length;
string outputDir = outputRootDir;
if (sourceDir.Length > rootDirLen)
{
Debug.Assert(sourceDir.ToCharArray()[rootDirLen] == '\\');
string relativeDir = sourceDir.Substring(rootDirLen + 1);
outputDir = Path.Combine(outputRootDir, relativeDir);
}
string[] filePaths = Directory.GetFiles(sourceDir, filter, SearchOption.TopDirectoryOnly);
foreach (string filePath in filePaths)
{
CopyItem copyItem = new CopyItem();
copyItem.Source = filePath;
copyItem.Destination = Path.Combine(outputDir, Path.GetFileName(filePath));
if (copyItems.Contains(copyItem) == false)
{
copyItems.Add(copyItem);
}
}
}
void CopyBone(CopyItem item)
{
var source = item.Source;
var target = item.Destination;
MeshFilter meshFilter = source.GetComponent <MeshFilter>();
MeshRenderer meshRenderer = source.GetComponent <MeshRenderer>();
CapsuleCollider capsuleCollider = source.GetComponent <CapsuleCollider>();
SphereCollider sphereCollider = source.GetComponent <SphereCollider>();
Rigidbody rigidbody = source.GetComponent <Rigidbody>();
FixedJoint fixedJoint = source.GetComponent <FixedJoint>();
ConfigurableJoint configurableJoint = source.GetComponent <ConfigurableJoint>();
SensorBehavior sensorBehavior = source.GetComponent <SensorBehavior>();
HandleOverlap handleOverlap = source.GetComponent <HandleOverlap>();
if (meshFilter != null)
{
CopyComponent(meshFilter, target.gameObject);
}
if (meshRenderer != null)
{
CopyComponent(meshRenderer, target.gameObject);
}
if (capsuleCollider != null)
{
CopyComponent(capsuleCollider, target.gameObject);
}
if (sphereCollider != null)
{
CopyComponent(sphereCollider, target.gameObject);
}
if (rigidbody != null)
{
CopyComponent(rigidbody, target.gameObject);
}
if (fixedJoint != null)
{
CopyComponent(fixedJoint, target.gameObject);
}
if (configurableJoint != null)
{
CopyComponent(configurableJoint, target.gameObject);
}
if (sensorBehavior != null)
{
CopyComponent(sensorBehavior, target.gameObject);
}
if (handleOverlap != null)
{
CopyComponent(handleOverlap, target.gameObject);
}
if (sphereCollider != null)
{
target.localScale = source.localScale;
}
target.GetComponent <Rigidbody>().isKinematic = true;
}
void CopyBoneAsChild(string sourceName, string targetName, Vector3 offset, Quaternion rotation)
{
var item = new CopyItem {
Source = _sources.First(x => x.name == sourceName),
Destination = _targets.First(x => x.name == targetName)
};
CopyBoneAsChild(item, offset, rotation);
}
public (bool succes, string error) Copy(string source, string target, CopyItem Item)
{
return(Copy(new CopySettings
{
Item = Item,
Source = source,
Target = target
}));
}
protected void DrawCopyItems()
{
if (Copier.CopyItems == null || Copier.SourceRootObject == null || Copier.DestinationRootObject == null || !componentsChecked)
{
return;
}
for (int i = 0; i < Copier.CopyItems.Count; i++)
{
AnimBool foldState = foldStates[i];
CopyItem copyItem = Copier.CopyItems[i];
EditorGUILayout.BeginHorizontal();
{
foldState.target = EditorGUILayout.Toggle(foldState.target, EditorStyles.foldout, GUILayout.Width(checkboxWidth));
copyItem.Enabled = EditorGUILayout.Toggle("", copyItem.Enabled, GUILayout.Width(checkboxWidth));
EditorGUILayout.LabelField("From", GUILayout.Width(32));
EditorGUILayout.ObjectField(copyItem.SourceObject, typeof(GameObject), true);
EditorGUILayout.LabelField("to", GUILayout.Width(14));
copyItem.DestinationObject = (GameObject)EditorGUILayout.ObjectField(copyItem.DestinationObject, typeof(GameObject), true);
}
EditorGUILayout.EndHorizontal();
if (EditorGUILayout.BeginFadeGroup(foldState.faded))
{
EditorGUI.indentLevel += 3;
EditorGUILayout.BeginVertical();
{
for (int j = 0; j < copyItem.Copyables.Count; j++)
{
CopyItem.Copyable copyable = copyItem.Copyables[j];
EditorGUILayout.BeginHorizontal();
{
copyable.Enabled = EditorGUILayout.ToggleLeft(GetComponentIcon(copyable.Component.GetType()), copyable.Enabled);
}
EditorGUILayout.EndHorizontal();
}
}
EditorGUILayout.EndVertical();
EditorGUI.indentLevel -= 3;
}
EditorGUILayout.EndFadeGroup();
}
if (GUILayout.Button(new GUIContent("Copy components")))
{
List <Component> newComponents = Copier.CopyComponents();
Copier.UpdateReferences(newComponents);
copiedCount = newComponents.Count;
}
if (copiedCount >= 0)
{
EditorGUILayout.HelpBox(string.Format("Succesfully copied {0} components from {1} to {2}.", copiedCount, Copier.SourceRootObject.name, Copier.DestinationRootObject.name), MessageType.Info);
}
}
/// <summary>
/// Handle clipboard changes.
/// </summary>
/// <param name="data"></param>
/// <param name="formats"></param>
void ClipboardMonitor_OnClipboardChange(System.Windows.Forms.DataObject data, string[] formats)
{
System.Windows.Application.Current.Dispatcher.Invoke(
System.Windows.Threading.DispatcherPriority.Normal,
(Action) delegate()
{
CopyItem it;
it = new CopyItem(data, formats);
AddToList(it);
});
}
private static CopyItem CreateJob(string sourceAccountToken, string destinationAccountToken, string sourceContainer, string destinationContainer, bool isIncremental, TextWriter log)
{
string job = "Incremental Backup, Account: " + sourceAccountToken + ", Source Container: " + sourceContainer + ", Destination Container: " + destinationContainer;
// Create CopyItem object, pass it to WebJobs queue
CopyItem copyitem = new CopyItem(job, sourceAccountToken, destinationAccountToken, sourceContainer, destinationContainer, isIncremental);
// Log Job Creation
log.WriteLine("Create Job: " + job);
return(copyitem);
}
static void Postfix(Kingmaker.UI.ServiceWindow.ItemSlot __instance, ref string __result)
{
UnitEntityData currentCharacter = UIUtility.GetCurrentCharacter();
CopyItem component = __instance.Item.Blueprint.GetComponent <CopyItem>();
string actionName = component?.GetActionName(currentCharacter) ?? string.Empty;
if (!string.IsNullOrWhiteSpace(actionName))
{
actionName = GetSpellbookActionName(actionName, __instance.Item, currentCharacter);
}
__result = actionName;
}
public SamplePage(CopyItem parameter = null)
{
InitializeComponent();
// Check that our BindingContext exists, then assign the value(s) as appropriate
var viewModel = this.BindingContext as ViewModels.SamplePageViewModel;
if (viewModel != null && parameter != null)
{
viewModel.Subject = parameter.Title;
viewModel.Copy = parameter.Body;
}
}
public Communication CreateCommunication(CopyItem copyItem)
{
var communication = new Communication
{
Id = Id,
Definition = Definition,
AffiliateId = AffiliateId,
From = From,
To = To,
Copy = Copy == null ? null : Copy.Cast <ICommunicationRecipient>().ToList(),
BlindCopy = BlindCopy == null ? null : BlindCopy.Cast <ICommunicationRecipient>().ToList(),
Return = Return,
Subject = copyItem.Text,
Views = new List <CommunicationView>()
};
foreach (var viewItem in copyItem.ViewItems)
{
if (viewItem.ResourceItems.Count == 0)
{
communication.Views.Add(new CommunicationView(viewItem.Text, viewItem.MimeType));
}
else
{
communication.Views.Add(new CommunicationView(viewItem.Text, viewItem.MimeType, GetCommunicationResources(viewItem.ResourceItems)));
}
}
// Look for temporary files.
var tempFiles = GetTempFileAttachments();
if (tempFiles != null)
{
communication.AddAttachments(tempFiles);
}
// Look for permanent files.
var attachments = GetAttachments();
if (attachments != null)
{
communication.AddAttachments(attachments);
}
return(communication);
}
private async void Navigate(CopyItem parameter)
{
// Pass the parameters as part of the navigation string. A typical example would be to pass only the ID of an object, which
// the resulting page would lookup/resolve via a service
await _navigationService.NavigateAsync($"{nameof(Views.SamplePage)}?subject={parameter.Title}&content={parameter.Body}");
// Or, we can declare them explicity using a strongly typed object, which is ideal for passing
// a more complex model that the next page won't lookup on its own
/*
* var payload = new NavigationParameters();
* payload.Add("content", parameter);
*
* await _navigationService.NavigateAsync($"Navigation/{nameof(Views.SamplePage)}", payload);
*/
}
public void Paste(IGraphClipboardContext context, IModelItem item, object userData)
{
//Note: This is the _copied_ item
INode node = (INode)item;
if (node.Labels.Count > 0)
{
if (userData is CopyItem && context != null)
{
// If we are a Node with at least one label, we
// change our text to the one that is provided by userData.
CopyItem cs = (CopyItem)userData;
context.TargetGraph.SetLabelText(node.Labels[0], cs.ToString());
}
}
}
public async Task <int> CreateAsync(string copyName, string copyBody)
{
var newCopy = new CopyItem
{
CopyName = copyName,
CopyBody = copyBody,
DateAdded = DateTime.Now,
DateUpdated = DateTime.Now
};
_context.CopyItems.Add(newCopy);
await _context.SaveChangesAsync();
return(newCopy.CopyId);
}
public int Create(string copyName, string copyBody)
{
var newCopy = new CopyItem
{
CopyName = copyName,
CopyBody = copyBody,
DateAdded = DateTime.Now,
DateUpdated = DateTime.Now
};
_context.CopyItems.Add(newCopy);
_context.SaveChanges();
return(newCopy.CopyId);
}
public void GetCpCommandTest(string from, CopyType expType, string expArgs)
{
var cp = new UtilCopier();
cp.SetCopyArg(copyArgs);
cp.SetCopyCmd(copyCmds);
cp.SetWipDir("");
var ci = new CopyItem();
ci.From = from;
(CopyType type, string cmd, string args) = cp.GetCpCommand(ci);
Assert.AreEqual(expType, type);
Assert.AreEqual(expArgs, args);
}
void CopyBoneAsChild(CopyItem item, Vector3 offset, Quaternion rotation)
{
var source = item.Source;
var target = item.Destination;
var copy = Instantiate(source, item.Destination);
Rigidbody rigidbody = copy.GetComponent <Rigidbody>();
if (rigidbody != null)
{
rigidbody.isKinematic = true;
}
CapsuleCollider capsuleCollider = copy.GetComponent <CapsuleCollider>();
copy.transform.localPosition = offset;
copy.transform.localRotation = rotation;
return;
}
/// <summary>
/// Add an item to cliboard.
/// </summary>
/// <param name="item"></param>
public static void AddToClipboard(CopyItem item)
{
try
{
Clipboard.SetDataObject(item.Item, true, 2, 100);
}
catch (Exception e)
{
try
{
System.Threading.Thread.Sleep(100);
//Clipboard.SetData(item.Format.ToString(), item.Item);
}
catch (Exception)
{
// Ignore
}
}
}
private static CopyItem CreateJob(string jobName, string sourceAccountToken, string destinationAccountToken, string sourceContainer, string sourceDirectory, string destinationContainer, string destinationDirectory, bool isIncremental, TextWriter log)
{
string jobId = Guid.NewGuid().ToString();
// Create CopyItem object, pass it to WebJobs queue
CopyItem copyitem = new CopyItem(jobId, jobName, sourceAccountToken, destinationAccountToken, sourceContainer, sourceDirectory, destinationContainer, destinationDirectory, isIncremental);
// Log Job Creation
StringBuilder message = new StringBuilder();
message.AppendLine("Queued Job: " + jobName);
message.AppendLine("Source Account: " + sourceAccountToken);
message.AppendLine("Source Container: " + sourceContainer);
message.AppendLine("Destination Container: " + destinationContainer);
message.AppendLine("");
log.WriteLine(message);
return(copyitem);
}
public void GetDestPathTest(string toFile, string toDir, CopyType type, string expValue)
{
string tmpPath = Path.GetTempPath();
var cp = new UtilCopier();
cp.SetCopyArg(copyArgs);
cp.SetCopyCmd(copyCmds);
cp.SetWipDir(tmpPath);
var ci = new CopyItem();
ci.ToFile = toFile;
ci.ToDir = toDir;
var dstPath = cp.GetDestPath(ci, type);
var expPath = String.Format(expValue, tmpPath);
Assert.AreEqual(expPath, dstPath);
}
private CopyItem CreateCopyItem(string htmlText, string plainText)
{
var copyItem = new CopyItem(GetSubject(htmlText));
// Add the html view.
var viewItem = new ViewItem(MediaTypeNames.Text.Html, htmlText);
AddResourceItems(htmlText, viewItem);
copyItem.ViewItems.Add(viewItem);
// Add the plain text view.
if (!string.IsNullOrEmpty(plainText))
{
viewItem = new ViewItem(MediaTypeNames.Text.Plain, plainText);
copyItem.ViewItems.Add(viewItem);
}
return(copyItem);
}
/// <summary>
/// do copy
/// </summary>
/// <returns></returns>
protected override bool DoCopy()
{
if (innerControl == null)
{
return(false);
}
ShapeBase[] sources = innerControl.ShapeControl.Document.SelectedShapes;
if (sources != null && sources.Length > 0)
{
int length = sources.Length;
var clones = new CopyItem[length];
for (int i = 0; i < length; i++)
{
if (sources[i].LayerType != LayerType.Paint)
{
var source = sources[i].Tag as IShapeLayer;
if (source == null)
{
source = innerControl.GetLayerControl(sources[i]).Layer;
}
var target = source.Copy();
clones[i] = new CopyItem(target.Shape, target);
}
else
{
var source = innerControl.GetLayerControl(sources[i]).Layer.Copy();
var target = sources[i].Copy();
clones[i] = new CopyItem(target, source);
}
clones[i].Shape.ID = sources[i].ID;
}
ClipboardService.Set(clones);
return(true);
}
return(false);
}
public (CopyType type, string cmd, string args) GetCpCommand(CopyItem ci)
{
string args = copyArgs[CopyType.Cp];
string cpCmd = copyCmds[CopyType.Cp];
CopyType type = CopyType.Cp;
if (httpRegex.IsMatch(ci.From))
{
cpCmd = copyCmds[CopyType.Http];
args = copyArgs[CopyType.Http];
type = CopyType.Http;
}
else if (gsutilRegex.IsMatch(ci.From))
{
cpCmd = copyCmds[CopyType.GsUtilCp];
args = copyArgs[CopyType.GsUtilCp];
type = CopyType.GsUtilCp;
}
return(type, cpCmd, args);
}
/// <summary>
/// Add an item from clipboard to the list.
/// </summary>
/// <param name="it"></param>
private void AddToList(CopyItem it)
{
try
{
if (!it.IsCopyBaseItem())
{
if (CopyItems == null)
{
CopyItems = new ObservableCollection <CopyItem>();
}
CopyItems.Add(it);
SelectedCopyItem = it;
}
}
catch (Exception)
{
CopyItems = new ObservableCollection <CopyItem>();
CopyItems.Add(it);
SelectedCopyItem = it;
}
finally
{
}
}
private void Navigate(CopyItem parameter)
{
this._navigationService.NavigateTo(nameof(Views.SamplePage), parameter);
}
private void ProcessProjectManifest(string sourceRootDir, string outputRootDir, string projectName, bool referenced, List <CopyItem> copyItems)
{
string sourceProjectDir = Path.Combine(sourceRootDir, projectName);
string sourceContentDir = Path.Combine(sourceRootDir, "Content");
string sourceExternalsDir = Path.Combine(sourceRootDir, "Externals");
string sourceFrameworkDir = Path.Combine(sourceRootDir, "SampleFramework12");
string outputProjectDir = Path.Combine(outputRootDir, projectName);
string outputContentDir = Path.Combine(outputRootDir, "Content");
string outputExternalsDir = Path.Combine(outputRootDir, "Externals");
string outputFrameworkDir = Path.Combine(outputRootDir, "SampleFramework12");
string projectManifestPath = Path.Combine(sourceProjectDir, "Manifest.txt");
if (File.Exists(projectManifestPath) == false)
{
WriteLog("No manifest file found in source directory, aborting");
return;
}
WriteLog("Parsing project manifest file");
Manifest projectManifest = ParseManifestFile(projectManifestPath);
List <string> codeFiles = new List <string>()
{
"*.h", "*.cpp", "*.cs", "*.vcxproj", "*.vcxproj.filters",
"*.sln", "*.ico", "*.rc", "*.sublime-project", "*.csproj",
"*.resx", "*.props", "*.config", "*.natvis"
};
List <string> shaderFiles = new List <string>()
{
"*.hlsl"
};
if (includeCodeCB.Checked)
{
WriteLog("Gathering code files");
if (copyGitIgnoreCB.Checked)
{
codeFiles.Add(".gitignore");
}
foreach (string filter in codeFiles)
{
GenerateCopyItems(sourceProjectDir, sourceRootDir, outputRootDir, filter, copyItems);
}
foreach (string manifestItem in projectManifest.Code)
{
ProcessManifestItem(sourceProjectDir, sourceRootDir, outputRootDir, manifestItem, copyItems);
}
if (copyGitIgnoreCB.Checked && referenced == false)
{
CopyItem gitIgnoreItem = new CopyItem();
gitIgnoreItem.Source = Path.Combine(sourceProjectDir, @"..\.gitignore");
gitIgnoreItem.Destination = Path.Combine(outputRootDir, @".gitignore");
copyItems.Add(gitIgnoreItem);
}
WriteLog("Gathering external files");
foreach (string manifestItem in projectManifest.External)
{
ProcessManifestItem(sourceExternalsDir, sourceRootDir, outputRootDir, manifestItem, copyItems);
}
}
WriteLog("Gathering shader files");
foreach (string filter in shaderFiles)
{
GenerateCopyItems(sourceProjectDir, sourceRootDir, outputRootDir, filter, copyItems);
}
if (includeExeCB.Checked && referenced == false)
{
string exeDir = Path.Combine(sourceProjectDir, "x64", "Release");
string exeName = projectName + ".exe";
string exePath = Path.Combine(exeDir, exeName);
if (File.Exists(exePath))
{
WriteLog("Gathering executable files");
CopyItem exeItem = new CopyItem();
exeItem.Source = exePath;
exeItem.Destination = Path.Combine(outputProjectDir, exeName);
copyItems.Add(exeItem);
// Also copy external DLL;s
string[] dllPaths = Directory.GetFiles(exeDir, "*.dll", SearchOption.TopDirectoryOnly);
foreach (string dllPath in dllPaths)
{
string dllName = Path.GetFileName(dllPath);
CopyItem dllItem = new CopyItem();
dllItem.Source = dllPath;
dllItem.Destination = Path.Combine(outputProjectDir, dllName);
copyItems.Add(dllItem);
}
GenerateCopyItems(sourceProjectDir, sourceRootDir, outputRootDir, "*.dll", copyItems);
}
else
{
WriteLog("No executable detected! Skipping.");
}
}
if (includeShaderCacheCB.Checked && referenced == false)
{
string cacheDir = Path.Combine(sourceProjectDir, "ShaderCache", "Release");
if (Directory.Exists(cacheDir))
{
WriteLog("Gathering shader cache files");
GenerateCopyItems(cacheDir, sourceRootDir, outputRootDir, "*.cache", copyItems);
}
else
{
WriteLog("No shader cache directory found, skipping.");
}
}
foreach (string manifestItem in projectManifest.Local)
{
ProcessManifestItem(sourceProjectDir, sourceRootDir, outputRootDir, manifestItem, copyItems);
}
WriteLog("Gathering content files");
foreach (string manifestItem in projectManifest.Content)
{
ProcessManifestItem(sourceContentDir, sourceRootDir, outputRootDir, manifestItem, copyItems);
}
if (copyGitIgnoreCB.Checked && includeCodeCB.Checked && referenced == false)
{
CopyItem contentIgnore;
contentIgnore.Source = Path.Combine(sourceContentDir, @".gitignore");
contentIgnore.Destination = Path.Combine(outputContentDir, @".gitignore");
copyItems.Add(contentIgnore);
CopyItem externalsIgnore;
externalsIgnore.Source = Path.Combine(sourceExternalsDir, @".gitignore");
externalsIgnore.Destination = Path.Combine(outputExternalsDir, @".gitignore");
copyItems.Add(externalsIgnore);
}
foreach (string reference in projectManifest.References)
{
WriteLog(string.Format("Processing referenced project '{0}'", reference));
string referenceDir = Path.Combine(sourceRootDir, reference);
if (Directory.Exists(referenceDir) == false)
{
WriteLog(string.Format("Referenced project directory '{0}' doesn't exist, or isn't a directory. Skipping.", referenceDir));
continue;
}
ProcessProjectManifest(sourceRootDir, outputRootDir, reference, true, copyItems);
}
}
public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken)
{
// Copy TextWrite into Log Helper class
Logger.log = log;
// Log Job Start
await Logger.JobStartAsync(copyItem.JobName);
// This class accumulates transfer data during the copy
ProgressRecorder progressRecorder = new ProgressRecorder();
try
{
// OpContext to track PreCopy Retries on Azure Storage
// DML has its own context object and retry
_opContext = new OperationContext();
_opContext.Retrying += StorageRequest_Retrying;
// Define Blob Request Options
_blobRequestOptions = new BlobRequestOptions
{
// Defined Exponential Retry Policy above
RetryPolicy = _retryPolicy
};
// Set the number of parallel tasks in DML.
// This allows it to copy multiple items at once when copying a container or directory
// The best (and default value) is Environment.ProcessorCount * 8
int parallelTasks = Environment.ProcessorCount * 8;
TransferManager.Configurations.ParallelOperations = parallelTasks;
// Set the number of connections.
// This should match ParallelOperations so each DML copy task has its own connection to Azure Storage
ServicePointManager.DefaultConnectionLimit = parallelTasks;
// Short circuit additional request round trips. We are not chunking and
// uploading large amounts of data where we'd send 100's so set to false
ServicePointManager.Expect100Continue = false;
// User Agent for tracing
TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup";
// CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
// Open connections to both storage accounts
CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken);
CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken);
// Represents a checkpoint from which a transfer may be resumed and continued.
// This is initalized as null first time then hydrated within CopyDirectoryAsync().
// However if this job is being resumed from a previous failure this function will hydrate
// from a serialized checkpoint saved to blob storage.
TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId);
// Context object for the transfer, provides additional runtime information about its execution
// If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy
TransferContext transferContext = new TransferContext(transferCheckpoint)
{
// Pipe transfer progress data to ProgressRecorder
// ProgressRecorder is used to log the results of the copy operation
ProgressHandler = progressRecorder,
// If the destination already exists this delegate is called.
// Return true to overwrite or false to skip the file during the transfer
OverwriteCallback = (source, destination) =>
{
return(OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental));
}
};
// This event is used to log files skipped during the transfer
transferContext.FileSkipped += TransferContext_FileSkipped;
// This event is used to catch exceptions for files that fail during a transfer
transferContext.FileFailed += TransferContext_FileFailed;
// Set Options for copying the container such as search patterns, recursive, etc.
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
IncludeSnapshots = true,
Recursive = true
};
// Get the root source and destination directories for the two containers to be copied
CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory);
CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory);
// Copy the container
await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource);
// Check if any files failed during transfer
if (_failedFiles.Count > 0)
{
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint);
// Throw an exception to fail the job so WebJobs will rerun it
throw new Exception("One or more errors occurred during the transfer.");
}
// Log job completion
await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles);
}
catch (Exception ex)
{
// Log Job Error
await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles);
// Rethrow the error to fail the web job
throw ex;
}
}
/// <summary>
/// A simple function that takes a string and does a ToUpper
/// </summary>
/// <param name="input"></param>
/// <param name="context"></param>
/// <returns></returns>
public void FunctionHandler(ILambdaContext context)
{
string storageConnectionString;
if (!ssmParameterManager.TryGetValue(Constants.StorageConnectionStringSSMPath, out storageConnectionString))
{
throw new Exception("Storage connection path not defined.");
}
else
{
Console.WriteLine($"Storage connection path is set at '{storageConnectionString}'");
}
if (!S3Manager.CheckS3Parameters())
{
return;
}
// Check whether the connection string can be parsed.
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
string ContainerNames;
if (ssmParameterManager.TryGetValue(Constants.StorageContainerNamesSSMPath, out ContainerNames))
{
var arrContainerNames = ContainerNames.Split(',', StringSplitOptions.RemoveEmptyEntries);
List <Task> lstTasks = new List <Task>();
foreach (var ContainerName in arrContainerNames)
{
var t = Task.Factory.StartNew(() =>
{
var lstBlobItems = AzureManager.ListBlobContainer(storageAccount, ContainerName, null);
if (lstBlobItems.Count > 0)
{
Console.WriteLine("Listing S3 items.");
var S3Objects = S3Manager.GetS3Items();
Console.WriteLine($"{S3Objects.Count.ToString()} items retrieved from S3.");
foreach (var S3Object in S3Objects)
{
Console.WriteLine($"Item retrieved from S3: {S3Object.BucketName} - {S3Object.Key} ({S3Object.Size.ToString()} bytes) - ETag {S3Object.ETag}");
}
StringBuilder sb = new StringBuilder();
Amazon.SQS.AmazonSQSClient sqsClient = new Amazon.SQS.AmazonSQSClient();
lstBlobItems.ForEach((a) =>
{
if (!S3Objects.Any(s => s.Key == a.BlobName && s.Size == a.Size))
{
CopyItem copyItem = new CopyItem {
BlobItem = a
};
// No S3 objects that match the Azure blob, copy it over
var strCopyItem = JsonConvert.SerializeObject(copyItem);
SendMessageRequest sendRequest = new SendMessageRequest {
MessageBody = strCopyItem, QueueUrl = System.Environment.GetEnvironmentVariable("QueueName")
};
var sendMessageResult = sqsClient.SendMessageAsync(sendRequest).GetAwaiter().GetResult();
Console.WriteLine($"Item not found in S3, adding to copy queue - {a.BlobName} - send message result: {sendMessageResult.HttpStatusCode.ToString()}");
}
});
}
});
lstTasks.Add(t);
}
Task.WaitAll(lstTasks.ToArray());
}
}
else
{
// Otherwise, let the user know that they need to define the environment variable.
throw new Exception("Storage connection path not defined.");
}
}
