/// <summary>
/// Downloads content from the image store to local destination.
/// </summary>
/// <param name="remoteSource">The relative path of source image store to be downloaded from.</param>
/// <param name="localDestination">The local destination path to download the content.</param>
/// <param name="progressHandler">Defines the behavior to process progress information from the downloading operation.</param>
/// <param name="timeout">The timeout for performing the downloading operation.</param>
/// <param name="imageCopyFlag">The copying control information to specify how file can be overwritten.</param>
public void DownloadContent(string remoteSource, string localDestination, IImageStoreProgressHandler progressHandler, TimeSpan timeout, CopyFlag imageCopyFlag)
{
Utility.WrapNativeSyncInvokeInMTA(() =>
{
this.DownloadContentWrapper(remoteSource, localDestination, progressHandler, Utility.ToMilliseconds(timeout, "timeout"), imageCopyFlag);
}, "DownloadContent");
}
/// <summary>
/// Uploads content of the local source to the native image store at the remote destination.
/// </summary>
/// <param name="remoteDestination">Location relative to image store root where the content needs to be uploaded.</param>
/// <param name="localSource">local location of the source.</param>
/// <param name="progressHandler">Defines the behavior to process progress information from the uploading operation.</param>
/// <param name="timeout">The timeout for performing the uploading operation.</param>
/// <param name="imageCopyFlag">Copy control information to specify how file can be overwritten.</param>
/// <param name="acquireSourceReaderLock">Indicates whether to acquire reader lock.</param>
public void UploadContent(string remoteDestination, string localSource, IImageStoreProgressHandler progressHandler, TimeSpan timeout, CopyFlag imageCopyFlag, bool acquireSourceReaderLock)
{
Utility.WrapNativeSyncInvokeInMTA(() =>
{
this.UploadContentWrapper(remoteDestination, localSource, progressHandler, Utility.ToMilliseconds(timeout, "timeout"), imageCopyFlag);
}, "UploadContent");
}
public void UploadContent(string tag, string sourceSoftLink, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag, bool acquireSourceReaderLock)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
sourceSoftLink = this.GetLocalPath(sourceSoftLink);
if ((!FabricFile.Exists(sourceSoftLink)) &&
(!FabricDirectory.Exists(sourceSoftLink)))
{
throw new IOException(
string.Format(
CultureInfo.CurrentCulture,
StringResources.ImageStoreError_DoesNotExistError,
sourceSoftLink));
}
try
{
using (XStoreFileOperations xstoreFileOperation =
new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
{
tag = XStoreCommon.FormatXStoreUri(tag);
if (copyFlag == CopyFlag.AtomicCopy)
{
this.DeleteContent(tag, helper == null ? timeout : helper.GetRemainingTime());
}
if (FabricFile.Exists(sourceSoftLink))
{
// This is a file copy
xstoreFileOperation.CopyFile(
sourceSoftLink,
tag,
XStoreFileOperationType.CopyFromSMBToXStore,
helper);
}
else
{
// this is a folder copy
xstoreFileOperation.CopyFolder(
sourceSoftLink,
tag,
XStoreFileOperationType.CopyFromSMBToXStore,
copyFlag,
null,
null,
helper);
}
}
}
catch (Exception ex)
{
this.HandleException(ex);
throw;
}
}
private static bool ExtractApplicationPackageHelper(
string appPackageRootDirectory,
IImageStoreProgressHandler progressHandler)
{
using (var pin = new PinCollection())
{
return(NativeImageStore.TryExtractApplicationPackage(
pin.AddObject(appPackageRootDirectory),
new NativeImageStoreProgressEventHandlerBroker(progressHandler)));
}
}
private static void ArchiveApplicationPackageHelper(
string appPackageRootDirectory,
IImageStoreProgressHandler progressHandler)
{
using (var pin = new PinCollection())
{
NativeImageStore.ArchiveApplicationPackage(
pin.AddObject(appPackageRootDirectory),
new NativeImageStoreProgressEventHandlerBroker(progressHandler));
}
}
/// <summary>
/// Generates checksum files for all service package sub-directories (i.e. code/config/data packages) and service manifests found under the application package root directory.
/// </summary>
/// <param name="appPackageRootDirectory">The application package root directory.</param>
/// <param name="progressHandler">An optional handler for tracking progress of the checksum generation.</param>
/// <param name="isImageStoreServiceEnabled">Should be set to true if the cluster's ImageStoreConnectionString is set to "fabric:ImageStore", false otherwise.</param>
/// <remarks>
/// <para>
/// The cluster will automatically generate checksums during application type registration if they're not already part of the application package. This method can be used to incur the cost of checksum file generation while preparing the application package, which will reduce the cost of registering the application type package for large packages.
/// </para>
/// </remarks>
public static void GenerateApplicationPackageChecksumFiles(
string appPackageRootDirectory,
IImageStoreProgressHandler progressHandler,
bool isImageStoreServiceEnabled = true)
{
var layoutSpecification = BuildLayoutSpecification.Create();
var files = new List <string>();
var directories = new List <string>();
foreach (var servicePackageDirectory in FabricDirectory.GetDirectories(appPackageRootDirectory))
{
files.AddRange(FabricDirectory.GetFiles(servicePackageDirectory));
directories.AddRange(FabricDirectory.GetDirectories(servicePackageDirectory));
}
int completedItems = 0;
int totalItems = files.Count + directories.Count;
foreach (var file in files)
{
GenerateChecksumFile(file, layoutSpecification, isImageStoreServiceEnabled);
try
{
if (progressHandler != null)
{
progressHandler.UpdateProgress(++completedItems, totalItems, ProgressUnitType.Files);
}
}
catch (Exception)
{
// Do not fail checksum processing if progress update fails
}
}
foreach (var directory in directories)
{
GenerateChecksumFile(directory, layoutSpecification, isImageStoreServiceEnabled);
try
{
if (progressHandler != null)
{
progressHandler.UpdateProgress(++completedItems, totalItems, ProgressUnitType.Files);
}
}
catch (Exception)
{
// Do not fail checksum processing if progress update fails
}
}
}
public bool DownloadIfContentExists(string source, string destination, IImageStoreProgressHandler progressHandler, TimeSpan timeout)
{
TimeoutHelper timeoutHelper = new TimeoutHelper(timeout);
if (this.ImageStore.DoesContentExist(source, timeoutHelper.GetRemainingTime()))
{
this.ImageStore.DownloadContent(source, destination, progressHandler, timeoutHelper.GetRemainingTime(), CopyFlag.AtomicCopy);
ImageBuilderUtility.RemoveReadOnlyFlag(destination);
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// Downloads content from the XStore to local destination.
/// </summary>
/// <param name="tag">Location (relative to RootUri) from where to download the content.</param>
/// <param name="destinationSoftLink">Physical location where to download the content.</param>
/// <param name="handler">Defines the behavior to process progress information from the downloading operation.</param>
/// <param name="timeout">The timeout for performing the downloading operation.</param>
/// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
public void DownloadContent(string tag, string destinationSoftLink, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
destinationSoftLink = this.GetLocalPath(destinationSoftLink);
try
{
using (XStoreFileOperations xstoreFileOperation =
new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
{
tag = XStoreCommon.FormatXStoreUri(tag);
bool xstoreFileExist = xstoreFileOperation.XStoreFileExists(tag, helper);
bool xstoreFolderExist = xstoreFileOperation.XStoreFolderExists(tag, true, helper);
if ((!xstoreFileExist) && (!xstoreFolderExist))
{
throw new FileNotFoundException(string.Format(CultureInfo.InvariantCulture, StringResources.ImageStoreError_DoesNotExistError, tag));
}
if (xstoreFileExist)
{
// this is a single blob copy from xstore to smb
xstoreFileOperation.CopyFile(
tag,
destinationSoftLink,
XStoreFileOperationType.CopyFromXStoreToSMB,
helper);
}
else
{
// this is a multiple blob (folder) copy from xstore to smb
xstoreFileOperation.CopyFolder(
tag,
destinationSoftLink,
XStoreFileOperationType.CopyFromXStoreToSMB,
copyFlag,
null,
null,
helper);
}
}
}
catch (Exception ex)
{
this.HandleException(ex);
throw;
}
}
/// <summary>
/// Downloads content from the XStore to local destination.
/// </summary>
/// <param name="remoteSource">Location (relative to RootUri) from where to download the content.</param>
/// <param name="localDestination">Physical location where to download the content.</param>
/// <param name="handler">Defines the behavior to process progress information from the downloading operation.</param>
/// <param name="timeout">The timeout for performing the downloading operation.</param>
/// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
public void DownloadContent(string remoteSource, string localDestination, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
{
MethodInfo downloadContentMethodInfo = this.XStoreType.GetMethod(
"DownloadContent",
new[] { typeof(string), typeof(string), typeof(IImageStoreProgressHandler), typeof(TimeSpan), typeof(CopyFlag) });
if (downloadContentMethodInfo == null)
{
throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, StringResources.Error_MethodLookupFailed, "DownloadContent"));
}
downloadContentMethodInfo.Invoke(
this.XStore,
BindingFlags.InvokeMethod,
Type.DefaultBinder,
new object[] { remoteSource, localDestination, handler, timeout, copyFlag },
CultureInfo.CurrentCulture);
}
public void UploadContent(string remoteDestination, string localSource, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag, bool acquireSourceReaderLock)
{
CheckForReservedImageStoreFolders(remoteDestination);
MethodInfo uploadContentMethodInfo = this.XStoreType.GetMethod(
"UploadContent",
new[] { typeof(string), typeof(string), typeof(IImageStoreProgressHandler), typeof(TimeSpan), typeof(CopyFlag), typeof(bool) });
if (uploadContentMethodInfo == null)
{
throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, StringResources.Error_MethodLookupFailed, "UploadContent"));
}
uploadContentMethodInfo.Invoke(
this.XStore,
BindingFlags.InvokeMethod,
Type.DefaultBinder,
new object[] { remoteDestination, localSource, handler, timeout, copyFlag, acquireSourceReaderLock },
CultureInfo.CurrentCulture);
}
private void DownloadContentWrapper(string remoteSource, string localDestination, IImageStoreProgressHandler progressHandler, UInt32 timeoutMilliseconds, CopyFlag imageCopyFlag)
{
using (var pin = new PinCollection())
{
this.nativeClient.DownloadContent2(
pin.AddObject(remoteSource),
pin.AddObject(localDestination),
progressHandler == null ? null : new NativeImageStoreProgressEventHandlerBroker(progressHandler),
timeoutMilliseconds,
ToNative(imageCopyFlag));
}
}
/// <summary>
/// Downloads content from the file image store to local destination.
/// </summary>
/// <param name="storeSource">The relative path of source file image store to be downloaded from.</param>
/// <param name="localDestination">The local destination path to download the content.</param>
/// <param name="handler">The image store progress handler which is not supported at the file image store.</param>
/// <param name="timeout">The timeout for performing the downloading operation.</param>
/// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
public void DownloadContent(string storeSource, string localDestination, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
localDestination = this.GetLocalPath(localDestination);
string tempDirectory = null;
FileReaderLock readerLock = null;
try
{
string smbTag = this.ConvertTagToSMBPath(storeSource);
if ((!FabricFile.Exists(smbTag)) &&
(!FabricDirectory.Exists(smbTag)))
{
throw new IOException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_DoesNotExistError, smbTag));
}
readerLock = new FileReaderLock(smbTag);
if (!readerLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (accessDescription != null && !accessDescription.HasWriteAccess)
{
// Copy the content from the remote Store to a temp location using the ImpersonationContext
string tempLocalResource = null;
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = accessDescription.WindowsIdentity.Impersonate())
#endif
{
tempDirectory = CreateAndAclDirectory();
tempLocalResource = Path.Combine(tempDirectory, Path.GetRandomFileName());
CopyCallerHoldsReaderLock(smbTag, tempLocalResource, CopyFlag.AtomicCopy, helper);
}
readerLock.Release();
// Copy the content from the temp location to the local destination outside the ImpersonationContext
CopyCallerHoldsReaderLock(tempLocalResource, localDestination, copyFlag, helper);
}
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
CopyCallerHoldsReaderLock(smbTag, localDestination, copyFlag, helper);
}
}
catch (IOException exception)
{
if (exception.GetType() == typeof(IOException))
{
throw new FabricImageStoreException(StringResources.Error_ImageStoreIOException, exception);
}
else
{
throw;
}
}
finally
{
if (readerLock != null)
{
readerLock.Dispose();
}
if (tempDirectory != null && FabricDirectory.Exists(tempDirectory))
{
FabricDirectory.Delete(tempDirectory, recursive: true, deleteReadOnlyFiles: true);
}
}
}
/// <summary>
/// Compresses all service package sub-directories (i.e. code/config/data packages) found under the application package root directory.
/// </summary>
/// <param name="appPackageRootDirectory">The application package root directory.</param>
/// <param name="progressHandler">An optional handler for tracking progress of the compression operation.</param>
/// <remarks>
/// <para>
/// Package checksums are computed on the resulting compressed package files rather than the original uncompressed contents. This means that when uploading compressed packages that were previously uncompressed, their versions must be changed in the Service Manifest.
/// Alternatively, simply omit the sub-package whose contents have not changed from the overall application package.
/// Also note that changing the last modified timestamp of a compressed package's contents will also result in changing the package's checksum.
/// </para>
/// </remarks>
public static void ArchiveApplicationPackage(
string appPackageRootDirectory,
IImageStoreProgressHandler progressHandler)
{
Utility.WrapNativeSyncInvokeInMTA(() => { ArchiveApplicationPackageHelper(appPackageRootDirectory, progressHandler); }, "ArchiveApplicationPackage");
}
/// <summary>
/// Uncompresses all service package sub-directories (i.e. code/config/data packages) found under the application package root directory.
/// </summary>
/// <param name="appPackageRootDirectory">The application package root directory.</param>
/// <param name="progressHandler">An optional handler for tracking progress of the uncompression operation.</param>
/// <returns>
/// <para>True if any compressed packages were found, false otherwise.</para>
/// </returns>
public static bool TryExtractApplicationPackage(
string appPackageRootDirectory,
IImageStoreProgressHandler progressHandler)
{
return(Utility.WrapNativeSyncInvokeInMTA <bool>(() => { return ExtractApplicationPackageHelper(appPackageRootDirectory, progressHandler); }, "ExtractApplicationPackage"));
}
public void UploadContent(string imageStoreConnectionString, string remoteDestination, string localSource, IImageStoreProgressHandler progressHandler, TimeSpan timeout, CopyFlag imageCopyFlag, bool acquireSourceReaderLock)
{
this.fabricClient.ThrowIfDisposed();
Requires.Argument <string>("localSource", localSource).NotNullOrEmpty();
Requires.Argument <string>("remoteDestination", remoteDestination).NotNullOrEmpty();
Requires.Argument <string>("imageStoreConnectionString", imageStoreConnectionString).NotNullOrEmpty();
IImageStore imageStore = this.fabricClient.GetImageStore(imageStoreConnectionString);
if (progressHandler == null)
{
imageStore.UploadContent(
remoteDestination,
localSource,
timeout,
CopyFlag.AtomicCopy,
acquireSourceReaderLock);
}
else
{
imageStore.UploadContent(
remoteDestination,
localSource,
progressHandler,
timeout,
imageCopyFlag,
acquireSourceReaderLock);
}
}
public NativeImageStoreProgressEventHandlerBroker(IImageStoreProgressHandler progressHandler)
{
this.progressHandler = progressHandler;
}
/// <summary>
/// Downloads content from the XStore to local destination.
/// </summary>
/// <param name="tag">Location (relative to RootUri) from where to download the content.</param>
/// <param name="destinationSoftLink">Physical location where to download the content.</param>
/// <param name="handler">Defines the behavior to process progress information from the downloading operation.</param>
/// <param name="timeout">The timeout for performing the downloading operation.</param>
/// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
public void DownloadContent(string tag, string destinationSoftLink, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
destinationSoftLink = this.GetLocalPath(destinationSoftLink);
try
{
using (XStoreFileOperations xstoreFileOperation =
new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
{
tag = XStoreCommon.FormatXStoreUri(tag);
bool xstoreFileExist = xstoreFileOperation.XStoreFileExists(tag, helper);
bool xstoreFolderExist = xstoreFileOperation.XStoreFolderExists(tag, true, helper);
if ((!xstoreFileExist) && (!xstoreFolderExist))
{
throw new FileNotFoundException(string.Format(CultureInfo.InvariantCulture, StringResources.ImageStoreError_DoesNotExistError, tag));
}
if (xstoreFileExist)
{
// this is a single blob copy from xstore to smb
xstoreFileOperation.CopyFile(
tag,
destinationSoftLink,
XStoreFileOperationType.CopyFromXStoreToSMB,
helper);
}
else
{
var operationId = Guid.NewGuid();
// this is a multiple blob (folder) copy from xstore to smb
xstoreFileOperation.CopyFolder(
tag,
destinationSoftLink,
XStoreFileOperationType.CopyFromXStoreToSMB,
copyFlag,
null,
null,
helper,
operationId);
var blobContentEntries = XStoreCommon.GetDownloadContentEntries(operationId);
if (blobContentEntries == null)
{
return;
}
var missingFiles = blobContentEntries.Where(entry => !File.Exists(entry.Item2));
if (missingFiles.Count() > 0)
{
//The missing file count will be traced out and there will be no retrying without remaining time.
//The following step to do will return proper error code after sync up with hosting and image builder.
if (helper != null && helper.GetRemainingTime() == TimeSpan.Zero)
{
this.traceSource.WriteWarning(
ClassName,
"{0} files missing after downloading, OperationID: {1}", missingFiles.Count(), operationId);
return;
}
this.traceSource.WriteWarning(
ClassName,
"Retry to download the {0} missing files, operationID: {1}", missingFiles.Count(), operationId);
foreach (var file in missingFiles)
{
xstoreFileOperation.CopyFile(
file.Item1,
file.Item2,
XStoreFileOperationType.CopyFromXStoreToSMB,
helper);
}
}
}
}
}
catch (Exception ex)
{
this.HandleException(ex);
throw;
}
}
