/// <summary>
        /// Find the corresponding blob uri based on the smb path
        /// SMB path is \\XXX\YYY; XStore path is XXX/YYY
        /// </summary>
        /// <param name="smbPath">The smb path to be converted.</param>
        /// <param name="srcRootUri">The source root uri.</param>
        /// <param name="dstRootUri">The destination root uri.</param>
        /// <returns>The blob uri</returns>
        private static string ConvertSMBPathToBlobReference(string smbPath, string srcRootUri, string dstRootUri)
        {
            smbPath = XStoreCommon.FormatSMBUri(smbPath);
            if (smbPath.StartsWith(srcRootUri, StringComparison.OrdinalIgnoreCase))
            {
                smbPath = smbPath.Substring(srcRootUri.Length);
            }

            return(dstRootUri + smbPath.Replace('\\', '/'));
        }
Ejemplo n.º 2
0
        public void UploadContent(string tag, string sourceSoftLink, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag, bool acquireSourceReaderLock)
        {
            TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            sourceSoftLink = this.GetLocalPath(sourceSoftLink);
            if ((!FabricFile.Exists(sourceSoftLink)) &&
                (!FabricDirectory.Exists(sourceSoftLink)))
            {
                throw new IOException(
                          string.Format(
                              CultureInfo.CurrentCulture,
                              StringResources.ImageStoreError_DoesNotExistError,
                              sourceSoftLink));
            }

            try
            {
                using (XStoreFileOperations xstoreFileOperation =
                           new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    tag = XStoreCommon.FormatXStoreUri(tag);
                    if (copyFlag == CopyFlag.AtomicCopy)
                    {
                        this.DeleteContent(tag, helper == null ? timeout : helper.GetRemainingTime());
                    }

                    if (FabricFile.Exists(sourceSoftLink))
                    {
                        // This is a file copy
                        xstoreFileOperation.CopyFile(
                            sourceSoftLink,
                            tag,
                            XStoreFileOperationType.CopyFromSMBToXStore,
                            helper);
                    }
                    else
                    {
                        // this is a folder copy
                        xstoreFileOperation.CopyFolder(
                            sourceSoftLink,
                            tag,
                            XStoreFileOperationType.CopyFromSMBToXStore,
                            copyFlag,
                            null,
                            null,
                            helper);
                    }
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }
        }
        private static string ConvertBlobReferenceToBlobReference(string blobReference, string srcRootUri, string dstRootUri)
        {
            char[] trims = { '/', ' ' };
            blobReference = XStoreCommon.FormatXStoreUri(blobReference);
            if (blobReference.StartsWith(srcRootUri, StringComparison.OrdinalIgnoreCase))
            {
                blobReference = blobReference.Substring(srcRootUri.Length).Trim(trims);
            }

            return(string.Format("{0}/{1}", dstRootUri, blobReference));
        }
        /// <summary>
        /// Find the corresponding smb path based on the blob uri
        /// SMB path is \\XXX\YYY; XStore path is XXX/YYY
        /// </summary>
        /// <returns>The SMB path corresponding to the XStore Path.</returns>
        /// <param name="blobReference">The reference to blob.</param>
        /// <param name="srcRootUri">The root uri of the source.</param>
        /// <param name="dstRootUri">The root uri of the destination.</param>
        private static string ConvertBlobReferenceToSMBPath(string blobReference, string srcRootUri, string dstRootUri)
        {
            char[] trims = { '/', ' ' };
            blobReference = XStoreCommon.FormatXStoreUri(blobReference);
            if (blobReference.StartsWith(srcRootUri, StringComparison.OrdinalIgnoreCase))
            {
                blobReference = blobReference.Substring(srcRootUri.Length).Trim(trims);
            }

            return(Path.Combine(dstRootUri, blobReference.Replace('/', '\\')));
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Copies content from a path in the XStore to the destination path in the XStore
        /// </summary>
        /// <param name="remoteSource"> Location relative to RootUri where the content to be copied will be found. </param>
        /// <param name="remoteDestination"> Location relative to RootUri where the content needs to be copied. </param>
        /// <param name="timeout">The timeout for performing the copying operation.</param>
        /// <param name="skipFiles">The list of the blobs to be skipped for copying </param>
        /// <param name="copyFlag">The copying control information to specify how blob can be overwritten.</param>
        /// <param name="checkMarkFile">Indicats whether to check mark file during copying.</param>
        public void CopyContent(string remoteSource, string remoteDestination, TimeSpan timeout, string[] skipFiles, CopyFlag copyFlag, bool checkMarkFile)
        {
            TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            try
            {
                using (
                    XStoreFileOperations xstoreFileOperation =
                        new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    remoteSource      = XStoreCommon.FormatXStoreUri(remoteSource);
                    remoteDestination = XStoreCommon.FormatXStoreUri(remoteDestination);

                    bool xstoreFileExist   = xstoreFileOperation.XStoreFileExists(remoteSource, helper);
                    bool xstoreFolderExist = xstoreFileOperation.XStoreFolderExists(remoteSource, checkMarkFile, helper);

                    if ((!xstoreFileExist) && (!xstoreFolderExist))
                    {
                        throw new FileNotFoundException(string.Format(CultureInfo.InvariantCulture, StringResources.ImageStoreError_DoesNotExistError, remoteSource));
                    }

                    if (copyFlag == CopyFlag.AtomicCopy)
                    {
                        this.DeleteContent(remoteDestination, helper == null ? timeout : helper.GetRemainingTime());
                    }

                    if (xstoreFileExist)
                    {
                        // this is a single blob copy from xstore to xstore
                        xstoreFileOperation.CopyFile(
                            remoteSource,
                            remoteDestination,
                            XStoreFileOperationType.CopyFromXStoreToXStore,
                            helper);
                    }
                    else
                    {
                        xstoreFileOperation.CopyFolder(
                            remoteSource,
                            remoteDestination,
                            XStoreFileOperationType.CopyFromXStoreToXStore,
                            copyFlag,
                            null,
                            skipFiles,
                            helper);
                    }
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }
        }
Ejemplo n.º 6
0
            /// <summary>
            /// Initializes a new instance of the XStoreProviderParameters class.
            /// </summary>
            /// <param name="imageStoreUri">Image store uri: format = "xstore:{connection string};Container={2}"</param>
            public XStoreProviderParameters(string imageStoreUri)
            {
                string accountName, endpointSuffix;

                this.imageStoreUri = imageStoreUri;
                XStoreCommon.TryParseImageStoreUri(
                    imageStoreUri,
                    out this.connectionString,
                    out this.secondaryConnectionString,
                    out this.blobEndpoint,
                    out this.containerName,
                    out accountName,
                    out endpointSuffix);
            }
Ejemplo n.º 7
0
        /// <summary>
        /// Downloads content from the XStore to local destination.
        /// </summary>
        /// <param name="tag">Location (relative to RootUri) from where to download the content.</param>
        /// <param name="destinationSoftLink">Physical location where to download the content.</param>
        /// <param name="handler">Defines the behavior to process progress information from the downloading operation.</param>
        /// <param name="timeout">The timeout for performing the downloading operation.</param>
        /// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
        public void DownloadContent(string tag, string destinationSoftLink, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
        {
            TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            destinationSoftLink = this.GetLocalPath(destinationSoftLink);
            try
            {
                using (XStoreFileOperations xstoreFileOperation =
                           new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    tag = XStoreCommon.FormatXStoreUri(tag);
                    bool xstoreFileExist   = xstoreFileOperation.XStoreFileExists(tag, helper);
                    bool xstoreFolderExist = xstoreFileOperation.XStoreFolderExists(tag, true, helper);

                    if ((!xstoreFileExist) && (!xstoreFolderExist))
                    {
                        throw new FileNotFoundException(string.Format(CultureInfo.InvariantCulture, StringResources.ImageStoreError_DoesNotExistError, tag));
                    }

                    if (xstoreFileExist)
                    {
                        // this is a single blob copy from xstore to smb
                        xstoreFileOperation.CopyFile(
                            tag,
                            destinationSoftLink,
                            XStoreFileOperationType.CopyFromXStoreToSMB,
                            helper);
                    }
                    else
                    {
                        // this is a multiple blob (folder) copy from xstore to smb
                        xstoreFileOperation.CopyFolder(
                            tag,
                            destinationSoftLink,
                            XStoreFileOperationType.CopyFromXStoreToSMB,
                            copyFlag,
                            null,
                            null,
                            helper);
                    }
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }
        }
        /// <summary>
        /// Post-processing of copying (from XStore to XStore)
        /// </summary>
        /// <param name="blobContainer">The blob container from which copy is happening.</param>
        /// <param name="task">The task to end.</param>
        private static void EndCopyFromXStoreToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
        {
            if (task.IsSucceeded)
            {
                // Generate a manifest on the xstore for the copied folder - only need for atomic copy
                // When doing atomic copy we have following assumptions:
                // 1) only one writer
                // 2) we don't overwrite to the exisiting folder
                // 3) later this folder is copied to smb with the exact path, no sub folder or parent folder
                // therefore it is enough to only generate a manifest file on the folder only
                var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
                blob.UploadText(DateTime.Now.ToString());
#else
                blob.UploadTextAsync(DateTime.Now.ToString()).Wait();
#endif
            }
        }
Ejemplo n.º 9
0
        /// <summary>
        /// List all blobs under the the given relative blob path/sub blob path
        /// </summary>
        /// <param name="tag">Location (relative to RootUri)</param>
        /// <param name="isRecursive">flag list subhierarchy</param>
        /// <param name="timeout">The timeout for performing the listing operation</param>
        /// <returns>All blobs within the input blob path/sub blob path</returns>
        public ImageStoreContent ListContentWithDetails(string tag, bool isRecursive, TimeSpan timeout)
        {
            ImageStoreContent content = null;
            TimeoutHelper     helper  = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            try
            {
                using (XStoreFileOperations xstoreFileOperation =
                           new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    content = xstoreFileOperation.ListXStoreFile(XStoreCommon.FormatXStoreUri(tag), isRecursive, helper);
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }

            return(content);
        }
Ejemplo n.º 10
0
        /// <summary>
        /// Remove tag from store, clear data assosciate with tag.
        /// </summary>
        /// <param name="tag">Location (relative to RootUri) from where to delete the content.</param>
        ///<param name="timeout">The timeout for performing the deleting operation</param>
        public void DeleteContent(string tag, TimeSpan timeout)
        {
            TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            try
            {
                using (
                    XStoreFileOperations xstoreFileOperation =
                        new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    tag = XStoreCommon.FormatXStoreUri(tag);

                    if (!xstoreFileOperation.XStoreContainerExists(helper))
                    {
                        return;
                    }

                    if (string.IsNullOrEmpty(tag))
                    {
                        // remove the container in this case
                        xstoreFileOperation.RemoveContainer(helper);
                    }
                    else if (xstoreFileOperation.XStoreFileExists(tag, helper))
                    {
                        xstoreFileOperation.RemoveXStoreFile(tag, helper);
                    }
                    else
                    {
                        xstoreFileOperation.RemoveXStoreFolder(tag, helper);
                    }
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }
        }
Ejemplo n.º 11
0
        /// <summary>
        /// Check if content is present in the store.
        /// </summary>
        /// <param name="tag"> Location (relative to RootUri) where to check the presence of the content. </param>
        /// <param name="timeout">The timeout for performing the checking existence operation</param>
        /// <returns> True if the content exists, false otherwise. </returns>
        public bool DoesContentExist(string tag, TimeSpan timeout)
        {
            TimeoutHelper helper  = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
            bool          isExist = false;

            try
            {
                using (
                    XStoreFileOperations xstoreFileOperation =
                        new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    tag     = XStoreCommon.FormatXStoreUri(tag);
                    isExist = xstoreFileOperation.XStoreFileExists(tag, helper) || xstoreFileOperation.XStoreFolderExists(tag, true, helper);
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }

            return(isExist);
        }
        /// <summary>
        /// Processing folder
        /// </summary>
        /// <param name="blobContainer">The blob container.</param>
        /// <param name="task">The task to be performed.</param>
        private void HandleFolder(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
        {
            // expand the task and enqueue new tasks
            string[] files;
            string[] subFolders;

            // See if we need to handle the folder
            if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB)
            {
                if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists && FabricDirectory.Exists(task.DstUri))
                {
                    // We will skip the copy because the final destination exists
                    return;
                }
                else if ((task.FileCopyFlag == CopyFlag.AtomicCopy ||
                          task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists) &&
                         task.IsRoot)
                {
                    XStoreFileOperationTask endTask = new XStoreFileOperationTask(
                        XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB,
                        task.SrcUri,
                        task.DstUri,
                        true,
                        0,
                        task.TimeoutHelper);

                    endTask.IsSucceeded  = true;
                    endTask.FileCopyFlag = task.FileCopyFlag;
                    this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
                    task.DstUri = task.DstUri + NewExtension;

                    // step1: delete dstFolder.new
                    if (FabricDirectory.Exists(task.DstUri))
                    {
                        FabricDirectory.Delete(task.DstUri, recursive: true, deleteReadOnlyFiles: true);
                    }
                }
            }
            else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore)
            {
                if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists &&
                    XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null))
                {
                    // We will skip the copy because the final destination exists
                    return;
                }
                else if (task.IsRoot)
                {
                    // if this is the root uri of this operation
                    XStoreFileOperationTask endTask = new XStoreFileOperationTask(
                        XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore,
                        task.SrcUri,
                        task.DstUri,
                        true,
                        0,
                        task.TimeoutHelper);

                    endTask.IsSucceeded  = true;
                    endTask.FileCopyFlag = task.FileCopyFlag;
                    this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);

                    // remove the manifest of this folder
                    var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
                    blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null);
#else
                    blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null, null).Wait();
#endif
                }
            }
            else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore)
            {
                if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists &&
                    XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null))
                {
                    // We will skip the copy because the final destination exists
                    return;
                }
                else if (task.IsRoot)
                {
                    // if this is the root uri of this operation
                    XStoreFileOperationTask endTask = new XStoreFileOperationTask(
                        XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore,
                        task.SrcUri,
                        task.DstUri,
                        true,
                        0,
                        task.TimeoutHelper);

                    endTask.IsSucceeded  = true;
                    endTask.FileCopyFlag = task.FileCopyFlag;

                    this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);

                    // remove the manifest of this folder
                    var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));

#if !DotNetCoreClr
                    blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper));
#else
                    blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper), null).Wait();
#endif
                }
            }

            string dstUri = null;

            switch (task.OpType)
            {
            case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
            case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
            case XStoreFileOperationTask.XStoreTaskType.RemoveFromXStore:
                this.GetFilesAndSubfoldersFromXStore(blobContainer, task.SrcUri, out files, out subFolders, task.TimeoutHelper);
                break;

            case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
            default:
                GetFilesAndSubfoldersFromSMB(task.SrcUri, out files, out subFolders);
                break;
            }

            Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
            foreach (var file in files)
            {
                switch (task.OpType)
                {
                case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
                    dstUri = ConvertBlobReferenceToSMBPath(file, task.SrcUri, task.DstUri);
                    if (task.OperationId != null)
                    {
                        XStoreCommon.AddDownloadContentEntry(task.OperationId.GetValueOrDefault(), file, dstUri);
                    }
                    break;

                case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
                    dstUri = ConvertSMBPathToBlobReference(file, task.SrcUri, task.DstUri);
                    break;

                case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
                    dstUri = ConvertBlobReferenceToBlobReference(file, task.SrcUri, task.DstUri);
                    break;
                }

                XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, file, dstUri, false, 0, task.TimeoutHelper, task.OperationId);
                newTask.FileCopyFlag = task.FileCopyFlag;
                tasks.Enqueue(newTask);
            }

            foreach (var folder in subFolders)
            {
                switch (task.OpType)
                {
                case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
                    dstUri = ConvertBlobReferenceToSMBPath(folder, task.SrcUri, task.DstUri);
                    break;

                case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
                    dstUri = ConvertSMBPathToBlobReference(folder, task.SrcUri, task.DstUri);
                    break;

                case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
                    dstUri = ConvertBlobReferenceToBlobReference(folder, task.SrcUri, task.DstUri);
                    break;
                }

                XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, folder, dstUri, true, 0, task.TimeoutHelper);
                newTask.FileCopyFlag = task.FileCopyFlag;
                tasks.Enqueue(newTask);
            }

            this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
        }
        /// <summary>
        /// Download file from XStore to SMB
        /// </summary>
        /// <param name="blobContainer">The blob container having the file.</param>
        /// <param name="task">The task to be performed.</param>
        private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
        {
            if (!this.ShouldCopy(task.SrcUri))
            {
                return;
            }

            // download the file
            if (task.PartialID >= 0)
            {
                // This is already a divided work; let's go ahead and download
                string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID);

                // Delete the file if it exists
                string directoryName = FabricPath.GetDirectoryName(dstSMBPath);
                if (!FabricDirectory.Exists(directoryName))
                {
                    FabricDirectory.CreateDirectory(directoryName);
                }

                if (FabricFile.Exists(dstSMBPath))
                {
                    FabricFile.Delete(dstSMBPath, deleteReadonly: true);
                }

                var blob        = blobContainer.GetBlockBlobReference(task.SrcUri);
                var blobWrapper = new XStoreBlobWrapper(blob);
                if (task.TimeoutHelper != null)
                {
                    //task.TimeoutHelper.ThrowIfExpired();
                    blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime());
                }
                else
                {
                    blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue);
                }
            }
            else
            {
                // we are going to download a file, which we don't know the size yet
                var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
                blob.FetchAttributes(null, this.defaultRequestOption);
#else
                blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait();
#endif
                int blobLength = (int)blob.Properties.Length;

                // Delete the file if it exists
                string directoryName = FabricPath.GetDirectoryName(task.DstUri);
                if (!FabricDirectory.Exists(directoryName))
                {
                    FabricDirectory.CreateDirectory(directoryName);
                }

                if (FabricFile.Exists(task.DstUri))
                {
                    FabricFile.Delete(task.DstUri, deleteReadonly: true);
                }

                if (blobLength < DownloadSizeThreshold)
                {
                    var blobWrapper = new XStoreBlobWrapper(blob);
                    if (task.TimeoutHelper != null)
                    {
                        blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext);
                    }
                    else
                    {
                        blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext);
                    }

                    return;
                }
                else
                {
                    // For large files we divided the work to couple threads.
                    int numThreads = Math.Min(
                        blobLength / ParrallelDownloadSize,
                        ParrallelDownloadThreadCount);

                    // Create new tasks to parallel download
                    Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
                    int offset = 0;
                    for (int i = 0; i < numThreads; i++)
                    {
                        int length;
                        if (i < numThreads - 1)
                        {
                            length = blobLength / numThreads;
                        }
                        else
                        {
                            length = blobLength - offset;
                        }

                        XStoreFileOperationTask newTask = new XStoreFileOperationTask(
                            XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB,
                            task.SrcUri,
                            task.DstUri,
                            false,
                            0,
                            task.TimeoutHelper);

                        newTask.FileCopyFlag = task.FileCopyFlag;
                        newTask.Offset       = offset;
                        newTask.Length       = length;
                        newTask.PartialID    = i;

                        tasks.Enqueue(newTask);
                        offset += length;
                    }

                    // enqueue all divided tasks
                    this.xstoreTaskPool.AddTaskToTaskQueue(tasks);

                    // Add an EndTask to the endTaskQueue as well; to combine all downloads
                    XStoreFileOperationTask endTask = new XStoreFileOperationTask(
                        XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles,
                        task.SrcUri,
                        task.DstUri,
                        false,
                        0,
                        task.TimeoutHelper);

                    endTask.IsSucceeded  = true;
                    endTask.FileCopyFlag = task.FileCopyFlag;
                    endTask.PartialID    = numThreads;
                    endTask.Content      = task.DstUri;
                    this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
                }
            }
        }
Ejemplo n.º 14
0
 /// <summary>
 /// Checks whether the uri is of the xstore store.
 /// </summary>
 /// <param name="imageStoreUri">The uri of the image store.</param>
 /// <returns>True if the uri is of a xstore store and false otherwise.</returns>
 public static bool IsXStoreUri(string imageStoreUri)
 {
     return(XStoreCommon.IsXStoreUri(imageStoreUri));
 }
Ejemplo n.º 15
0
        /// <summary>
        /// Downloads content from the XStore to local destination.
        /// </summary>
        /// <param name="tag">Location (relative to RootUri) from where to download the content.</param>
        /// <param name="destinationSoftLink">Physical location where to download the content.</param>
        /// <param name="handler">Defines the behavior to process progress information from the downloading operation.</param>
        /// <param name="timeout">The timeout for performing the downloading operation.</param>
        /// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
        public void DownloadContent(string tag, string destinationSoftLink, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
        {
            TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);

            destinationSoftLink = this.GetLocalPath(destinationSoftLink);
            try
            {
                using (XStoreFileOperations xstoreFileOperation =
                           new XStoreFileOperations(new XStoreParameters(this.providerParams.ConnectionString, this.providerParams.SecondaryConnectionString, this.providerParams.ContainerName)))
                {
                    tag = XStoreCommon.FormatXStoreUri(tag);
                    bool xstoreFileExist   = xstoreFileOperation.XStoreFileExists(tag, helper);
                    bool xstoreFolderExist = xstoreFileOperation.XStoreFolderExists(tag, true, helper);

                    if ((!xstoreFileExist) && (!xstoreFolderExist))
                    {
                        throw new FileNotFoundException(string.Format(CultureInfo.InvariantCulture, StringResources.ImageStoreError_DoesNotExistError, tag));
                    }

                    if (xstoreFileExist)
                    {
                        // this is a single blob copy from xstore to smb
                        xstoreFileOperation.CopyFile(
                            tag,
                            destinationSoftLink,
                            XStoreFileOperationType.CopyFromXStoreToSMB,
                            helper);
                    }
                    else
                    {
                        var operationId = Guid.NewGuid();

                        // this is a multiple blob (folder) copy from xstore to smb
                        xstoreFileOperation.CopyFolder(
                            tag,
                            destinationSoftLink,
                            XStoreFileOperationType.CopyFromXStoreToSMB,
                            copyFlag,
                            null,
                            null,
                            helper,
                            operationId);

                        var blobContentEntries = XStoreCommon.GetDownloadContentEntries(operationId);
                        if (blobContentEntries == null)
                        {
                            return;
                        }

                        var missingFiles = blobContentEntries.Where(entry => !File.Exists(entry.Item2));
                        if (missingFiles.Count() > 0)
                        {
                            //The missing file count will be traced out and there will be no retrying without remaining time.
                            //The following step to do will return proper error code after sync up with hosting and image builder.
                            if (helper != null && helper.GetRemainingTime() == TimeSpan.Zero)
                            {
                                this.traceSource.WriteWarning(
                                    ClassName,
                                    "{0} files missing after downloading, OperationID: {1}", missingFiles.Count(), operationId);
                                return;
                            }

                            this.traceSource.WriteWarning(
                                ClassName,
                                "Retry to download the {0} missing files, operationID: {1}", missingFiles.Count(), operationId);

                            foreach (var file in missingFiles)
                            {
                                xstoreFileOperation.CopyFile(
                                    file.Item1,
                                    file.Item2,
                                    XStoreFileOperationType.CopyFromXStoreToSMB,
                                    helper);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                this.HandleException(ex);
                throw;
            }
        }