Beispiel #1
0
        private IEnumerable <TransferEntry> EnumerateLocationNonRecursive(string fileName, CancellationToken cancellationToken)
        {
            Utils.CheckCancellation(cancellationToken);

            if (fileName == null || fileName.Length == 0 || fileName.Length > MaxDirectoryAndFileNameLength)
            {
                // Empty string or exceed-limit-length file name surely match no files.
                yield break;
            }

            if (this.listContinuationToken != null)
            {
                int compareResult = string.Compare(fileName, this.listContinuationToken.FilePath, StringComparison.Ordinal);
                if (compareResult <= 0)
                {
                    yield break;
                }
            }

            CloudFile          cloudFile      = this.location.FileDirectory.GetFileReference(fileName);
            FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions;
            ErrorEntry         errorEntry     = null;
            bool exist = false;

            try
            {
                exist = cloudFile.ExistsAsync(requestOptions, null, cancellationToken).Result;
            }
            catch (Exception ex)
            {
                string errorMessage = string.Format(
                    CultureInfo.CurrentCulture,
                    Resources.FailedToEnumerateDirectory,
                    this.location.FileDirectory.SnapshotQualifiedUri.AbsoluteUri,
                    fileName);

                // Use TransferException to be more specific about the cloud file URI.
                TransferException exception =
                    new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex);

                errorEntry = new ErrorEntry(exception);
            }

            if (null != errorEntry)
            {
                yield return(errorEntry);
            }
            else if (exist)
            {
                yield return(new AzureFileEntry(fileName, cloudFile, new AzureFileListContinuationToken(fileName)));
            }
        }
        /// <summary>
        /// Enumerates the blobs present in the storage location referenced by this object.
        /// </summary>
        /// <param name="cancellationToken">CancellationToken to cancel the method.</param>
        /// <returns>Enumerable list of TransferEntry objects found in the storage location referenced by this object.</returns>
        public IEnumerable <TransferEntry> EnumerateLocation(CancellationToken cancellationToken)
        {
            Utils.CheckCancellation(cancellationToken);

            string filePattern = this.SearchPattern ?? string.Empty;

            // Exceed-limit-length patterns surely match no files.
            int maxFileNameLength = this.GetMaxFileNameLength();

            if (filePattern.Length > maxFileNameLength)
            {
                yield break;
            }

            CloudBlobContainer    container         = this.location.BlobDirectory.Container;
            BlobRequestOptions    requestOptions    = Transfer_RequestOptions.DefaultBlobRequestOptions;
            BlobContinuationToken continuationToken = (this.listContinuationToken == null ? null : this.listContinuationToken.BlobContinuationToken);
            bool   passedContinuationToken          = (this.listContinuationToken == null);
            string dirPrefix     = this.location.BlobDirectory.Prefix;
            string patternPrefix = dirPrefix + filePattern;

            do
            {
                BlobResultSegment resultSegment = null;

                ErrorEntry errorEntry = null;

                Utils.CheckCancellation(cancellationToken);
                try
                {
                    resultSegment = container.ListBlobsSegmentedAsync(
                        patternPrefix,
                        true,
                        BlobListingDetails.Snapshots | BlobListingDetails.Metadata,
                        ListBlobsSegmentSize,
                        continuationToken,
                        requestOptions,
                        null,
                        cancellationToken).Result;
                }
                catch (Exception ex)
                {
                    string errorMessage = string.Format(
                        CultureInfo.CurrentCulture,
                        Resources.FailedToEnumerateDirectory,
                        this.location.BlobDirectory.Uri.AbsoluteUri,
                        filePattern);

                    TransferException exception =
                        new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex);
                    errorEntry = new ErrorEntry(exception);
                }

                if (null != errorEntry)
                {
                    // Just return an error entry if we cannot access the container
                    yield return(errorEntry);

                    // TODO: What should we do if some entries have been listed successfully?
                    yield break;
                }

                foreach (IListBlobItem blobItem in resultSegment.Results)
                {
                    Utils.CheckCancellation(cancellationToken);
                    CloudBlob blob = blobItem as CloudBlob;

                    if (null != blob)
                    {
                        if (!this.IncludeSnapshots && blob.SnapshotTime.HasValue)
                        {
                            continue;
                        }

                        if (!passedContinuationToken)
                        {
                            int compareResult = string.Compare(this.listContinuationToken.BlobName, blob.Name, StringComparison.Ordinal);
                            if (compareResult < 0)
                            {
                                passedContinuationToken = true;
                            }
                            else if (0 == compareResult)
                            {
                                if (IsSnapshotTimeEarlier(this.listContinuationToken.SnapshotTime, blob.SnapshotTime))
                                {
                                    passedContinuationToken = true;
                                }
                            }

                            if (!passedContinuationToken)
                            {
                                continue;
                            }
                        }

                        // TODO: currrently not support search for files with prefix specified without considering sub-directory.
                        bool returnItOrNot = this.Recursive ?
                                             blob.Name.StartsWith(patternPrefix, StringComparison.Ordinal) :
                                             blob.Name.Equals(patternPrefix, StringComparison.Ordinal);

                        if (returnItOrNot)
                        {
                            yield return(new AzureBlobEntry(
                                             blob.Name.Remove(0, dirPrefix.Length),
                                             blob,
                                             new AzureBlobListContinuationToken(continuationToken, blob.Name, blob.SnapshotTime)));
                        }
                    }
                }

                continuationToken = resultSegment.ContinuationToken;
            }while (continuationToken != null);
        }
        /// <summary>
        /// Enumerates the files present in the storage location referenced by this object.
        /// </summary>
        /// <param name="cancellationToken">CancellationToken to cancel the method.</param>
        /// <returns>Enumerable list of TransferEntry objects found in the storage location referenced by this object.</returns>
        public IEnumerable <TransferEntry> EnumerateLocation(CancellationToken cancellationToken)
        {
            Utils.CheckCancellation(cancellationToken);

            string filePattern = string.IsNullOrEmpty(this.SearchPattern) ? DefaultFilePattern : this.SearchPattern;

            SearchOption searchOption = this.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
            IEnumerable <EnumerateDirectoryHelper.LocalEnumerateItem> directoryEnumerator = null;
            ErrorEntry errorEntry = null;

            Utils.CheckCancellation(cancellationToken);

            string fullPath     = null;
            string baseFullPath = null;

            if (Interop.CrossPlatformHelpers.IsWindows)
            {
                fullPath = TransferManager.Configurations.SupportUncPath ?
                           LongPath.ToUncPath(this.location.DirectoryPath) :
                           LongPath.GetFullPath(this.location.DirectoryPath);
                baseFullPath = TransferManager.Configurations.SupportUncPath ?
                               LongPath.ToUncPath(this.baseDirectory) :
                               LongPath.GetFullPath(this.baseDirectory);
            }
            else
            {
                fullPath     = Path.GetFullPath(this.location.DirectoryPath);
                baseFullPath = Path.GetFullPath(this.baseDirectory);
            }

            fullPath     = AppendDirectorySeparator(fullPath);
            baseFullPath = AppendDirectorySeparator(baseFullPath);

            bool isBaseDirectory = string.Equals(fullPath, baseFullPath);

            try
            {
                // Directory.GetFiles/EnumerateFiles will be broken when encounted special items, such as
                // files in recycle bins or the folder "System Volume Information". Rewrite this function
                // because our listing should not be stopped by these unexpected files.
                directoryEnumerator = EnumerateDirectoryHelper.EnumerateAllEntriesInDirectory(
                    fullPath,
                    filePattern,
                    this.listContinuationToken == null ? null : this.listContinuationToken.FilePath,
                    searchOption,
                    followSymlink,
                    isBaseDirectory,
                    cancellationToken);
            }
            catch (Exception ex)
            {
                string errorMessage = string.Format(
                    CultureInfo.CurrentCulture,
                    Resources.FailedToEnumerateDirectory,
                    this.location.DirectoryPath,
                    filePattern);

                TransferException exception =
                    new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex);
                errorEntry = new ErrorEntry(exception);
            }

            if (null != errorEntry)
            {
                // We any exception we might get from Directory.GetFiles/
                // Directory.EnumerateFiles. Just return an error entry
                // to indicate error occured in this case.
                yield return(errorEntry);
            }

            if (null != directoryEnumerator)
            {
                foreach (var entry in directoryEnumerator)
                {
                    Utils.CheckCancellation(cancellationToken);

                    string relativePath = entry.Path;

                    if (relativePath.StartsWith(baseFullPath, StringComparison.OrdinalIgnoreCase))
                    {
                        relativePath = relativePath.Remove(0, baseFullPath.Length);
                    }

                    string continuationTokenPath = entry.Path;
                    continuationTokenPath = continuationTokenPath.Remove(0, fullPath.Length);
                    var continuationToken = new FileListContinuationToken(continuationTokenPath);

                    if (relativePath.Length > Constants.MaxRelativePathLength)
                    {
                        relativePath = relativePath.Substring(0, Constants.MaxRelativePathLength / 2) + "..." + relativePath.Substring(relativePath.Length - Constants.MaxRelativePathLength / 2);
                    }

                    if (entry.IsDirectory)
                    {
                        yield return(new DirectoryEntry(
                                         relativePath,
                                         LongPath.Combine(this.baseDirectory, relativePath),
                                         continuationToken));
                    }
                    else
                    {
                        yield return(new FileEntry(
                                         relativePath,
                                         LongPath.Combine(this.baseDirectory, relativePath),
                                         continuationToken));
                    }
                }
            }
        }
Beispiel #4
0
        private IEnumerable <TransferEntry> EnumerateLocationRecursive(CancellationToken cancellationToken)
        {
            string fullPrefix = null;

            if (null != this.baseDirectory)
            {
                fullPrefix = Uri.UnescapeDataString(this.baseDirectory.SnapshotQualifiedUri.AbsolutePath);
            }
            else
            {
                fullPrefix = Uri.UnescapeDataString(this.location.FileDirectory.SnapshotQualifiedUri.AbsolutePath);
            }

            // Normalize full prefix to end with slash.
            if (!string.IsNullOrEmpty(fullPrefix) && !fullPrefix.EndsWith("/", StringComparison.OrdinalIgnoreCase))
            {
                fullPrefix += '/';
            }

            CloudFileDirectory directory = this.location.FileDirectory;

            Stack <CloudFileDirectory> innerDirList = new Stack <CloudFileDirectory>();

            FileContinuationToken continuationToken = null;
            bool passedContinuationToken            = false;

            if (null == this.listContinuationToken)
            {
                passedContinuationToken = true;
            }

            do
            {
                FileResultSegment resultSegment = null;
                Utils.CheckCancellation(cancellationToken);

                ErrorEntry errorEntry = null;

                try
                {
                    FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions;
                    resultSegment = directory.ListFilesAndDirectoriesSegmentedAsync(
                        ListFilesSegmentSize,
                        continuationToken,
                        requestOptions,
                        null,
                        cancellationToken).Result;
                }
                catch (Exception ex)
                {
                    string errorMessage = string.Format(
                        CultureInfo.CurrentCulture,
                        Resources.FailedToEnumerateDirectory,
                        directory.SnapshotQualifiedUri.AbsoluteUri,
                        string.Empty);

                    TransferException exception =
                        new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex);
                    errorEntry = new ErrorEntry(exception);
                }

                if (null != errorEntry)
                {
                    yield return(errorEntry);

                    yield break;
                }

                continuationToken = resultSegment.ContinuationToken;

                foreach (IListFileItem fileItem in resultSegment.Results)
                {
                    Utils.CheckCancellation(cancellationToken);

                    if (fileItem is CloudFileDirectory)
                    {
                        CloudFileDirectory cloudDir = fileItem as CloudFileDirectory;

                        if (!passedContinuationToken)
                        {
                            if (string.Equals(cloudDir.Name, this.listContinuationToken.FilePath, StringComparison.Ordinal))
                            {
                                passedContinuationToken = true;
                                continue;
                            }
                            else
                            {
                                continue;
                            }
                        }
                        string fullPath     = Uri.UnescapeDataString(cloudDir.SnapshotQualifiedUri.AbsolutePath);
                        string relativePath = fullPath.Remove(0, fullPrefix.Length);

                        yield return(new AzureFileDirectoryEntry(
                                         relativePath,
                                         cloudDir,
                                         new AzureFileListContinuationToken(cloudDir.Name)));
                    }
                    else if (fileItem is CloudFile)
                    {
                        CloudFile cloudFile = fileItem as CloudFile;

                        if (!passedContinuationToken)
                        {
                            if (string.Equals(cloudFile.Name, this.listContinuationToken.FilePath, StringComparison.Ordinal))
                            {
                                passedContinuationToken = true;
                                continue;
                            }
                            else
                            {
                                continue;
                            }
                        }

                        string fullPath     = Uri.UnescapeDataString(cloudFile.SnapshotQualifiedUri.AbsolutePath);
                        string relativePath = fullPath.Remove(0, fullPrefix.Length);

                        yield return(new AzureFileEntry(
                                         relativePath,
                                         cloudFile,
                                         new AzureFileListContinuationToken(cloudFile.Name)));
                    }
                }
            }while (continuationToken != null);
        }
Beispiel #5
0
        private IEnumerable <TransferEntry> EnumerateLocationRecursive(CancellationToken cancellationToken)
        {
            string fullPrefix = Uri.UnescapeDataString(this.location.FileDirectory.SnapshotQualifiedUri.AbsolutePath);

            // Normalize full prefix to end with slash.
            if (!string.IsNullOrEmpty(fullPrefix) && !fullPrefix.EndsWith("/", StringComparison.OrdinalIgnoreCase))
            {
                fullPrefix += '/';
            }

            Stack <CloudFileDirectory> directoriesToList = new Stack <CloudFileDirectory>();

            directoriesToList.Push(this.location.FileDirectory);

            string[] pathSegList             = null;
            bool     passedContinuationToken = false;
            int      pathSegListIndex        = 0;

            if (null != this.listContinuationToken)
            {
                pathSegList = this.listContinuationToken.FilePath.Split(new char[] { UriDelimiter });
            }
            else
            {
                passedContinuationToken = true;
            }

            while (0 != directoriesToList.Count)
            {
                CloudFileDirectory directory       = directoriesToList.Pop();
                string             dirAbsolutePath = Uri.UnescapeDataString(directory.SnapshotQualifiedUri.AbsolutePath);
                if (dirAbsolutePath[dirAbsolutePath.Length - 1] != UriDelimiter)
                {
                    dirAbsolutePath = dirAbsolutePath + UriDelimiter;
                }

                Stack <CloudFileDirectory> innerDirList = new Stack <CloudFileDirectory>();

                FileContinuationToken continuationToken = null;

                // To check whether reached continuation token by dir or file in this round.
                bool   checkFile            = false;
                bool   passedSubFolder      = false;
                string continuationTokenSeg = null;
                if (!passedContinuationToken)
                {
                    if (pathSegList.Length - 1 == pathSegListIndex)
                    {
                        checkFile = true;
                    }

                    continuationTokenSeg = pathSegList[pathSegListIndex];
                    pathSegListIndex++;
                }

                do
                {
                    FileResultSegment resultSegment = null;
                    Utils.CheckCancellation(cancellationToken);

                    ErrorEntry errorEntry = null;

                    try
                    {
                        FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions;
                        resultSegment = directory.ListFilesAndDirectoriesSegmentedAsync(
                            ListFilesSegmentSize,
                            continuationToken,
                            requestOptions,
                            null,
                            cancellationToken).Result;
                    }
                    catch (Exception ex)
                    {
                        string errorMessage = string.Format(
                            CultureInfo.CurrentCulture,
                            Resources.FailedToEnumerateDirectory,
                            directory.SnapshotQualifiedUri.AbsoluteUri,
                            string.Empty);

                        TransferException exception =
                            new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex);
                        errorEntry = new ErrorEntry(exception);
                    }

                    if (null != errorEntry)
                    {
                        yield return(errorEntry);

                        yield break;
                    }

                    continuationToken = resultSegment.ContinuationToken;

                    foreach (IListFileItem fileItem in resultSegment.Results)
                    {
                        Utils.CheckCancellation(cancellationToken);

                        if (fileItem is CloudFileDirectory)
                        {
                            if (checkFile || passedContinuationToken || passedSubFolder)
                            {
                                innerDirList.Push(fileItem as CloudFileDirectory);
                            }
                            else
                            {
                                CloudFileDirectory cloudDir = fileItem as CloudFileDirectory;
                                string             fullPath = Uri.UnescapeDataString(cloudDir.SnapshotQualifiedUri.AbsolutePath);
                                string             segName  = fullPath.Remove(0, dirAbsolutePath.Length);

                                int compareResult = string.Compare(segName, continuationTokenSeg, StringComparison.OrdinalIgnoreCase);

                                if (compareResult >= 0)
                                {
                                    passedSubFolder = true;
                                    innerDirList.Push(cloudDir);

                                    if (compareResult > 0)
                                    {
                                        passedContinuationToken = true;
                                    }
                                }
                            }
                        }
                        else if (fileItem is CloudFile)
                        {
                            if (!checkFile && !passedContinuationToken)
                            {
                                continue;
                            }

                            CloudFile cloudFile = fileItem as CloudFile;

                            string fullPath     = Uri.UnescapeDataString(cloudFile.SnapshotQualifiedUri.AbsolutePath);
                            string relativePath = fullPath.Remove(0, fullPrefix.Length);

                            if (passedContinuationToken)
                            {
                                yield return(new AzureFileEntry(
                                                 relativePath,
                                                 cloudFile,
                                                 new AzureFileListContinuationToken(relativePath)));
                            }
                            else
                            {
                                string segName       = fullPath.Remove(0, dirAbsolutePath.Length);
                                int    compareResult = string.Compare(segName, continuationTokenSeg, StringComparison.OrdinalIgnoreCase);

                                if (compareResult < 0)
                                {
                                    continue;
                                }

                                passedContinuationToken = true;

                                if (compareResult > 0)
                                {
                                    yield return(new AzureFileEntry(
                                                     relativePath,
                                                     cloudFile,
                                                     new AzureFileListContinuationToken(relativePath)));
                                }
                            }
                        }
                    }
                }while (continuationToken != null);

                if (checkFile)
                {
                    passedContinuationToken = true;
                }

                if (innerDirList.Count <= 0)
                {
                    if (!checkFile && !passedContinuationToken)
                    {
                        passedContinuationToken = true;
                    }
                }
                else
                {
                    while (innerDirList.Count > 0)
                    {
                        directoriesToList.Push(innerDirList.Pop());
                    }
                }
            }
        }