Ejemplo n.º 1
0
        private void PrepareForCopy()
        {
            var calculatedBlockSize = ((long)Math.Ceiling((double)this.SourceHandler.TotalLength / 50000 / Constants.DefaultTransferChunkSize)) * Constants.DefaultTransferChunkSize;

            this.blockSize = Math.Max(TransferManager.Configurations.BlockSize, calculatedBlockSize);

            SingleObjectCheckpoint checkpoint = this.TransferJob.CheckPoint;

            if (null != checkpoint.TransferWindow)
            {
                this.lastTransferWindow = new Queue <long>(checkpoint.TransferWindow);
            }

            this.blockIds      = new SortedDictionary <int, string>();
            this.BlockIdPrefix = GenerateBlockIdPrefix();
            this.InitializeBlockIds();
            int blockCount = null == this.lastTransferWindow ? 0 : this.lastTransferWindow.Count + (int)Math.Ceiling((double)(this.SourceHandler.TotalLength - checkpoint.EntryTransferOffset) / this.blockSize);

            if (0 == blockCount)
            {
                this.state = State.Commit;
            }
            else
            {
                this.countdownEvent = new CountdownEvent(blockCount);
                // Handle record overwrite here.
                this.state = State.Copy;
            }
        }
Ejemplo n.º 2
0
        private void InitializeCopyStatus()
        {
            int pageLength = TransferManager.Configurations.BlockSize;

            SingleObjectCheckpoint checkpoint = this.TransferJob.CheckPoint;

            if ((null != checkpoint.TransferWindow) &&
                (0 != checkpoint.TransferWindow.Count))
            {
                this.lastTransferWindow = new Queue <long>(checkpoint.TransferWindow);
            }

            int blockCount = (null == this.lastTransferWindow ? 0 : this.lastTransferWindow.Count)
                             + (int)Math.Ceiling((double)(totalLength - checkpoint.EntryTransferOffset) / pageLength);

            this.countdownEvent = new CountdownEvent(blockCount);
        }
        private async Task HandleFetchAttributesResultAsync(bool existingBlob)
        {
            this.destLocation.CheckedAccessCondition = true;

            if (!this.Controller.IsForceOverwrite)
            {
                // If destination file exists, query user whether to overwrite it.
                await this.Controller.CheckOverwriteAsync(
                    existingBlob,
                    this.SharedTransferData.TransferJob.Source.Instance,
                    this.appendBlob).ConfigureAwait(false);
            }

            this.Controller.UpdateProgressAddBytesTransferred(0);

            if (existingBlob)
            {
                if (this.appendBlob.Properties.BlobType == BlobType.Unspecified)
                {
                    throw new InvalidOperationException(Resources.FailedToGetBlobTypeException);
                }

                if (this.appendBlob.Properties.BlobType != BlobType.AppendBlob)
                {
                    throw new InvalidOperationException(Resources.DestinationBlobTypeNotMatch);
                }
            }

            // We do check point consistency validation in reader, so directly use it here.
            SingleObjectCheckpoint checkpoint = this.SharedTransferData.TransferJob.CheckPoint;

            if ((null != checkpoint.TransferWindow) &&
                (checkpoint.TransferWindow.Any()))
            {
                checkpoint.TransferWindow.Sort();
                this.expectedOffset = checkpoint.TransferWindow[0];
            }
            else
            {
                this.expectedOffset = checkpoint.EntryTransferOffset;
            }

            if (0 == this.expectedOffset)
            {
                this.state = State.Create;
            }
            else
            {
                if (!this.Controller.IsForceOverwrite && !existingBlob)
                {
                    throw new TransferException(Resources.DestinationChangedException);
                }

                this.PreProcessed = true;

                if (this.expectedOffset == this.SharedTransferData.TotalLength)
                {
                    this.state = State.Commit;
                }
                else
                {
                    this.state = State.UploadBlob;
                }
            }

            this.workToken = 1;
        }
        private async Task FetchAttributesAsync()
        {
            Debug.Assert(
                this.state == State.FetchAttributes,
                "FetchAttributesAsync called, but state isn't FetchAttributes",
                "Current state is {0}",
                this.state);

            this.hasWork = false;

            this.CheckInputStreamLength(this.SharedTransferData.TotalLength);

            bool exist = true;

            try
            {
                await this.DoFetchAttributesAsync();
            }
            catch (StorageException se)
            {
                // Getting a storage exception is expected if the file doesn't
                // exist. In this case we won't error out, but set the
                // exist flag to false to indicate we're uploading
                // a new file instead of overwriting an existing one.
                if (null != se.RequestInformation &&
                    se.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
                {
                    exist = false;
                }
                else
                {
                    this.HandleFetchAttributesResult(se);
                    throw;
                }
            }
            catch (Exception e)
            {
                this.HandleFetchAttributesResult(e);
                throw;
            }

            if (this.TransferJob.Destination.Type == TransferLocationType.AzureBlob)
            {
                (this.TransferJob.Destination as AzureBlobLocation).CheckedAccessCondition = true;
            }
            else
            {
                (this.TransferJob.Destination as AzureFileLocation).CheckedAccessCondition = true;
            }

            this.Controller.CheckOverwrite(
                exist,
                this.SharedTransferData.SourceLocation,
                this.DestUri.ToString());

            this.Controller.UpdateProgressAddBytesTransferred(0);

            if (exist)
            {
                // If the destination has already existed,
                // and if we haven't uploaded anything to it, try to resize it to the expected length.
                // Or if we have uploaded something, the destination should be created by the last transferring,
                // don't do resize again.
                SingleObjectCheckpoint checkpoint = this.TransferJob.CheckPoint;
                bool shouldResize = (checkpoint.EntryTransferOffset == 0) && (!checkpoint.TransferWindow.Any());

                if (shouldResize)
                {
                    this.state = State.Resize;
                }
                else
                {
                    this.InitUpload();
                }
            }
            else
            {
                this.state = State.Create;
            }

            this.hasWork = true;
        }
Ejemplo n.º 5
0
        private void AddRangesByCheckPoint(long startOffset, long endOffset, bool hasData, ref bool reachLastTransferOffset, ref int lastTransferWindowIndex)
        {
            SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint;

            if (reachLastTransferOffset)
            {
                this.rangeList.AddRange(
                    new Range
                {
                    StartOffset = startOffset,
                    EndOffset   = endOffset,
                    HasData     = hasData,
                }.SplitRanges(Constants.DefaultBlockSize));
            }
            else
            {
                Range range = new Range()
                {
                    StartOffset = -1,
                    HasData     = hasData
                };

                while (lastTransferWindowIndex < checkpoint.TransferWindow.Count)
                {
                    long lastTransferWindowStart = checkpoint.TransferWindow[lastTransferWindowIndex];
                    long lastTransferWindowEnd   = Math.Min(checkpoint.TransferWindow[lastTransferWindowIndex] + this.SharedTransferData.BlockSize - 1, this.SharedTransferData.TotalLength);

                    if (lastTransferWindowStart <= endOffset)
                    {
                        if (-1 == range.StartOffset)
                        {
                            // New range
                            range.StartOffset = Math.Max(lastTransferWindowStart, startOffset);
                            range.EndOffset   = Math.Min(lastTransferWindowEnd, endOffset);
                        }
                        else
                        {
                            if (range.EndOffset != lastTransferWindowStart - 1)
                            {
                                // Store the previous range and create a new one
                                this.rangeList.AddRange(range.SplitRanges(Constants.DefaultBlockSize));
                                range = new Range()
                                {
                                    StartOffset = Math.Max(lastTransferWindowStart, startOffset),
                                    HasData     = hasData
                                };
                            }

                            range.EndOffset = Math.Min(lastTransferWindowEnd, endOffset);
                        }

                        if (range.EndOffset == lastTransferWindowEnd)
                        {
                            // Reach the end of transfer window, move to next
                            ++lastTransferWindowIndex;
                            continue;
                        }
                    }

                    break;
                }

                if (-1 != range.StartOffset)
                {
                    this.rangeList.AddRange(range.SplitRanges(Constants.DefaultBlockSize));
                }

                if (checkpoint.EntryTransferOffset <= endOffset + 1)
                {
                    reachLastTransferOffset = true;

                    if (checkpoint.EntryTransferOffset <= endOffset)
                    {
                        this.rangeList.AddRange(new Range()
                        {
                            StartOffset = checkpoint.EntryTransferOffset,
                            EndOffset   = endOffset,
                            HasData     = hasData,
                        }.SplitRanges(Constants.DefaultBlockSize));
                    }
                }
            }
        }
Ejemplo n.º 6
0
        /// <summary>
        /// It might fail to get large ranges list from storage. This method is to split the whole file to spans of 148MB to get ranges.
        /// In restartable, we only need to get ranges for chunks in TransferWindow and after TransferEntryOffset in check point.
        /// In TransferWindow, there might be some chunks adjacent to TransferEntryOffset, so this method will first merge these chunks into TransferEntryOffset;
        /// Then in remained chunks in the TransferWindow, it's very possible that ranges of several chunks can be got in one 148MB span.
        /// To avoid sending too many get ranges requests, this method will merge the chunks to 148MB spans.
        /// </summary>
        private void PrepareToGetRanges()
        {
            this.getRangesSpanIndex = -1;
            this.rangesSpanList     = new List <RangesSpan>();
            this.rangeList          = new List <Range>();

            this.nextDownloadIndex = 0;

            SingleObjectCheckpoint checkpoint = this.transferJob.CheckPoint;
            int blockSize = this.SharedTransferData.BlockSize;

            RangesSpan rangesSpan = null;

            if ((null != checkpoint.TransferWindow) &&
                (checkpoint.TransferWindow.Any()))
            {
                checkpoint.TransferWindow.Sort();

                long lastOffset = 0;
                if (checkpoint.EntryTransferOffset == this.SharedTransferData.TotalLength)
                {
                    long lengthBeforeLastChunk = checkpoint.EntryTransferOffset % blockSize;
                    lastOffset = 0 == lengthBeforeLastChunk ?
                                 checkpoint.EntryTransferOffset - blockSize :
                                 checkpoint.EntryTransferOffset - lengthBeforeLastChunk;
                }
                else
                {
                    lastOffset = checkpoint.EntryTransferOffset - blockSize;
                }

                for (int i = checkpoint.TransferWindow.Count - 1; i >= 0; i--)
                {
                    if (lastOffset == checkpoint.TransferWindow[i])
                    {
                        checkpoint.TransferWindow.RemoveAt(i);
                        checkpoint.EntryTransferOffset = lastOffset;
                    }
                    else if (lastOffset < checkpoint.TransferWindow[i])
                    {
                        throw new FormatException(Resources.RestartableInfoCorruptedException);
                    }
                    else
                    {
                        break;
                    }

                    lastOffset = checkpoint.EntryTransferOffset - blockSize;
                }

                if (this.transferJob.CheckPoint.TransferWindow.Any())
                {
                    rangesSpan             = new RangesSpan();
                    rangesSpan.StartOffset = checkpoint.TransferWindow[0];
                    rangesSpan.EndOffset   = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;

                    for (int i = 1; i < checkpoint.TransferWindow.Count; ++i)
                    {
                        if (checkpoint.TransferWindow[i] + blockSize > rangesSpan.EndOffset)
                        {
                            long lastEndOffset = rangesSpan.EndOffset;
                            this.rangesSpanList.Add(rangesSpan);
                            rangesSpan             = new RangesSpan();
                            rangesSpan.StartOffset = checkpoint.TransferWindow[i] > lastEndOffset ? checkpoint.TransferWindow[i] : lastEndOffset + 1;
                            rangesSpan.EndOffset   = Math.Min(rangesSpan.StartOffset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1;
                        }
                    }

                    this.rangesSpanList.Add(rangesSpan);
                }
            }

            long offset = null != rangesSpan ?
                          rangesSpan.EndOffset > checkpoint.EntryTransferOffset ?
                          rangesSpan.EndOffset + 1 :
                          checkpoint.EntryTransferOffset :
                          checkpoint.EntryTransferOffset;

            while (offset < this.SharedTransferData.TotalLength)
            {
                rangesSpan = new RangesSpan()
                {
                    StartOffset = offset,
                    EndOffset   = Math.Min(offset + Constants.PageRangesSpanSize, this.SharedTransferData.TotalLength) - 1
                };

                this.rangesSpanList.Add(rangesSpan);
                offset = rangesSpan.EndOffset + 1;
            }

            if (this.rangesSpanList.Any())
            {
                this.getRangesCountDownEvent = new CountdownEvent(this.rangesSpanList.Count);
            }
        }
Ejemplo n.º 7
0
        protected override async Task DoPreCopyAsync()
        {
            this.hasWork = false;
            long rangeSpanOffset = this.nextRangesSpanOffset;
            long rangeSpanLength = Math.Min(Constants.PageRangesSpanSize, this.totalLength - rangeSpanOffset);

            this.nextRangesSpanOffset += Constants.PageRangesSpanSize;
            this.hasWork = (this.nextRangesSpanOffset < this.totalLength);

            var pageRanges = await this.sourcePageBlob.GetPageRangesAsync(
                rangeSpanOffset,
                rangeSpanLength,
                Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition, this.destLocation.CheckedAccessCondition),
                Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions),
                Utils.GenerateOperationContext(this.TransferContext),
                this.CancellationToken);

            long        pageOffset = rangeSpanOffset;
            List <long> pageList   = new List <long>();

            foreach (var pageRange in pageRanges)
            {
                if (pageOffset <= pageRange.StartOffset)
                {
                    while (pageOffset + Constants.DefaultTransferChunkSize < pageRange.StartOffset)
                    {
                        pageOffset += Constants.DefaultTransferChunkSize;
                    }

                    pageList.Add(pageOffset);
                    pageOffset += Constants.DefaultTransferChunkSize;
                }

                // pageOffset > pageRange.StartOffset
                while (pageOffset < pageRange.EndOffset)
                {
                    pageList.Add(pageOffset);
                    pageOffset += Constants.DefaultTransferChunkSize;
                }
            }

            this.pageListBag.Add(pageList);

            if (this.getRangesCountdownEvent.Signal())
            {
                foreach (var pageListInARange in this.pageListBag)
                {
                    this.pagesToCopy.AddRange(pageListInARange);
                }

                this.pagesToCopy.Sort();
                this.nextPageToCopy = this.pagesToCopy.GetEnumerator();
                this.hasNextPage    = this.nextPageToCopy.MoveNext();

                int pageLength = TransferManager.Configurations.BlockSize;

                SingleObjectCheckpoint checkpoint = this.TransferJob.CheckPoint;
                if ((null != checkpoint.TransferWindow) &&
                    (0 != checkpoint.TransferWindow.Count))
                {
                    this.lastTransferWindow = new Queue <long>(checkpoint.TransferWindow);
                }

                int blockCount = null == this.lastTransferWindow ? 0 : this.lastTransferWindow.Count;
                blockCount += this.pagesToCopy.Count;

                if (0 == blockCount)
                {
                    this.state = State.Commit;
                }
                else
                {
                    this.countdownEvent = new CountdownEvent(blockCount);
                    this.state          = State.Copy;
                }

                this.hasWork = true;
            }
        }
Ejemplo n.º 8
0
        private async Task FetchAttributesAsync()
        {
            Debug.Assert(
                this.state == State.FetchAttributes,
                "FetchAttributesAsync called, but state isn't FetchAttributes",
                "Current state is {0}",
                this.state);

            if (Interlocked.CompareExchange(ref workToken, 0, 1) == 0)
            {
                return;
            }

            this.CheckInputStreamLength(this.SharedTransferData.TotalLength);

            bool exist = !this.Controller.IsForceOverwrite;

            if (!this.Controller.IsForceOverwrite)
            {
                try
                {
                    await Utils.ExecuteXsclApiCallAsync(
                        async() => await this.DoFetchAttributesAsync().ConfigureAwait(false),
                        this.CancellationToken).ConfigureAwait(false);
                }
#if EXPECT_INTERNAL_WRAPPEDSTORAGEEXCEPTION
                catch (Exception e) when(e is StorageException || (e is AggregateException && e.InnerException is StorageException))
                {
                    var se = e as StorageException ?? e.InnerException as StorageException;
#else
                catch (StorageException se)
                {
#endif
                    // Getting a storage exception is expected if the file doesn't
                    // exist. In this case we won't error out, but set the
                    // exist flag to false to indicate we're uploading
                    // a new file instead of overwriting an existing one.
                    if (null != se.RequestInformation &&
                        se.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
                    {
                        exist = false;
                    }
                    else
                    {
                        this.HandleFetchAttributesResult(se);
                        throw;
                    }
                }
                catch (Exception e)
                {
                    this.HandleFetchAttributesResult(e);
                    throw;
                }

                await this.Controller.CheckOverwriteAsync(
                    exist,
                    this.TransferJob.Source.Instance,
                    this.TransferJob.Destination.Instance).ConfigureAwait(false);
            }

            if (this.TransferJob.Destination.Type == TransferLocationType.AzureBlob)
            {
                (this.TransferJob.Destination as AzureBlobLocation).CheckedAccessCondition = true;
            }
            else
            {
                (this.TransferJob.Destination as AzureFileLocation).CheckedAccessCondition = true;
            }

            this.Controller.UpdateProgressAddBytesTransferred(0);

            SingleObjectCheckpoint checkpoint = this.TransferJob.CheckPoint;

            // We should create the destination if there's no previous transfer progress in the checkpoint.
            // Create will clear all data if the destination already exists.
            bool shouldCreate = (checkpoint.EntryTransferOffset == 0) && (!checkpoint.TransferWindow.Any());

            if (!shouldCreate)
            {
                this.InitUpload();
            }
            else
            {
                this.state = State.Create;
            }

            this.workToken = 1;
        }
        private void HandleFetchAttributesResult(Exception e)
        {
            bool existingBlob = !this.Controller.IsForceOverwrite;

            if (null != e)
            {
                StorageException se = e as StorageException;

                if (null != se)
                {
                    // Getting a storage exception is expected if the blob doesn't
                    // exist. In this case we won't error out, but set the
                    // existingBlob flag to false to indicate we're uploading
                    // a new blob instead of overwriting an existing blob.
                    if (null != se.RequestInformation &&
                        se.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
                    {
                        existingBlob = false;
                    }
                    else if (null != se &&
                             (0 == string.Compare(se.Message, Constants.BlobTypeMismatch, StringComparison.OrdinalIgnoreCase)))
                    {
                        throw new InvalidOperationException(Resources.DestinationBlobTypeNotMatch);
                    }
                    else
                    {
                        throw se;
                    }
                }
                else
                {
                    throw e;
                }
            }

            this.destLocation.CheckedAccessCondition = true;

            if (string.IsNullOrEmpty(this.destLocation.BlockIdPrefix))
            {
                // BlockIdPrefix is never set before that this is the first time to transfer this file.
                // In block blob upload, it stores uploaded but not committed blocks on Azure Storage.
                // In DM, we use block id to identify the blocks uploaded so we only need to upload it once.
                // Keep BlockIdPrefix in upload job object for restarting the transfer if anything happens.
                this.destLocation.BlockIdPrefix = Guid.NewGuid().ToString("N") + "-";
            }

            if (!this.Controller.IsForceOverwrite)
            {
                // If destination file exists, query user whether to overwrite it.
                this.Controller.CheckOverwrite(
                    existingBlob,
                    this.SharedTransferData.TransferJob.Source.Instance,
                    this.destLocation.Blob);
            }

            this.Controller.UpdateProgressAddBytesTransferred(0);

            if (existingBlob)
            {
                if (this.destLocation.Blob.Properties.BlobType == BlobType.Unspecified)
                {
                    throw new InvalidOperationException(Resources.FailedToGetBlobTypeException);
                }
                if (this.destLocation.Blob.Properties.BlobType != BlobType.BlockBlob)
                {
                    throw new InvalidOperationException(Resources.DestinationBlobTypeNotMatch);
                }

                Debug.Assert(
                    this.destLocation.Blob.Properties.BlobType == BlobType.BlockBlob,
                    "BlobType should be BlockBlob if we reach here.");
            }

            // Calculate number of blocks.
            int numBlocks = (int)Math.Ceiling(
                this.SharedTransferData.TotalLength / (double)this.Scheduler.TransferOptions.BlockSize);

            // Create sequence array.
            this.blockIdSequence = new string[numBlocks];

            for (int i = 0; i < numBlocks; ++i)
            {
                string blockIdSuffix  = i.ToString("D6", CultureInfo.InvariantCulture);
                byte[] blockIdInBytes = System.Text.Encoding.UTF8.GetBytes(this.destLocation.BlockIdPrefix + blockIdSuffix);
                string blockId        = Convert.ToBase64String(blockIdInBytes);
                this.blockIdSequence[i] = blockId;
            }

            SingleObjectCheckpoint checkpoint = this.SharedTransferData.TransferJob.CheckPoint;

            int leftBlockCount = (int)Math.Ceiling(
                (this.SharedTransferData.TotalLength - checkpoint.EntryTransferOffset) / (double)this.Scheduler.TransferOptions.BlockSize) + checkpoint.TransferWindow.Count;

            if (0 == leftBlockCount)
            {
                this.state = State.Commit;
            }
            else
            {
                this.countdownEvent = new CountdownEvent(leftBlockCount);

                this.state = State.UploadBlob;
            }

            this.PreProcessed = true;
            this.hasWork      = true;
        }
Ejemplo n.º 10
0
        private void HandleFetchAttributesResult(Exception e)
        {
            bool existingBlob = !this.Controller.IsForceOverwrite;

            if (null != e)
            {
                StorageException se = e as StorageException;

                if (null != se)
                {
                    // Getting a storage exception is expected if the blob doesn't
                    // exist. In this case we won't error out, but set the
                    // existingBlob flag to false to indicate we're uploading
                    // a new blob instead of overwriting an existing blob.
                    if (null != se.RequestInformation &&
                        se.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
                    {
                        existingBlob = false;
                    }
                    else if ((0 == string.Compare(se.Message, Constants.BlobTypeMismatch, StringComparison.OrdinalIgnoreCase)))
                    {
                        throw new InvalidOperationException(Resources.DestinationBlobTypeNotMatch, se);
                    }
                    else
                    {
                        throw se;
                    }
                }
                else
                {
                    throw e;
                }
            }

            this.destLocation.CheckedAccessCondition = true;

            if (!this.Controller.IsForceOverwrite)
            {
                // If destination file exists, query user whether to overwrite it.
                this.Controller.CheckOverwrite(
                    existingBlob,
                    this.SharedTransferData.TransferJob.Source.Instance,
                    this.destLocation.Blob);
            }

            this.Controller.UpdateProgressAddBytesTransferred(0);

            if (existingBlob)
            {
                if (this.destLocation.Blob.Properties.BlobType == BlobType.Unspecified)
                {
                    throw new InvalidOperationException(Resources.FailedToGetBlobTypeException);
                }

                if (this.destLocation.Blob.Properties.BlobType != BlobType.BlockBlob)
                {
                    throw new InvalidOperationException(Resources.DestinationBlobTypeNotMatch);
                }

                Debug.Assert(
                    this.destLocation.Blob.Properties.BlobType == BlobType.BlockBlob,
                    "BlobType should be BlockBlob if we reach here.");
            }

            SingleObjectCheckpoint checkpoint = this.SharedTransferData.TransferJob.CheckPoint;

            int leftBlockCount = (int)Math.Ceiling(
                (this.SharedTransferData.TotalLength - checkpoint.EntryTransferOffset) / (double)this.SharedTransferData.BlockSize) + checkpoint.TransferWindow.Count;

            if (this.SharedTransferData.TotalLength > 0 &&
                this.SharedTransferData.TotalLength <= Constants.SingleRequestBlobSizeThreshold)
            {
                this.PrepareForPutBlob(leftBlockCount);
            }
            else
            {
                this.PrepareForPutBlockAndPutBlockList(leftBlockCount);
            }

            this.PreProcessed = true;
            this.hasWork      = true;
        }