/// <summary> /// Attempts to load the metadata from an existing file in its canonical location. /// </summary> /// <param name="metadataFilePath">The metadata file path.</param> /// <returns></returns> public TransferFolderMetadata GetExistingMetadata(string metadataFilePath) { //load from file (based on input parameters) var metadata = TransferFolderMetadata.LoadFrom(metadataFilePath, _parameters.ConcurrentFileCount); metadata.ValidateConsistency(); return(metadata); }
/// <summary> /// Creates a new metadata based on the given input parameters, and saves it to its canonical location. /// </summary> /// <returns></returns> public TransferFolderMetadata CreateNewMetadata(string metadataFilePath) { //create metadata var metadata = new TransferFolderMetadata(metadataFilePath, _parameters, _frontend); //save the initial version metadata.Save(); return(metadata); }
/// <summary> /// Populates the specified metadata. /// </summary> /// <param name="metadata">The metadata.</param> private void Populate(TransferFolderMetadata metadata) { this.TotalFileLength = metadata.TotalFileBytes; this.TotalFileCount = metadata.FileCount; _fileProgress = new List <TransferProgress>(this.TotalFileCount); _progressBacklog = new ConcurrentQueue <TransferProgress>(); foreach (var fileMetadata in metadata.Files) { var toAdd = new TransferProgress(fileMetadata); if (fileMetadata.Status == SegmentTransferStatus.Complete) { this.TransferredByteCount += fileMetadata.FileLength; this.TransferredFileCount++; toAdd.TransferredByteCount = toAdd.TotalFileLength; foreach (var segment in toAdd._segmentProgress) { segment.TransferredByteCount = segment.Length; } } _fileProgress.Add(toAdd); } }
/// <summary> /// Attempts to load an TransferFolderMetadata object from the given file. /// </summary> /// <param name="filePath">The full path to the file where to load the metadata from</param> /// <param name="threadCount">The number of threads to use while populating the metadata. This is determined by the concurrent file count</param> /// <returns></returns> /// <exception cref="System.IO.FileNotFoundException">Could not find metadata file</exception> /// <exception cref="Microsoft.Azure.Management.DataLake.Store.InvalidMetadataException">Unable to parse metadata file</exception> internal static TransferFolderMetadata LoadFrom(string filePath, int threadCount) { if (!File.Exists(filePath)) { throw new FileNotFoundException("Could not find metadata file", filePath); } try { TransferFolderMetadata result = JsonConvert.DeserializeObject <TransferFolderMetadata>(File.ReadAllText(filePath)); if (result != null) { result.MetadataFilePath = filePath; // in the case where thread count is the default, explicitly set it to the ideal file/folder count for loading // the metadata. Note that this value may be changed once the metadata is loaded when the ideal is re-computed // based on: // 1. Total files remaining (since this is a resume) // 2. Total remaining file size. if (threadCount < 1) { threadCount = DataLakeStoreTransferClient.DefaultIdealPerFileThreadCountForFolders; } // populate all child metadata file paths as well var localMetadataFolder = Path.GetDirectoryName(filePath); int updatesPerThread = (int)Math.Ceiling((double)result.Files.Length / threadCount); var threads = new List <Thread>(threadCount); for (int i = 0; i < threadCount; i++) { var t = new Thread(() => { int startIndex = i * updatesPerThread; int endIndex = startIndex + updatesPerThread; for (int j = startIndex; j < endIndex; j++) { if (j >= result.Files.Length) { // just in case. break; } var transferMetadataPath = Path.Combine(localMetadataFolder, string.Format("{0}.transfer.xml", Path.GetFileName(result.Files[j].InputFilePath))); result.Files[j].MetadataFilePath = transferMetadataPath; } }); t.Start(); threads.Add(t); } foreach (var t in threads) { t.Join(); } } return(result); } catch (Exception ex) { throw new InvalidMetadataException("Unable to parse metadata file", ex); } }
/// <summary> /// Initializes a new instance of the <see cref="TransferFolderProgress"/> class. /// </summary> /// <param name="metadata">The metadata.</param> internal TransferFolderProgress(TransferFolderMetadata metadata) { Populate(metadata); }