/// <summary>
/// Attempts to load the metadata from an existing file in its canonical location.
/// </summary>
/// <param name="metadataFilePath">The metadata file path.</param>
/// <returns></returns>
public TransferFolderMetadata GetExistingMetadata(string metadataFilePath)
{
//load from file (based on input parameters)
var metadata = TransferFolderMetadata.LoadFrom(metadataFilePath, _parameters.ConcurrentFileCount);
metadata.ValidateConsistency();
return(metadata);
}
/// <summary>
/// Creates a new metadata based on the given input parameters, and saves it to its canonical location.
/// </summary>
/// <returns></returns>
public TransferFolderMetadata CreateNewMetadata(string metadataFilePath)
{
//create metadata
var metadata = new TransferFolderMetadata(metadataFilePath, _parameters, _frontend);
//save the initial version
metadata.Save();
return(metadata);
}
/// <summary>
/// Populates the specified metadata.
/// </summary>
/// <param name="metadata">The metadata.</param>
private void Populate(TransferFolderMetadata metadata)
{
this.TotalFileLength = metadata.TotalFileBytes;
this.TotalFileCount = metadata.FileCount;
_fileProgress = new List <TransferProgress>(this.TotalFileCount);
_progressBacklog = new ConcurrentQueue <TransferProgress>();
foreach (var fileMetadata in metadata.Files)
{
var toAdd = new TransferProgress(fileMetadata);
if (fileMetadata.Status == SegmentTransferStatus.Complete)
{
this.TransferredByteCount += fileMetadata.FileLength;
this.TransferredFileCount++;
toAdd.TransferredByteCount = toAdd.TotalFileLength;
foreach (var segment in toAdd._segmentProgress)
{
segment.TransferredByteCount = segment.Length;
}
}
_fileProgress.Add(toAdd);
}
}
/// <summary>
/// Attempts to load an TransferFolderMetadata object from the given file.
/// </summary>
/// <param name="filePath">The full path to the file where to load the metadata from</param>
/// <param name="threadCount">The number of threads to use while populating the metadata. This is determined by the concurrent file count</param>
/// <returns></returns>
/// <exception cref="System.IO.FileNotFoundException">Could not find metadata file</exception>
/// <exception cref="Microsoft.Azure.Management.DataLake.Store.InvalidMetadataException">Unable to parse metadata file</exception>
internal static TransferFolderMetadata LoadFrom(string filePath, int threadCount)
{
if (!File.Exists(filePath))
{
throw new FileNotFoundException("Could not find metadata file", filePath);
}
try
{
TransferFolderMetadata result = JsonConvert.DeserializeObject <TransferFolderMetadata>(File.ReadAllText(filePath));
if (result != null)
{
result.MetadataFilePath = filePath;
// in the case where thread count is the default, explicitly set it to the ideal file/folder count for loading
// the metadata. Note that this value may be changed once the metadata is loaded when the ideal is re-computed
// based on:
// 1. Total files remaining (since this is a resume)
// 2. Total remaining file size.
if (threadCount < 1)
{
threadCount = DataLakeStoreTransferClient.DefaultIdealPerFileThreadCountForFolders;
}
// populate all child metadata file paths as well
var localMetadataFolder = Path.GetDirectoryName(filePath);
int updatesPerThread = (int)Math.Ceiling((double)result.Files.Length / threadCount);
var threads = new List <Thread>(threadCount);
for (int i = 0; i < threadCount; i++)
{
var t = new Thread(() =>
{
int startIndex = i * updatesPerThread;
int endIndex = startIndex + updatesPerThread;
for (int j = startIndex; j < endIndex; j++)
{
if (j >= result.Files.Length)
{
// just in case.
break;
}
var transferMetadataPath = Path.Combine(localMetadataFolder, string.Format("{0}.transfer.xml", Path.GetFileName(result.Files[j].InputFilePath)));
result.Files[j].MetadataFilePath = transferMetadataPath;
}
});
t.Start();
threads.Add(t);
}
foreach (var t in threads)
{
t.Join();
}
}
return(result);
}
catch (Exception ex)
{
throw new InvalidMetadataException("Unable to parse metadata file", ex);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TransferFolderProgress"/> class.
/// </summary>
/// <param name="metadata">The metadata.</param>
internal TransferFolderProgress(TransferFolderMetadata metadata)
{
Populate(metadata);
}
