/// <summary>
/// A Breath First Traversal method to traverse through the directories and files in a cloud.
/// </summary>
private async Task Traverse()
{
try
{
if (queue.Count == 0)
{
return;
}
DirectoryBlock currentDirectory = null;
while (queue.Count > 0)
{
currentDirectory = queue.Dequeue();
Task task = Task.Run(() => cloudService.GetFiles(currentDirectory));
task.Wait();
if (currentDirectory.SubDirectories != null)
{
List <Task> enqueueTask = new List <Task>();
foreach (var subDirectory in currentDirectory.SubDirectories)
{
if (subDirectory != null)
{
enqueueTask.Add(Task.Run(() => queue.Enqueue(subDirectory)));
}
}
Task.WaitAll(enqueueTask.ToArray());
}
}
}
catch (Exception e)
{
throw e;
}
await Traverse();
}
internal async Task AddBlobsToDirectory(DirectoryBlock directory)
{
var container = client.GetContainerReference(directory.container);
CloudBlobDirectory blobDirectory = container.GetDirectoryReference(directory.prefix);
var items = await blobDirectory.ListBlobsSegmentedAsync(false, new BlobListingDetails(),
maxListResults,
new BlobContinuationToken(),
new BlobRequestOptions(),
new OperationContext());
var list_continue = false;
do
{
foreach (IListBlobItem item in items.Results)
{
AddToParentDirectoryObject(directory, item);
}
if (items.ContinuationToken != null)
{
items = await blobDirectory.ListBlobsSegmentedAsync(false, new BlobListingDetails(),
maxListResults,
items.ContinuationToken,
new BlobRequestOptions(),
new OperationContext());
list_continue = true;
}
else
{
list_continue = false;
}
} while (list_continue);
}
/// <summary>
/// This Method takes in a parent directory as parameter.
/// Lists all the sub-directories and adds them to the parent directory.
/// Lists all the files and adds them to the parent directory.
/// </summary>
public void GetFiles(DirectoryBlock parentDirectory)
{
using (DriveService service = DriveClient.GetInfo())
{
try
{
FilesResource.ListRequest listRequest = service.Files.List();
listRequest.Q = "'" + parentDirectory.id + "' in parents and trashed = false";
listRequest.OrderBy = "name";
listRequest.Fields = "nextPageToken, files" +
"(id, name, mimeType, permissions, webContentLink, description, size, shared, owners, parents)";
Google.Apis.Drive.v3.Data.FileList ListResponse = listRequest.Execute();
while (ListResponse.Files != null && ListResponse.Files.Count != 0)
{
List <Task> tasks = new List <Task>();
foreach (var item in ListResponse.Files)
{
tasks.Add(Task.Run(() => AddToParentDirectoryObject(item, parentDirectory)));
}
Task.WaitAll(tasks.ToArray());
if (ListResponse.NextPageToken == null)
{
break;
}
listRequest.PageToken = ListResponse.NextPageToken;
ListResponse = listRequest.Execute();
}
}
catch (Exception e)
{
throw new Exception(e.StackTrace);
}
}
}
internal async Task AddContainersToRoot(DirectoryBlock directory)
{
var items = await client.ListContainersSegmentedAsync(null, new ContainerListingDetails(),
maxListResults,
new BlobContinuationToken(),
new BlobRequestOptions(),
new OperationContext());
var list_continue = false;
do
{
if (items != null)
{
foreach (var item in items.Results)
{
AddToParentDirectoryObject(directory, item);
}
}
if (items.ContinuationToken != null)
{
items = await client.ListContainersSegmentedAsync(null, new ContainerListingDetails(), maxListResults, items.ContinuationToken, new BlobRequestOptions(), new OperationContext());
list_continue = true;
}
else
{
list_continue = false;
}
} while (list_continue);
}
internal DirectoryBlock CreateDirectoryObject(Box.V2.Models.BoxItem item, DirectoryBlock parentDirectory)
{
string parent = null;
if (item.Parent == null)
{
parent = "";
}
DirectoryBlock directory = new DirectoryBlock()
{
id = item.Id,
Name = item.Name,
Description = item.Description,
Size = item.Size,
SharedLink = GetSharedLink(item),
Owners = GetOwners(item),
CreationTime = item.CreatedAt.ToString(),
Users = GetUsers(item),
Parent = new List <string> {
parent
},
path = parentDirectory.path + "\\" + item.Name,
};
return(directory);
}
internal async Task AddBlobsToContainer(DirectoryBlock directory)
{
var container = client.GetContainerReference(directory.Name);
var items = await container.ListBlobsSegmentedAsync(null, false, new BlobListingDetails(), maxListResults, new BlobContinuationToken(), null, null);
var list_continue = false;
do
{
foreach (var item in items.Results)
{
AddToParentDirectoryObject(directory, item);
}
if (items.ContinuationToken != null)
{
items = await client.ListBlobsSegmentedAsync(null, false, new BlobListingDetails(), maxListResults, items.ContinuationToken, null, null);
list_continue = true;
}
else
{
list_continue = false;
}
} while (list_continue);
}
public MigrationResult MigrateToOstor(DirectoryBlock directoryBlock, ICloudServiceIO cloudService)
{
MigrationResult result = new MigrationResult();
DownloadCloudDirectory downloadDirectory = new DownloadCloudDirectory(directoryBlock, "C:\\CloudFiles\\", cloudService);
downloadDirectory.DownloadAsync();
return(result);
}
private void CleanUpTransactionFile()
{
TransactionFile transactionFile = new TransactionFile();
DirectoryBlock directoryBlock = transactionFile.Deserialize();
queue = new Queue <DirectoryBlock>();
queue.Enqueue(directoryBlock);
Traverse();
}
public DirectoryBlock GetRoot()
{
DirectoryBlock root = new DirectoryBlock()
{
id = sourceFolder,
};
return(root);
}
internal DirectoryBlock CreateDirectoryObject(IListBlobItem item, DirectoryBlock directory)
{
DirectoryBlock newDirectory = new DirectoryBlock()
{
Name = GetName(item),
container = item.Container.Name,
type = "BlobDirectory",
prefix = directory.prefix + GetName(item) + "/",
path = directory.path + "\\" + GetName(item),
CloudUri = item.Uri,
};
return(newDirectory);
}
internal FileBlock CreateFileObject(IListBlobItem item, DirectoryBlock directory)
{
FileBlock file = new FileBlock()
{
Name = GetName(item),
//Size = blob.Properties.Length,
container = item.Container.Name,
SharedLink = item.Uri.ToString(),
path = directory.path + "\\" + GetName(item),
CloudUri = item.Uri,
};
return(file);
}
internal void AddToParentDirectoryObject(DirectoryBlock directory, CloudBlobContainer item)
{
if (directory.SubDirectories == null)
{
directory.SubDirectories = new List <DirectoryBlock>();
}
DirectoryBlock newContainer = new DirectoryBlock()
{
Name = item.Name,
type = "container",
path = item.Name,
accessType = GetAccessType((CloudBlobContainer)item),
};
directory.SubDirectories.Add(newContainer);
}
public void GetFiles(DirectoryBlock parentDirectory)
{
if (parentDirectory.id == "root")
{
Task task = Task.Run(() => AddContainersToRoot(parentDirectory));
task.Wait();
}
else if (parentDirectory.type == "container")
{
Task task = Task.Run(() => AddBlobsToContainer(parentDirectory));
task.Wait();
}
else if (parentDirectory.type == "BlobDirectory")
{
Task task = Task.Run(() => AddBlobsToDirectory(parentDirectory));
task.Wait();
}
}
public MigrationResult MigrateToOstor(DirectoryBlock directoryBlock)
{
MigrationResult result = new MigrationResult();
while (System.IO.File.Exists(TransactionFilePath))
{
TransactionFile logFile = new TransactionFile();
DirectoryBlock toDownloadBlock = logFile.Deserialize();
DownloadCloudDirectory downloadDirectory = new DownloadCloudDirectory(toDownloadBlock, "C:\\CloudFiles\\", cloudService);
downloadDirectory.DownloadAsync();
CleanUpTransactionFile();
if (DownloadCompleteStatus == true)
{
System.IO.File.Delete(TransactionFilePath);
}
}
return(result);
}
internal DirectoryBlock CreateDirectoryObject(Google.Apis.Drive.v3.Data.File item, DirectoryBlock parentDirectory)
{
DirectoryBlock directory = new DirectoryBlock()
{
id = item.Id,
Name = item.Name,
Description = item.Description,
Size = item.Size,
SharedLink = GetSharedLink(item),
Owners = GetOwners(item),
CreationTime = item.CreatedTime.ToString(),
Users = GetUsers(item),
path = parentDirectory.path + "\\" + item.Name,
mimeType = "application/folder",
Parent = (List <string>)item.Parents,
};
return(directory);
}
internal FileBlock CreateFileObject(Google.Apis.Drive.v3.Data.File item, DirectoryBlock parentDirectory)
{
FileBlock file = new FileBlock()
{
id = item.Id,
Name = item.Name,
Description = item.Description,
Size = item.Size,
SharedLink = item.WebContentLink,
Owners = GetOwners(item),
CreationTime = item.CreatedTime.ToString(),
Users = GetUsers(item),
path = parentDirectory.path + "\\" + item.Name,
mimeType = item.MimeType,
Parent = (List <string>)item.Parents,
};
return(file);
}
public void Create(DirectoryBlock directoryBlock)
{
try
{
DataContractJsonSerializer serialize = new DataContractJsonSerializer(typeof(DirectoryBlock));
lock (thisLock)
{
using (FileStream stream = new FileStream(TransactionFilePath, FileMode.Create))
{
serialize.WriteObject(stream, directoryBlock);
stream.Flush();
}
}
}
catch (Exception e)
{
throw e;
}
}
public void GetFiles(DirectoryBlock parentDirectory)
{
try
{
Task <BoxCollection <BoxItem> > task = Task.Run(() => client.FoldersManager.GetFolderItemsAsync(parentDirectory.id, 500, fields: new string[]
{ "id", "name", "shared_link", "description", "size", "owned_by", "createdAt" }));
BoxCollection <BoxItem> items = task.Result;
if (items != null)
{
foreach (Box.V2.Models.BoxItem item in items.Entries)
{
AddToParentDirectoryObject(item, parentDirectory);
}
}
}
catch (Exception e)
{
throw new Exception(e.Message + " " + e.StackTrace);
}
}
internal void AddToParentDirectoryObject(DirectoryBlock directory, IListBlobItem item)
{
if (item.GetType() == typeof(CloudBlobDirectory))
{
if (directory.SubDirectories == null)
{
directory.SubDirectories = new List <DirectoryBlock>();
}
directory.SubDirectories.Add(CreateDirectoryObject(item, directory));
}
if (item.GetType() == typeof(CloudBlockBlob))
{
var file = (CloudBlob)item;
if (directory.Files == null)
{
directory.Files = new List <FileBlock>();
}
directory.Files.Add(CreateFileObject(item, directory));
}
}
public void Traverse()
{
if (queue.Count == 0)
{
return;
}
while (queue.Count > 0)
{
DirectoryBlock currentDirectory = queue.Dequeue();
System.IO.DirectoryInfo currentDirectoryInfo = new DirectoryInfo(currentDirectory.path);
FileSystemInfo[] files = currentDirectoryInfo.GetFileSystemInfos();
List <Task> tasks = new List <Task>();
foreach (var file in files)
{
if (file.GetType() == typeof(System.IO.DirectoryInfo))
{
string DirectoryId = cloudService.CreateDirectory(destinationDirectoryId, file.Name);
DirectoryBlock directory = new DirectoryBlock()
{
id = DirectoryId,
Name = file.Name,
path = file.FullName,
};
queue.Enqueue(directory);
}
if (file.GetType() == typeof(System.IO.FileInfo))
{
FileBlock newFile = new FileBlock()
{
Name = file.Name,
Parent = new List <string> {
currentDirectory.id
}
};
cloudService.UploadCloudFile(newFile, file.FullName);
}
}
Task.WaitAll(tasks.ToArray());
Traverse();
}
}
internal void AddToParentDirectoryObject(Box.V2.Models.BoxItem item, DirectoryBlock directory)
{
if (isDirectory(item.Type))
{
DirectoryBlock newDirectory = CreateDirectoryObject(item, directory);
if (directory.SubDirectories == null)
{
directory.SubDirectories = new List <DirectoryBlock>();
}
directory.SubDirectories.Add(newDirectory);
}
else
{
FileBlock newFile = CreateFileObject(item, directory);
if (directory.Files == null)
{
directory.Files = new List <FileBlock>();
}
directory.Files.Add(newFile);
}
}
public bool UploadDirectory()
{
bool status = false;
if (System.IO.Directory.Exists(toUploadDirectoryPath))
{
int index = toUploadDirectoryPath.LastIndexOf("/");
string directoryName = toUploadDirectoryPath.Substring(index);
string newDirectoryId = cloudService.CreateDirectory(destinationDirectoryId, directoryName);
DirectoryBlock directory = new DirectoryBlock()
{
id = newDirectoryId,
path = toUploadDirectoryPath,
Name = directoryName,
};
queue.Enqueue(directory);
Traverse();
}
return(status);
}
public DirectoryBlock Deserialize()
{
DirectoryBlock directory = new DirectoryBlock();
try
{
DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(DirectoryBlock));
lock (new object())
{
using (FileStream stream = new FileStream(TransactionFilePath, FileMode.Open))
{
stream.Position = 0;
directory = (DirectoryBlock)ser.ReadObject(stream);
}
}
return(directory);
}
catch (Exception e)
{
throw e;
}
}
//public bool UploadFile(Containers.File item)
//{
// bool status = false;
// try
// {
// var fileMetadata = new Google.Apis.Drive.v3.Data.File()
// {
// Name = item.Name,
// Parents = item.Parent,
// };
// FilesResource.CreateMediaUpload request;
// using (var service = DriveClient.GetInfo())
// {
// request = service.Files.Create(
// fileMetadata, new FileStream(item.path, FileMode.Open), item.mimeType);
// request.Upload();
// }
// status = true;
// }
// catch (Exception e)
// {
// throw e;
// }
// return status;
//}
internal bool UploadDirectory(DirectoryBlock item)
{
bool status = false;
try
{
using (var service = DriveClient.GetInfo())
{
var fileMetadata = new Google.Apis.Drive.v3.Data.File()
{
Name = item.Name,
MimeType = "application/vnd.google-apps.folder",
Parents = item.Parent,
};
var request = service.Files.Create(fileMetadata).Execute();
status = true;
}
}
catch (Exception e)
{
throw e;
}
return(status);
}
/// <summary>
/// Method to fetch the directories and files at the root level.
/// </summary>
/// <returns> Root Directory </returns>
public CloudStructure GetCloudRootStructure()
{
string rootId = cloudService.GetRootId();
CloudStructure rootStructure = new CloudStructure();
try
{
rootDirectory = new DirectoryBlock()
{
id = rootId,
Name = "",
};
queue = new Queue <DirectoryBlock>();
queue.Enqueue(rootDirectory);
Task task = Task.Run(() => Traverse());
task.Wait();
rootStructure.RootDirectory = rootDirectory;
}
catch (Exception e)
{
throw new Exception(e.Message + " " + e.Source + " " + e.StackTrace);
}
return(rootStructure);
}
public DownloadCloudDirectory(DirectoryBlock directory, string toDownloadDirectoryPath, ICloudServiceIO cloudService)
{
this.directory = directory;
this.toDownloadDirectoryPath = toDownloadDirectoryPath;
this.cloudService = cloudService;
}
internal void AddToParentDirectoryObject(Google.Apis.Drive.v3.Data.File item, DirectoryBlock directory)
{
if (isDirectory(item.MimeType))
{
if (directory.SubDirectories == null)
{
directory.SubDirectories = new List <DirectoryBlock>();
}
directory.SubDirectories.Add(CreateDirectoryObject(item, directory));
}
else
{
if (directory.Files == null)
{
directory.Files = new List <FileBlock>();
}
directory.Files.Add(CreateFileObject(item, directory));
}
}
public void UploadDirectory(DirectoryBlock item)
{
throw new NotImplementedException();
}
