public Google.Apis.Storage.v1.Data.Objects GetBlobsRaw(string bucketName, string prefix, string delimiter)
{
/*
* Gcs folder behavior, by wesly , 20200501
* folder name look likes aaa/ and the size is zero, and then if we put a ojbect in the folder, the folder name aaa/
* will be delete by system automatically, so you can get the object instnce of aaa/ any more.
* if list the bucket with delimiter, we can get all the direct child objects and folders names(as prefixes)
* and only these folders which contains no items can get the object instance.
* if list the bucket with delimiter and IncludeTrailingDelimiter, objects that end in exactly one
* instance of delimiter will have their metadata included in items in addition to prefixes.
* but pls pay attention to the duplicated item both in items and prefixes
*/
StorageService storageService = this.storage.Service;
ObjectsResource.ListRequest request = storageService.Objects.List(bucketName);
if (null != delimiter)
{
request.Delimiter = delimiter;
}
if (null != prefix)
{
request.Prefix = prefix;
}
//request.IncludeTrailingDelimiter = true;
Google.Apis.Storage.v1.Data.Objects response = request.Execute();
return(response);
}
/// <summary>
/// Asks the user about deleting bucket objects, and starts async tasks to do so.
/// </summary>
private List <Task <string> > AskDeleteObjects(StorageService service)
{
List <Task <string> > deleteTasks = new List <Task <string> >();
bool yesAll = false;
bool noAll = false;
ObjectsResource.ListRequest request = service.Objects.List(Name);
do
{
Objects bucketObjects = request.Execute();
string caption = $"Deleting {bucketObjects.Items.Count} bucket objects";
if (bucketObjects.NextPageToken != null)
{
caption = $"Deleting more than {bucketObjects.Items.Count} bucket objects";
}
foreach (var bucketObject in bucketObjects.Items)
{
string query = $"Delete bucket object {bucketObject.Name}?";
if (Force || ShouldContinue(query, caption, ref yesAll, ref noAll))
{
deleteTasks.Add(service.Objects.Delete(Name, bucketObject.Name).ExecuteAsync());
}
}
request.PageToken = bucketObjects.NextPageToken;
} while (request.PageToken != null && !noAll && !Stopping);
return(deleteTasks);
}
/// <summary>
/// Gets a clean set of data from the service. If the bucket has more than a single page of objects,
/// the model will only take the first page.
/// </summary>
public void PopulateModel()
{
ObjectsResource.ListRequest request = _service.Objects.List(_bucket);
request.Projection = ObjectsResource.ListRequest.ProjectionEnum.Full;
Objects objects = request.Execute();
_pageLimited = objects.NextPageToken != null;
foreach (Object gcsObject in objects.Items ?? Enumerable.Empty <Object>())
{
_objectMap[gcsObject.Name] = gcsObject;
// Find the prefixes (parent folders) of this object.
string prefix = gcsObject.Name.TrimEnd(Separator);
int lastSeparator = gcsObject.Name.LastIndexOf(Separator);
bool children = false;
while (lastSeparator > 0)
{
prefix = prefix.Substring(0, lastSeparator);
if (_prefixes.ContainsKey(prefix))
{
_prefixes[prefix] = children || _prefixes[prefix];
break;
}
_prefixes[prefix] = children;
children = true;
lastSeparator = prefix.LastIndexOf(Separator);
}
}
}
/// <summary>
/// Gets a clean set of data from the service. If the bucket has more than a single page of objects,
/// the model will only take the first page.
/// </summary>
public void PopulateModel()
{
ObjectsResource.ListRequest request = _service.Objects.List(_bucket);
request.Projection = ObjectsResource.ListRequest.ProjectionEnum.Full;
Objects objects = request.Execute();
_pageLimited = objects.NextPageToken != null;
foreach (Object gcsObject in objects.Items ?? Enumerable.Empty <Object>())
{
_objectMap[gcsObject.Name] = gcsObject;
// Find the prefixes (parent folders) of this object.
string prefix = gcsObject.Name.TrimEnd(Separator);
int lastSeparator = gcsObject.Name.LastIndexOf(Separator);
// If the object name is testing/blah.txt, then the first prefix "testing" has children
// since '/' is not at the end. Otherwise, we don't know yet.
bool children = 0 < lastSeparator && lastSeparator < prefix.Length - 1;
while (lastSeparator > 0)
{
prefix = prefix.Substring(0, lastSeparator);
if (_prefixes.ContainsKey(prefix))
{
_prefixes[prefix] = children || _prefixes[prefix];
break;
}
_prefixes[prefix] = children;
children = true;
lastSeparator = prefix.LastIndexOf(Separator);
}
}
}
protected override async Task <IReadOnlyCollection <Blob> > ListAtAsync(string path, ListOptions options, CancellationToken cancellationToken)
{
ObjectsResource.ListRequest request = _client.Service.Objects.List(_bucketName);
request.Prefix = StoragePath.IsRootPath(path) ? null : (NormalisePath(path) + "/");
request.Delimiter = "/";
var page = new List <Blob>();
do
{
Objects serviceObjects = await request.ExecuteAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
if (serviceObjects.Items != null)
{
page.AddRange(GConvert.ToBlobs(serviceObjects.Items, options));
}
if (serviceObjects.Prefixes != null)
{
//the only info we have about prefixes is it's name
page.AddRange(serviceObjects.Prefixes.Select(p => new Blob(p, BlobItemKind.Folder)));
}
request.PageToken = serviceObjects.NextPageToken;
}while(request.PageToken != null);
return(page);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
ObjectsResource.ListRequest listReq = Service.Objects.List(Bucket);
listReq.Projection = ObjectsResource.ListRequest.ProjectionEnum.Full;
listReq.Delimiter = Delimiter;
listReq.Prefix = Prefix;
listReq.MaxResults = 100;
// When used with WriteObject, expand the IEnumerable rather than
// returning the IEnumerable itself. IEnumerable<T> vs. IEnumerable<IEnumerable<T>>.
const bool enumerateCollection = true;
// First page.
Objects gcsObjects = listReq.Execute();
WriteObject(gcsObjects.Items, enumerateCollection);
// Keep paging through results as necessary.
while (gcsObjects.NextPageToken != null)
{
listReq.PageToken = gcsObjects.NextPageToken;
gcsObjects = listReq.Execute();
WriteObject(gcsObjects.Items, enumerateCollection);
}
}
private IEnumerable <Object> ListChildren(GcsPath gcsPath, bool recurse, bool allPages = true)
{
ObjectsResource.ListRequest request = Service.Objects.List(gcsPath.Bucket);
request.Projection = ObjectsResource.ListRequest.ProjectionEnum.Full;
request.Prefix = gcsPath.ObjectPath;
if (!string.IsNullOrEmpty(request.Prefix) && !request.Prefix.EndsWith("/"))
{
request.Prefix = request.Prefix + "/";
}
if (!recurse)
{
request.Delimiter = "/";
}
do
{
Objects response = request.Execute();
foreach (Object gcsObject in response.Items ?? Enumerable.Empty <Object>())
{
if (gcsObject.Name != request.Prefix)
{
GetBucketModel(gcsPath.Bucket).AddObject(gcsObject);
yield return(gcsObject);
}
}
foreach (string prefix in response.Prefixes ?? Enumerable.Empty <string>())
{
yield return(new Object {
Name = $"{prefix}", Bucket = gcsPath.Bucket
});
}
request.PageToken = response.NextPageToken;
} while (allPages && !Stopping && request.PageToken != null);
}
public void RenameDirectory(string directoryName, string newDirectoryName)
{
directoryName = StorageUtils.NormalizeDirectoryName(directoryName);
newDirectoryName = StorageUtils.NormalizeDirectoryName(newDirectoryName);
ObjectsResource.ListRequest request = Service.Objects.List(Bucket);
request.Prefix = directoryName;
Google.Apis.Storage.v1.Data.Objects response;
do
{
response = request.Execute();
foreach (Google.Apis.Storage.v1.Data.Object item in response.Items)
{
if (IsFile(item.Name))
{
Copy(item.Name, GxFileType.Public, item.Name.Replace(directoryName, newDirectoryName), GxFileType.Public);
Delete(item.Name, GxFileType.Public);
}
}
CreateDirectory(newDirectoryName);
if (Exists(directoryName, GxFileType.Public))
{
Delete(directoryName, GxFileType.Public);
}
request.PageToken = response.NextPageToken;
} while (response.NextPageToken != null);
foreach (string subdir in GetSubDirectories(directoryName))
{
RenameDirectory(subdir, subdir.Replace(directoryName, newDirectoryName));
DeleteDirectory(subdir);
}
}
public List <string> GetFiles(string directoryName, string filter = "")
{
directoryName = StorageUtils.NormalizeDirectoryName(directoryName);
List <string> files = new List <string>();
ObjectsResource.ListRequest request = Service.Objects.List(Bucket);
request.Prefix = directoryName;
Google.Apis.Storage.v1.Data.Objects response;
do
{
response = request.Execute();
if (response.Items == null)
{
continue;
}
foreach (Google.Apis.Storage.v1.Data.Object item in response.Items)
{
if (IsFile(item.Name, directoryName) && (String.IsNullOrEmpty(filter) || item.Name.Contains(filter)))
{
files.Add(item.Name);
}
}
request.PageToken = response.NextPageToken;
} while (response.NextPageToken != null);
return(files);
}
/// <summary>
/// Modifies the specified request for all non-null properties of this options object.
/// </summary>
/// <param name="request">The request to modify</param>
internal void ModifyRequest(ObjectsResource.ListRequest request)
{
if (PageSize != null)
{
request.MaxResults = PageSize.Value;
}
if (Delimiter != null)
{
request.Delimiter = Delimiter;
}
if (Versions != null)
{
request.Versions = Versions.Value;
}
}
private void DeleteObjects(GcsPath gcsPath)
{
string bucketName = gcsPath.Bucket;
List <Task <string> > deleteTasks = new List <Task <string> >();
ObjectsResource.ListRequest request = Service.Objects.List(bucketName);
do
{
Objects gcsObjects = request.Execute();
foreach (Object gcsObject in gcsObjects.Items ?? Enumerable.Empty <Object>())
{
deleteTasks.Add(Service.Objects.Delete(bucketName, gcsObject.Name).ExecuteAsync());
}
request.PageToken = gcsObjects.NextPageToken;
} while (request.PageToken != null && !Stopping);
WaitDeleteTasks(deleteTasks);
}
public bool HasChildren(string objectName)
{
if (string.IsNullOrEmpty(objectName))
{
return(_objectMap.Count > 0);
}
else if (_pageLimited && !_prefixes.ContainsKey(objectName))
{
ObjectsResource.ListRequest request = _service.Objects.List(_bucket);
request.Prefix = objectName;
Objects response = request.Execute();
return(response.Items != null && response.Items.Count > 0);
}
else
{
return(_prefixes.ContainsKey(objectName) && _prefixes[objectName]);
}
}
public bool ExistsDirectory(string directoryName)
{
directoryName = StorageUtils.NormalizeDirectoryName(directoryName);
bool exists = false;
ObjectsResource.ListRequest request = Service.Objects.List(Bucket);
request.Delimiter = StorageUtils.DELIMITER;
Google.Apis.Storage.v1.Data.Objects response;
do
{
response = request.Execute();
if (response.Prefixes != null && response.Prefixes.Contains(directoryName))
{
exists = true;
}
} while (response.NextPageToken != null);
return(exists);
}
/// <summary>
/// Checks to see if the given object is a folder.
/// </summary>
/// <param name="objectName">The name of the object to check.</param>
/// <returns>True if the object name is an existant object that ends with "/", or is a prefix for other
/// existant objects.</returns>
public bool IsContainer(string objectName)
{
if (_prefixes.ContainsKey(objectName.TrimEnd(Separator)))
{
return(true);
}
else if (_pageLimited)
{
ObjectsResource.ListRequest request = _service.Objects.List(_bucket);
request.Prefix = objectName;
request.Delimiter = SeparatorString;
Objects response = request.Execute();
return(response.Prefixes != null && response.Prefixes.Count > 0);
}
else
{
return(false);
}
}
public List <string> GetFilesList()
{
var newObject = new Google.Apis.Storage.v1.Data.Object()
{
Bucket = BucketToUpload,
};
List <string> resultStatus = null;
try
{
var listRequest = new ObjectsResource.ListRequest(StorageService,
BucketToUpload);
listRequest.OauthToken = UserCredential.Token.AccessToken;
resultStatus = listRequest.Execute().Items.Select(x => x.Name).ToList();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(resultStatus);
}
public List <string> GetSubDirectories(string directoryName)
{
directoryName = StorageUtils.NormalizeDirectoryName(directoryName);
List <string> subDirs = new List <string>();
ObjectsResource.ListRequest request = Service.Objects.List(Bucket);
request.Prefix = directoryName;
request.Delimiter = StorageUtils.DELIMITER;
Google.Apis.Storage.v1.Data.Objects response;
do
{
response = request.Execute();
if (response.Prefixes == null)
{
continue;
}
foreach (string dir in response.Prefixes)
{
subDirs.Add(dir);
}
request.PageToken = response.NextPageToken;
} while (response.NextPageToken != null);
return(subDirs);
}
