/// <summary>
/// PowerShell uses this to check if items exist.
/// </summary>
protected override bool ItemExists(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
return(true);
case GcsPath.GcsPathType.Bucket:
var bucketCache = BucketCache.Value;
if (bucketCache.ContainsKey(gcsPath.Bucket))
{
return(true);
}
try
{
var bucket = Service.Buckets.Get(gcsPath.Bucket).Execute();
bucketCache[bucket.Name] = bucket;
return(true);
}
catch
{
return(false);
}
case GcsPath.GcsPathType.Object:
BucketModel model = GetBucketModel(gcsPath.Bucket);
bool objectExists = model.ObjectExists(gcsPath.ObjectPath);
return(objectExists);
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
}
private Bucket NewBucket(GcsPath gcsPath, NewGcsBucketDynamicParameters dynamicParams)
{
if (dynamicParams.Project == null)
{
var property = dynamicParams.GetType().GetProperty(nameof(Project));
ConfigPropertyNameAttribute configPropertyName =
(ConfigPropertyNameAttribute)Attribute.GetCustomAttribute(
property, typeof(ConfigPropertyNameAttribute));
configPropertyName.SetObjectConfigDefault(property, dynamicParams);
}
var bucket = new Bucket
{
Name = gcsPath.Bucket,
Location = dynamicParams.Location,
StorageClass = dynamicParams.StorageClass
};
BucketsResource.InsertRequest insertReq = Service.Buckets.Insert(bucket, dynamicParams.Project);
insertReq.PredefinedAcl = dynamicParams.DefaultBucketAcl;
insertReq.PredefinedDefaultObjectAcl = dynamicParams.DefaultObjectAcl;
Bucket newBucket = insertReq.Execute();
BucketCache.ForceRefresh();
return(newBucket);
}
/// <summary>
/// Writes the object describing the item to the output. Used by Get-Item.
/// </summary>
/// <param name="path">The path of the item to get.</param>
protected override void GetItem(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
WriteItemObject(PSDriveInfo, path, true);
break;
case GcsPath.GcsPathType.Bucket:
Bucket bucket;
var bucketCache = BucketCache.Value;
if (bucketCache.ContainsKey(gcsPath.Bucket))
{
bucket = bucketCache[gcsPath.Bucket];
}
else
{
bucket = Service.Buckets.Get(gcsPath.Bucket).Execute();
}
WriteItemObject(bucket, path, true);
break;
case GcsPath.GcsPathType.Object:
Object gcsObject = GetBucketModel(gcsPath.Bucket).GetGcsObject(gcsPath.ObjectPath);
WriteItemObject(gcsObject, path, IsItemContainer(path));
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetItem));
}
/// <summary>
/// Deletes a Google Cloud Storage object or bucket. Used by Remove-Item.
/// </summary>
/// <param name="path">The path to the object or bucket to remove.</param>
/// <param name="recurse">If true, will remove the desendants of the item as well. Required for a
/// non-empty bucket.</param>
protected override void RemoveItem(string path, bool recurse)
{
if (!ShouldProcess(path, "Remove-Item"))
{
return;
}
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
throw new InvalidOperationException("Use Remove-PSDrive to remove a drive.");
case GcsPath.GcsPathType.Bucket:
RemoveBucket(gcsPath, recurse);
BucketCache.ForceRefresh();
break;
case GcsPath.GcsPathType.Object:
if (IsItemContainer(path))
{
RemoveFolder(GcsPath.Parse(path + "/"), recurse);
}
else
{
Service.Objects.Delete(gcsPath.Bucket, gcsPath.ObjectPath).Execute();
}
BucketModels.Clear();
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(RemoveItem));
}
/// <summary>
/// Clears the content of an object. Used by Clear-Content.
/// </summary>
/// <param name="path">The path of the object to clear.</param>
public void ClearContent(string path)
{
if (!ShouldProcess(path, "Clear-Content"))
{
return;
}
var gcsPath = GcsPath.Parse(path);
Object body = new Object
{
Name = gcsPath.ObjectPath,
Bucket = gcsPath.Bucket
};
var memoryStream = new MemoryStream();
var contentType = GcsCmdlet.UTF8TextMimeType;
ObjectsResource.InsertMediaUpload request =
Service.Objects.Insert(body, gcsPath.Bucket, memoryStream, contentType);
IUploadProgress response = request.Upload();
if (response.Exception != null)
{
throw response.Exception;
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(ClearContent));
}
/// <summary>
/// Writes the object descriptions of the items in the container to the output. Used by Get-ChildItem.
/// </summary>
/// <param name="path">The path of the container.</param>
/// <param name="recurse">If true, get all descendents of the container, not just immediate children.</param>
protected override void GetChildItems(string path, bool recurse)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
Action <Bucket> actionOnBuckets = (bucket) => GetChildItemBucketHelper(bucket, recurse);
PerformActionOnBucketAndOptionallyUpdateCache((bucket) => GetChildItemBucketHelper(bucket, recurse));
break;
case GcsPath.GcsPathType.Bucket:
case GcsPath.GcsPathType.Object:
if (IsItemContainer(path))
{
foreach (Object gcsObject in ListChildren(gcsPath, recurse))
{
string gcsObjectPath = new GcsPath(gcsObject).ToString();
bool isContainer = IsItemContainer(gcsObjectPath);
WriteItemObject(gcsObject, gcsObjectPath, isContainer);
}
}
else
{
GetItem(path);
}
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetChildItems));
}
private IEnumerable <Object> ListChildren(GcsPath gcsPath, bool recurse, bool allPages = true)
{
ObjectsResource.ListRequest request = Service.Objects.List(gcsPath.Bucket);
request.Projection = ObjectsResource.ListRequest.ProjectionEnum.Full;
request.Prefix = gcsPath.ObjectPath;
if (!string.IsNullOrEmpty(request.Prefix) && !request.Prefix.EndsWith("/"))
{
request.Prefix = request.Prefix + "/";
}
if (!recurse)
{
request.Delimiter = "/";
}
do
{
Objects response = request.Execute();
foreach (Object gcsObject in response.Items ?? Enumerable.Empty <Object>())
{
if (gcsObject.Name != request.Prefix)
{
GetBucketModel(gcsPath.Bucket).AddObject(gcsObject);
yield return(gcsObject);
}
}
foreach (string prefix in response.Prefixes ?? Enumerable.Empty <string>())
{
yield return(new Object {
Name = $"{prefix}", Bucket = gcsPath.Bucket
});
}
request.PageToken = response.NextPageToken;
} while (allPages && !Stopping && request.PageToken != null);
}
private Bucket NewBucket(GcsPath gcsPath, NewGcsBucketDynamicParameters dynamicParams)
{
if (dynamicParams.Project == null)
{
var property = dynamicParams.GetType().GetProperty(nameof(Project));
ConfigPropertyNameAttribute configPropertyName =
(ConfigPropertyNameAttribute)Attribute.GetCustomAttribute(
property, typeof(ConfigPropertyNameAttribute));
configPropertyName.SetObjectConfigDefault(property, dynamicParams);
}
var bucket = new Bucket
{
Name = gcsPath.Bucket,
Location = dynamicParams.Location,
StorageClass = dynamicParams.StorageClass
};
BucketsResource.InsertRequest insertReq = Service.Buckets.Insert(bucket, dynamicParams.Project);
insertReq.PredefinedAcl = dynamicParams.DefaultBucketAcl;
insertReq.PredefinedDefaultObjectAcl = dynamicParams.DefaultObjectAcl;
Bucket newBucket = insertReq.Execute();
// If the bucket cache is not initialized, then don't bother initializing it
// because that will cause a long wait time and we may not even know whether
// the user needs to use all the other buckets right away. Also, we should not
// refresh the whole cache right at this instance (which is why we call
// GetValueWithoutUpdate) for the same reason.
Dictionary <string, Bucket> bucketDict = BucketCache.GetLastValueWithoutUpdate();
if (bucketDict != null)
{
bucketDict[newBucket.Name] = newBucket;
}
return(newBucket);
}
/// <summary>
/// Writes the object descriptions of the items in the container to the output. Used by Get-ChildItem.
/// </summary>
/// <param name="path">The path of the container.</param>
/// <param name="recurse">If true, get all descendents of the container, not just immediate children.</param>
protected override void GetChildItems(string path, bool recurse)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
foreach (Bucket bucket in ListAllBuckets())
{
WriteItemObject(bucket, bucket.Name, true);
if (recurse)
{
try
{
GetChildItems(bucket.Name, true);
}
// It is possible to not have access to ojbects even if we have access to the bucket.
// We ignore those objects as if they did not exist.
catch (GoogleApiException e) when(e.HttpStatusCode == HttpStatusCode.Forbidden)
{
}
catch (AggregateException e)
{
foreach (Exception innerException in e.InnerExceptions)
{
WriteError(new ErrorRecord(
innerException, null, ErrorCategory.NotSpecified, bucket.Name));
}
}
catch (Exception e)
{
WriteError(new ErrorRecord(e, null, ErrorCategory.NotSpecified, bucket.Name));
}
}
}
break;
case GcsPath.GcsPathType.Bucket:
case GcsPath.GcsPathType.Object:
if (IsItemContainer(path))
{
foreach (Object gcsObject in ListChildren(gcsPath, recurse))
{
string gcsObjectPath = new GcsPath(gcsObject).ToString();
bool isContainer = IsItemContainer(gcsObjectPath);
WriteItemObject(gcsObject, gcsObjectPath, isContainer);
}
}
else
{
GetItem(path);
}
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetChildItems));
}
private void RemoveFolder(GcsPath gcsPath, bool recurse)
{
if (GetBucketModel(gcsPath.Bucket).IsReal(gcsPath.ObjectPath))
{
Service.Objects.Delete(gcsPath.Bucket, gcsPath.ObjectPath).Execute();
}
if (recurse)
{
foreach (var childObject in ListChildren(gcsPath, true))
{
Service.Objects.Delete(childObject.Bucket, childObject.Name).Execute();
}
}
}
/// <summary>
/// Copies a Google Cloud Storage object or folder to another object or folder. Used by Copy-Item.
/// </summary>
/// <param name="path">The path to copy from.</param>
/// <param name="copyPath">The path to copy to.</param>
/// <param name="recurse">If true, will copy all decendent objects as well.</param>
protected override void CopyItem(string path, string copyPath, bool recurse)
{
if (!ShouldProcess($"Copy-Item from {path} to {copyPath}"))
{
return;
}
var dyanmicParameters = (GcsCopyItemDynamicParameters)DynamicParameters;
if (recurse)
{
path = path.TrimEnd('\\') + "\\";
copyPath = copyPath.TrimEnd('\\') + "\\";
}
var gcsPath = GcsPath.Parse(path);
var gcsCopyPath = GcsPath.Parse(copyPath);
if (recurse)
{
IEnumerable <Object> children = ListChildren(gcsPath, true);
foreach (Object child in children)
{
string objectSubPath = gcsPath.RelativePathToChild(child.Name);
string destinationObject = GcsPath.Parse(MakePath(copyPath, objectSubPath)).ObjectPath;
ObjectsResource.CopyRequest childRequest = Service.Objects.Copy(null, child.Bucket,
child.Name, gcsCopyPath.Bucket, destinationObject);
childRequest.SourceGeneration = dyanmicParameters.SourceGeneration;
childRequest.DestinationPredefinedAcl = dyanmicParameters.DestinationAcl;
childRequest.Projection = ObjectsResource.CopyRequest.ProjectionEnum.Full;
Object childObject = childRequest.Execute();
bool isContainer = (new GcsPath(childObject).Type != GcsPath.GcsPathType.Object);
WriteItemObject(childObject, copyPath, isContainer);
}
}
if (!recurse || GetBucketModel(gcsPath.Bucket).IsReal(gcsPath.ObjectPath))
{
ObjectsResource.CopyRequest request =
Service.Objects.Copy(null, gcsPath.Bucket, gcsPath.ObjectPath, gcsCopyPath.Bucket,
gcsCopyPath.ObjectPath);
request.SourceGeneration = dyanmicParameters.SourceGeneration;
request.DestinationPredefinedAcl = dyanmicParameters.DestinationAcl;
request.Projection = ObjectsResource.CopyRequest.ProjectionEnum.Full;
Object response = request.Execute();
WriteItemObject(response, copyPath, gcsCopyPath.Type != GcsPath.GcsPathType.Object);
}
BucketModels.Clear();
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(CopyItem));
}
private void RemoveBucket(GcsPath gcsPath, bool removeObjects)
{
if (removeObjects)
{
DeleteObjects(gcsPath);
}
try
{
Service.Buckets.Delete(gcsPath.Bucket).Execute();
}
catch (GoogleApiException e) when(e.HttpStatusCode == HttpStatusCode.Conflict)
{
// The objects my not have been cleared yet.
Service.Buckets.Delete(gcsPath.Bucket).Execute();
}
}
/// <summary>
/// Checks if a container actually contains items.
/// </summary>
/// <param name="path">The path to the container.</param>
/// <returns>True if the container contains items.</returns>
protected override bool HasChildItems(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
return(true);
case GcsPath.GcsPathType.Bucket:
case GcsPath.GcsPathType.Object:
return(GetBucketModel(gcsPath.Bucket).HasChildren(gcsPath.ObjectPath));
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
}
private void DeleteObjects(GcsPath gcsPath)
{
string bucketName = gcsPath.Bucket;
List <Task <string> > deleteTasks = new List <Task <string> >();
ObjectsResource.ListRequest request = Service.Objects.List(bucketName);
do
{
Objects gcsObjects = request.Execute();
foreach (Object gcsObject in gcsObjects.Items ?? Enumerable.Empty <Object>())
{
deleteTasks.Add(Service.Objects.Delete(bucketName, gcsObject.Name).ExecuteAsync());
}
request.PageToken = gcsObjects.NextPageToken;
} while (request.PageToken != null && !Stopping);
WaitDeleteTasks(deleteTasks);
}
/// <summary>
/// Gets a content reader to read the contents of a downloaded Google Cloud Storage object.
/// Used by Get-Contents.
/// </summary>
/// <param name="path">The path to the object to read.</param>
/// <returns>A content reader of the contents of a given object.</returns>
public IContentReader GetContentReader(string path)
{
var gcsPath = GcsPath.Parse(path);
if (gcsPath.ObjectPath == null)
{
throw new InvalidOperationException($"Can not get the contents of a {gcsPath.Type}");
}
Object gcsObject = Service.Objects.Get(gcsPath.Bucket, gcsPath.ObjectPath).Execute();
var stream = Service.HttpClient.GetStreamAsync(gcsObject.MediaLink).Result;
IContentReader contentReader = new GcsStringReader(stream);
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetContentReader));
return(contentReader);
}
/// <summary>
/// PowerShell uses this to check if items exist.
/// </summary>
protected override bool ItemExists(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
return(true);
case GcsPath.GcsPathType.Bucket:
Dictionary <string, Bucket> bucketDict = null;
// If the bucket cache is not initialized, then don't bother initializing it
// because that will cause a long wait time and we may not even know whether
// the user needs to use all the other buckets right away. Also, we should not
// refresh the whole cache right at this instance (which is why we call
// GetValueWithoutUpdate) for the same reason.
bucketDict = BucketCache.GetLastValueWithoutUpdate();
if (bucketDict != null && bucketDict.ContainsKey(gcsPath.Bucket))
{
return(true);
}
try
{
var bucket = Service.Buckets.Get(gcsPath.Bucket).Execute();
if (bucketDict != null)
{
bucketDict[bucket.Name] = bucket;
}
return(true);
}
catch
{
return(false);
}
case GcsPath.GcsPathType.Object:
BucketModel model = GetBucketModel(gcsPath.Bucket);
bool objectExists = model.ObjectExists(gcsPath.ObjectPath);
return(objectExists);
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
}
/// <summary>
/// Deletes a Google Cloud Storage object or bucket. Used by Remove-Item.
/// </summary>
/// <param name="path">The path to the object or bucket to remove.</param>
/// <param name="recurse">If true, will remove the desendants of the item as well. Required for a
/// non-empty bucket.</param>
protected override void RemoveItem(string path, bool recurse)
{
if (!ShouldProcess(path, "Remove-Item"))
{
return;
}
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
throw new InvalidOperationException("Use Remove-PSDrive to remove a drive.");
case GcsPath.GcsPathType.Bucket:
RemoveBucket(gcsPath, recurse);
// If the bucket cache is not initialized, then don't bother initializing it
// because that will cause a long wait time and we may not even know whether
// the user needs to use all the other buckets right away. Also, we should not
// refresh the whole cache right at this instance (which is why we call
// GetValueWithoutUpdate) for the same reason.
Dictionary <string, Bucket> bucketDict = BucketCache.GetLastValueWithoutUpdate();
if (bucketDict != null)
{
bucketDict.Remove(gcsPath.Bucket);
}
break;
case GcsPath.GcsPathType.Object:
if (IsItemContainer(path))
{
RemoveFolder(GcsPath.Parse(path + "/"), recurse);
}
else
{
Service.Objects.Delete(gcsPath.Bucket, gcsPath.ObjectPath).Execute();
}
BucketModels.Clear();
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(RemoveItem));
}
/// <summary>
/// Writes the object describing the item to the output. Used by Get-Item.
/// </summary>
/// <param name="path">The path of the item to get.</param>
protected override void GetItem(string path)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
WriteItemObject(PSDriveInfo, path, true);
break;
case GcsPath.GcsPathType.Bucket:
Dictionary <string, Bucket> bucketDict = null;
Bucket bucket;
// If the bucket cache is not initialized, then don't bother initializing it
// because that will cause a long wait time and we may not even know whether
// the user needs to use all the other buckets right away. Also, we should not
// refresh the whole cache right at this instance (which is why we call
// GetValueWithoutUpdate) for the same reason.
bucketDict = BucketCache.GetLastValueWithoutUpdate();
if (bucketDict != null && bucketDict.ContainsKey(gcsPath.Bucket))
{
bucket = bucketDict[gcsPath.Bucket];
break;
}
bucket = Service.Buckets.Get(gcsPath.Bucket).Execute();
if (bucketDict != null)
{
bucketDict[bucket.Name] = bucket;
}
WriteItemObject(bucket, path, true);
break;
case GcsPath.GcsPathType.Object:
Object gcsObject = GetBucketModel(gcsPath.Bucket).GetGcsObject(gcsPath.ObjectPath);
WriteItemObject(gcsObject, path, IsItemContainer(path));
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetItem));
}
protected override object NewItemDynamicParameters(string path, string itemTypeName, object newItemValue)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
return(null);
case GcsPath.GcsPathType.Bucket:
return(new NewGcsBucketDynamicParameters());
case GcsPath.GcsPathType.Object:
return(new NewGcsObjectDynamicParameters());
default:
return(null);
}
}
/// <summary>
/// Writes the names of the children of the container to the output. Used for tab-completion.
/// </summary>
/// <param name="path">The path to the container to get the children of.</param>
/// <param name="returnContainers">The names of the children of the container.</param>
protected override void GetChildNames(string path, ReturnContainers returnContainers)
{
var gcsPath = GcsPath.Parse(path);
if (gcsPath.Type == GcsPath.GcsPathType.Drive)
{
Action <Bucket> writeBucket = (bucket) => WriteItemObject(GetChildName(bucket.Name), bucket.Name, true);
PerformActionOnBucketAndOptionallyUpdateCache(writeBucket);
}
else
{
foreach (Object child in ListChildren(gcsPath, false, false))
{
var childGcsPath = new GcsPath(child);
bool isContainer = IsItemContainer(childGcsPath.ToString());
string childName = GetChildName(childGcsPath.ToString());
WriteItemObject(childName, childGcsPath.ToString().TrimEnd('/'), isContainer);
}
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetChildNames));
}
/// <summary>
/// Gets a writer used to upload data to a Google Cloud Storage object. Used by Set-Content.
/// </summary>
/// <param name="path">The path of the object to upload to.</param>
/// <returns>The writer.</returns>
public IContentWriter GetContentWriter(string path)
{
var gcsPath = GcsPath.Parse(path);
Object body = new Object
{
Name = gcsPath.ObjectPath,
Bucket = gcsPath.Bucket
};
var inputStream = new AnonymousPipeServerStream(PipeDirection.Out);
var outputStream = new AnonymousPipeClientStream(PipeDirection.In, inputStream.ClientSafePipeHandle);
var contentType = ((GcsGetContentWriterDynamicParameters)DynamicParameters).ContentType ?? GcsCmdlet.UTF8TextMimeType;
ObjectsResource.InsertMediaUpload request =
Service.Objects.Insert(body, gcsPath.Bucket, outputStream, contentType);
request.UploadAsync();
IContentWriter contentWriter = new GcsContentWriter(inputStream);
// Force the bucket models to refresh with the potentially new object.
BucketModels.Clear();
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetContentWriter));
return(contentWriter);
}
private Object NewObject(GcsPath gcsPath, NewGcsObjectDynamicParameters dynamicParameters, Stream contentStream)
{
Object newGcsObject = new Object
{
Bucket = gcsPath.Bucket,
Name = gcsPath.ObjectPath,
ContentType = dynamicParameters.ContentType
};
ObjectsResource.InsertMediaUpload insertReq = Service.Objects.Insert(
newGcsObject, newGcsObject.Bucket, contentStream, newGcsObject.ContentType);
insertReq.PredefinedAcl = dynamicParameters.PredefinedAcl;
insertReq.Projection = ObjectsResource.InsertMediaUpload.ProjectionEnum.Full;
IUploadProgress finalProgress = insertReq.Upload();
if (finalProgress.Exception != null)
{
throw finalProgress.Exception;
}
return(insertReq.ResponseBody);
}
/// <summary>
/// Creates a new item at the given path.
/// </summary>
/// <param name="path">The path of the item ot create.</param>
/// <param name="itemTypeName">The type of item to create. "Directory" is the only special one.
/// That will create an object with a name ending in "/".</param>
/// <param name="newItemValue">The value of the item to create. We assume it is a string.</param>
protected override void NewItem(string path, string itemTypeName, object newItemValue)
{
if (!ShouldProcess(path, "New-Item"))
{
return;
}
bool newFolder = itemTypeName == "Directory";
if (newFolder && !path.EndsWith("/"))
{
path += "/";
}
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
throw new InvalidOperationException("Use New-PSDrive to create a new drive.");
case GcsPath.GcsPathType.Bucket:
Bucket newBucket = NewBucket(gcsPath, (NewGcsBucketDynamicParameters)DynamicParameters);
WriteItemObject(newBucket, path, true);
break;
case GcsPath.GcsPathType.Object:
var dynamicParameters = (NewGcsObjectDynamicParameters)DynamicParameters;
Stream contentStream = GetContentStream(newItemValue, dynamicParameters);
Object newObject = NewObject(gcsPath, dynamicParameters, contentStream);
WriteItemObject(newObject, path, newFolder);
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
BucketModels.Clear();
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(NewItem));
}
private IEnumerable<Object> ListChildren(GcsPath gcsPath, bool recurse, bool allPages = true)
{
ObjectsResource.ListRequest request = Service.Objects.List(gcsPath.Bucket);
request.Projection = ObjectsResource.ListRequest.ProjectionEnum.Full;
request.Prefix = gcsPath.ObjectPath;
if (!string.IsNullOrEmpty(request.Prefix) && !request.Prefix.EndsWith("/"))
{
request.Prefix = request.Prefix + "/";
}
if (!recurse)
{
request.Delimiter = "/";
}
do
{
Objects response = request.Execute();
foreach (Object gcsObject in response.Items ?? Enumerable.Empty<Object>())
{
if (gcsObject.Name != request.Prefix)
{
GetBucketModel(gcsPath.Bucket).AddObject(gcsObject);
yield return gcsObject;
}
}
foreach (string prefix in response.Prefixes ?? Enumerable.Empty<string>())
{
yield return new Object { Name = $"{prefix}", Bucket = gcsPath.Bucket };
}
request.PageToken = response.NextPageToken;
} while (allPages && !Stopping && request.PageToken != null);
}
private Object NewObject(GcsPath gcsPath, NewGcsObjectDynamicParameters dynamicParameters, Stream contentStream)
{
Object newGcsObject = new Object
{
Bucket = gcsPath.Bucket,
Name = gcsPath.ObjectPath,
ContentType = dynamicParameters.ContentType
};
ObjectsResource.InsertMediaUpload insertReq = Service.Objects.Insert(
newGcsObject, newGcsObject.Bucket, contentStream, newGcsObject.ContentType);
insertReq.PredefinedAcl = dynamicParameters.PredefinedAcl;
insertReq.Projection = ObjectsResource.InsertMediaUpload.ProjectionEnum.Full;
IUploadProgress finalProgress = insertReq.Upload();
if (finalProgress.Exception != null)
{
throw finalProgress.Exception;
}
return insertReq.ResponseBody;
}
/// <summary>
/// Writes the names of the children of the container to the output. Used for tab-completion.
/// </summary>
/// <param name="path">The path to the container to get the children of.</param>
/// <param name="returnContainers">The names of the children of the container.</param>
protected override void GetChildNames(string path, ReturnContainers returnContainers)
{
var gcsPath = GcsPath.Parse(path);
if (gcsPath.Type == GcsPath.GcsPathType.Drive)
{
Action<Bucket> writeBucket = (bucket) => WriteItemObject(GetChildName(bucket.Name), bucket.Name, true);
PerformActionOnBucketAndOptionallyUpdateCache(writeBucket);
}
else
{
foreach (Object child in ListChildren(gcsPath, false, false))
{
var childGcsPath = new GcsPath(child);
bool isContainer = IsItemContainer(childGcsPath.ToString());
string childName = GetChildName(childGcsPath.ToString());
WriteItemObject(childName, childGcsPath.ToString().TrimEnd('/'), isContainer);
}
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetChildNames));
}
/// <summary>
/// Writes the object descriptions of the items in the container to the output. Used by Get-ChildItem.
/// </summary>
/// <param name="path">The path of the container.</param>
/// <param name="recurse">If true, get all descendents of the container, not just immediate children.</param>
protected override void GetChildItems(string path, bool recurse)
{
var gcsPath = GcsPath.Parse(path);
switch (gcsPath.Type)
{
case GcsPath.GcsPathType.Drive:
Action<Bucket> actionOnBuckets = (bucket) => GetChildItemBucketHelper(bucket, recurse);
PerformActionOnBucketAndOptionallyUpdateCache((bucket) => GetChildItemBucketHelper(bucket, recurse));
break;
case GcsPath.GcsPathType.Bucket:
case GcsPath.GcsPathType.Object:
if (IsItemContainer(path))
{
foreach (Object gcsObject in ListChildren(gcsPath, recurse))
{
string gcsObjectPath = new GcsPath(gcsObject).ToString();
bool isContainer = IsItemContainer(gcsObjectPath);
WriteItemObject(gcsObject, gcsObjectPath, isContainer);
}
}
else
{
GetItem(path);
}
break;
default:
throw new InvalidOperationException($"Unknown Path Type {gcsPath.Type}");
}
TelemetryReporter.ReportSuccess(nameof(GoogleCloudStorageProvider), nameof(GetChildItems));
}
private void RemoveFolder(GcsPath gcsPath, bool recurse)
{
if (GetBucketModel(gcsPath.Bucket).IsReal(gcsPath.ObjectPath))
{
Service.Objects.Delete(gcsPath.Bucket, gcsPath.ObjectPath).Execute();
}
if (recurse)
{
foreach (var childObject in ListChildren(gcsPath, true))
{
Service.Objects.Delete(childObject.Bucket, childObject.Name).Execute();
}
}
}
private Bucket NewBucket(GcsPath gcsPath, NewGcsBucketDynamicParameters dynamicParams)
{
if (dynamicParams.Project == null)
{
var property = dynamicParams.GetType().GetProperty(nameof(Project));
ConfigPropertyNameAttribute configPropertyName =
(ConfigPropertyNameAttribute)Attribute.GetCustomAttribute(
property, typeof(ConfigPropertyNameAttribute));
configPropertyName.SetObjectConfigDefault(property, dynamicParams);
}
var bucket = new Bucket
{
Name = gcsPath.Bucket,
Location = dynamicParams.Location,
StorageClass = dynamicParams.StorageClass
};
BucketsResource.InsertRequest insertReq = Service.Buckets.Insert(bucket, dynamicParams.Project);
insertReq.PredefinedAcl = dynamicParams.DefaultBucketAcl;
insertReq.PredefinedDefaultObjectAcl = dynamicParams.DefaultObjectAcl;
Bucket newBucket = insertReq.Execute();
// If the bucket cache is not initialized, then don't bother initializing it
// because that will cause a long wait time and we may not even know whether
// the user needs to use all the other buckets right away. Also, we should not
// refresh the whole cache right at this instance (which is why we call
// GetValueWithoutUpdate) for the same reason.
Dictionary<string, Bucket> bucketDict = BucketCache.GetLastValueWithoutUpdate();
if (bucketDict != null)
{
bucketDict[newBucket.Name] = newBucket;
}
return newBucket;
}
private void RemoveBucket(GcsPath gcsPath, bool removeObjects)
{
if (removeObjects)
{
DeleteObjects(gcsPath);
}
try
{
Service.Buckets.Delete(gcsPath.Bucket).Execute();
}
catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.Conflict)
{
// The objects my not have been cleared yet.
Service.Buckets.Delete(gcsPath.Bucket).Execute();
}
}
private void DeleteObjects(GcsPath gcsPath)
{
string bucketName = gcsPath.Bucket;
List<Task<string>> deleteTasks = new List<Task<string>>();
ObjectsResource.ListRequest request = Service.Objects.List(bucketName);
do
{
Objects gcsObjects = request.Execute();
foreach (Object gcsObject in gcsObjects.Items ?? Enumerable.Empty<Object>())
{
deleteTasks.Add(Service.Objects.Delete(bucketName, gcsObject.Name).ExecuteAsync());
}
request.PageToken = gcsObjects.NextPageToken;
} while (request.PageToken != null && !Stopping);
WaitDeleteTasks(deleteTasks);
}