public async Task WriteAsync(string fullPath, Stream sourceStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckSourceStream(sourceStream);
(CloudBlobContainer container, string path) = await GetPartsAsync(fullPath).ConfigureAwait(false);
if (append)
{
CloudAppendBlob cab = container.GetAppendBlobReference(StoragePath.Normalize(path, false));
if (!await cab.ExistsAsync())
{
await cab.CreateOrReplaceAsync().ConfigureAwait(false);
}
await cab.AppendFromStreamAsync(sourceStream).ConfigureAwait(false);
}
else
{
CloudBlockBlob cloudBlob = container.GetBlockBlobReference(StoragePath.Normalize(path, false));
await cloudBlob.UploadFromStreamAsync(sourceStream).ConfigureAwait(false);
}
}
public async Task DeleteAsync(string fullPath, CancellationToken cancellationToken)
{
DecomposePath(fullPath, out string fs, out string rp, false);
if (StoragePath.IsRootPath(rp))
{
await DeleteFilesystemAsync(fs, cancellationToken).ConfigureAwait(false);
}
else
{
try
{
await InvokeAsync <Void>(
$"{fs}/{rp}?recursive=true",
RequestMethod.Delete,
cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound")
{
// file not found, ignore
}
}
}
public override async Task <StorageFile> MoveAsync(
StoragePath destinationPath,
NameCollisionOption options,
CancellationToken cancellationToken = default
)
{
_ = destinationPath ?? throw new ArgumentNullException(nameof(destinationPath));
if (!ReferenceEquals(destinationPath.FileSystem, FileSystem))
{
throw new ArgumentException(
ExceptionStrings.InMemoryFileSystem.MemberIncompatibleWithInstance(),
nameof(destinationPath)
);
}
if (!EnumInfo.IsDefined(options))
{
throw new ArgumentException(ExceptionStrings.Enum.UndefinedValue(options), nameof(options));
}
return(MoveInternal(destinationPath, options, cancellationToken));
}
private Blob ToBlob(IListBlobItem nativeBlob)
{
Blob blob;
if (nativeBlob is CloudBlockBlob blockBlob)
{
string fullName = StoragePath.Combine(_container.Name, blockBlob.Name);
blob = new Blob(fullName, BlobItemKind.File);
}
else if (nativeBlob is CloudAppendBlob appendBlob)
{
string fullName = StoragePath.Combine(_container.Name, appendBlob.Name);
blob = new Blob(fullName, BlobItemKind.File);
}
else if (nativeBlob is CloudBlobDirectory dirBlob)
{
string fullName = StoragePath.Combine(_container.Name, dirBlob.Prefix);
blob = new Blob(fullName, BlobItemKind.Folder);
}
else
{
throw new InvalidOperationException($"unknown item type {nativeBlob.GetType()}");
}
//attach metadata if we can
if (nativeBlob is CloudBlob cloudBlob)
{
//no need to fetch attributes, parent request includes the details
//await cloudBlob.FetchAttributesAsync().ConfigureAwait(false);
AzureUniversalBlobStorageProvider.AttachBlobMeta(blob, cloudBlob);
}
return(blob);
}
public async Task <string> GetSasUriAsync(
string fullPath,
SharedAccessBlobPolicy sasConstraints,
SharedAccessBlobHeaders headers,
bool createContainer,
CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
(CloudBlobContainer container, string path) = await GetPartsAsync(fullPath, createContainer);
if (container == null)
{
return(null);
}
CloudBlockBlob blob = container.GetBlockBlobReference(StoragePath.Normalize(path, false));
try
{
return($@"{blob.Uri}{blob.GetSharedAccessSignature(sasConstraints, headers)}");
}
catch (AzureStorageException ex)
{
if (AzureStorageValidation.IsDoesntExist(ex))
{
return(null);
}
if (!AzureStorageValidation.TryHandleStorageException(ex))
{
throw;
}
}
throw new InvalidOperationException("must not be here");
}
public override async Task <StorageFolder> CopyAsync(
StoragePath destinationPath,
NameCollisionOption options,
CancellationToken cancellationToken = default
)
{
_ = destinationPath ?? throw new ArgumentNullException(nameof(destinationPath));
if (!ReferenceEquals(destinationPath.FileSystem, FileSystem))
{
throw new ArgumentException(
ExceptionStrings.InMemoryFileSystem.MemberIncompatibleWithInstance(),
nameof(destinationPath)
);
}
if (!EnumInfo.IsDefined(options))
{
throw new ArgumentException(ExceptionStrings.Enum.UndefinedValue(options), nameof(options));
}
cancellationToken.ThrowIfCancellationRequested();
var replaceExisting = options switch
{
NameCollisionOption.Fail => false,
NameCollisionOption.ReplaceExisting => true,
_ => throw new NotSupportedException(ExceptionStrings.Enum.UnsupportedValue(options)),
};
lock (_inMemoryFileSystem.Storage)
{
var folderNode = _storage.GetFolderNode(Path);
folderNode.Copy(destinationPath, replaceExisting);
return(FileSystem.GetFolder(destinationPath.FullPath));
}
}
/// <summary>
/// Handle
/// </summary>
/// <param name="command"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <CommandResponse> Handle(UploadMultipleAttachmentCommand command, CancellationToken cancellationToken)
{
CommandResponse response = new CommandResponse()
{
UploadedAttachments = new List <AttachmentModel>()
};
foreach (var commandFile in command.Files)
{
AttachmentModel attachment = new
AttachmentModel();
// prepare the upload
string originalName = System.IO.Path.GetFileName(commandFile.FileName);
string extension = System.IO.Path.GetExtension(commandFile.FileName).ToLower();
string storedAsFileName = $"{attachment.Id}{extension}";
string fileFullPath = StoragePath.Combine("attachments",
storedAsFileName);
// upload
string fullPath = await _storageService.UploadFile(commandFile, fileFullPath);
attachment.FilePath = fullPath;
// prepare response
attachment.FileName = storedAsFileName;
attachment.FileType = extension;
attachment.ContentType = commandFile.ContentType;
attachment.OriginalFileName = originalName;
attachment.Name = originalName;
attachment.Description = $"Attachment {originalName}";
response.UploadedAttachments.Add(attachment);
}
return(response);
}
/// <summary>
/// Mounts a storage to virtual path
/// </summary>
/// <param name="path"></param>
/// <param name="storage"></param>
public void Mount(string path, IBlobStorage storage)
{
if (path is null)
{
throw new ArgumentNullException(nameof(path));
}
if (storage is null)
{
throw new ArgumentNullException(nameof(storage));
}
path = StoragePath.Normalize(path);
_mountPoints.Add(new Blob(path)
{
Tag = storage
});
string absPath = null;
string[] parts = StoragePath.Split(path);
if (parts.Length == 0) //mount at root
{
MountPath(path, storage, true);
}
else
{
for (int i = 0; i < parts.Length; i++)
{
absPath = StoragePath.Combine(absPath, parts[i]);
MountPath(absPath, storage, i == parts.Length - 1);
}
}
}
public async Task <Blob> GetBlobAsync(string fullPath, CancellationToken cancellationToken)
{
DecomposePath(fullPath, out string fs, out string rp, false);
if (StoragePath.IsRootPath(rp))
{
try
{
(Void _, IDictionary <string, string> headers) = await InvokeExtraAsync <Void>(
$"{fs}?resource=filesystem",
RequestMethod.Head,
cancellationToken).ConfigureAwait(false);
return(AzConvert.ToBlob(fullPath, headers, true));
}
catch (RequestFailedException ex) when(ex.ErrorCode == "FilesystemNotFound")
{
//filesystem doesn't exist
return(null);
}
}
try
{
(Void _, IDictionary <string, string> fheaders) = await InvokeExtraAsync <Void>(
$"{fs}/{rp.UrlEncode()}?action=getProperties",
RequestMethod.Head,
cancellationToken).ConfigureAwait(false);
return(AzConvert.ToBlob(fullPath, fheaders, false));
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound")
{
return(null);
}
}
public static IReadOnlyCollection <Blob> ToBlobs(this ListObjectsV2Response response, ListOptions options)
{
var result = new List <Blob>();
//the files are listed as the S3Objects member, but they don't specifically contain folders,
//but even if they do, they need to be filtered out
result.AddRange(
response.S3Objects
.Where(b => !b.Key.EndsWith("/")) //check if this is "virtual folder" as S3 console creates them (rubbish)
.Select(b => b.ToBlob())
.Where(options.IsMatch)
.Where(b => options.BrowseFilter == null || options.BrowseFilter(b)));
//subfolders are listed in another field (what a funny name!)
//prefix is absolute too
result.AddRange(
response.CommonPrefixes
.Where(p => !StoragePath.IsRootPath(p))
.Select(p => new Blob(p, BlobItemKind.Folder)));
return(result);
}
/// <summary>
/// Deletes files if they exist
/// </summary>
public Task DeleteAsync(IEnumerable <string> fullPaths, CancellationToken cancellationToken)
{
if (fullPaths == null)
{
return(Task.FromResult(true));
}
foreach (string fullPath in fullPaths)
{
GenericValidation.CheckBlobFullPath(fullPath);
string path = GetFilePath(StoragePath.Normalize(fullPath, false));
if (File.Exists(path))
{
File.Delete(path);
}
else if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
}
return(Task.FromResult(true));
}
public Task <IReadOnlyCollection <bool> > ExistsAsync(IEnumerable <string> fullPaths, CancellationToken cancellationToken = default)
{
lock (_lock)
{
ZipArchive zipArchive = GetArchive(false);
if (zipArchive == null)
{
return(Task.FromResult <IReadOnlyCollection <bool> >(new bool[fullPaths.Count()]));
}
var result = new List <bool>();
foreach (string fullPath in fullPaths)
{
string nid = StoragePath.Normalize(fullPath, false);
ZipArchiveEntry entry = zipArchive.GetEntry(nid);
result.Add(entry != null);
}
return(Task.FromResult <IReadOnlyCollection <bool> >(result));
}
}
/// <summary>
/// S3 doesnt support this natively and will cache everything in MemoryStream until disposed.
/// </summary>
public async Task WriteAsync(
string fullPath,
Stream dataStream,
bool append = false,
CancellationToken cancellationToken = default)
{
if (append)
{
throw new NotSupportedException();
}
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
await _client
.PutObjectAsync(
new PutObjectRequest
{
BucketName = this._bucketName,
Key = fullPath,
InputStream = dataStream,
UseChunkEncoding = false,
ServerSideEncryptionMethod = ServerSideEncryptionMethod.None
}, cancellationToken)
.ConfigureAwait(false);
}
public async Task List_FilesInFolder_Recursive()
{
string folderPath = RandomBlobPath();
string id1 = StoragePath.Combine(folderPath, "1.txt");
string id2 = StoragePath.Combine(folderPath, "sub", "2.txt");
string id3 = StoragePath.Combine(folderPath, "sub", "3.txt");
try
{
await _storage.WriteTextAsync(id1, RandomGenerator.RandomString);
await _storage.WriteTextAsync(id2, RandomGenerator.RandomString);
await _storage.WriteTextAsync(id3, RandomGenerator.RandomString);
IReadOnlyCollection <Blob> items = await _storage.ListAsync(recurse : true, folderPath : folderPath);
Assert.Equal(4, items.Count); //1.txt + sub (folder) + 2.txt + 3.txt
}
catch (NotSupportedException)
{
//it ok for providers not to support hierarchy
}
}
public async Task Rename_Folder_Renames()
{
string prefix = RandomBlobPath();
string file1 = StoragePath.Combine(prefix, "old", "1.txt");
string file11 = StoragePath.Combine(prefix, "old", "1", "1.txt");
string file111 = StoragePath.Combine(prefix, "old", "1", "1", "1.txt");
try
{
await _storage.WriteTextAsync(file1, string.Empty);
}
catch (NotSupportedException)
{
return;
}
await _storage.WriteTextAsync(file11, string.Empty);
await _storage.WriteTextAsync(file111, string.Empty);
await _storage.RenameAsync(StoragePath.Combine(prefix, "old"), StoragePath.Combine(prefix, "new"));
IReadOnlyCollection <Blob> list = await _storage.ListAsync(prefix);
}
//...
public bool Coerce(IList input)
{
if (input.Count > Value)
{
//Always >= 1
var blocks = new string[Floor(input.Count.Double() / Value.Double()).Int32()];
switch (Action)
{
case Actions.Clear:
for (var i = 0; i < (blocks.Length * Value); i++)
{
input.RemoveAt(0);
}
break;
case Actions.ClearAndArchive:
if (input is ISerialize serializer)
{
if (Value > 0)
{
for (var i = 0; i < blocks.Length; i++)
{
var items = new object[Value];
for (var j = 0; j < items.Length; j++)
{
items[j] = items[0];
input.RemoveAt(0);
}
blocks[i] = StoragePath.Clone(serializer.FilePath, StoragePath.DefaultCloneFormat, j => System.IO.File.Exists(j));
serializer?.Serialize(blocks[i], items);
}
}
}
else
{
goto case Actions.Clear;
}
break;
case Actions.RemoveFirst:
for (var i = 0; i < (blocks.Length * Value); i++)
{
input.RemoveAt(0);
}
break;
case Actions.RemoveLast:
for (var i = 0; i < (blocks.Length * Value); i++)
{
input.RemoveAt(input.Count - 1);
}
break;
}
return(true);
}
return(false);
}
public async Task RenameAsync(string oldPath, string newPath, CancellationToken cancellationToken = default)
{
AdlsClient client = await GetAdlsClientAsync().ConfigureAwait(false);
await client.RenameAsync(StoragePath.Normalize(oldPath), StoragePath.Normalize(newPath), true, cancellationToken).ConfigureAwait(false);
}
private async Task DeleteAsync(string fullPath)
{
fullPath = StoragePath.Normalize(fullPath);
await _dbfs.Delete(fullPath, true).ConfigureAwait(false);
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory, ILogger <Startup> logger)
{
StoragePath.Initialize(env.ContentRootPath);
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
app.InitializeDatabase();
}
else
{
app.UseExceptionHandler("/Error");
app.UseHsts();
}
app.UseEntityFrameworkLoggingScopeStateProvider();
app.UseHttpsRedirection();
app.UseStaticFiles();
if (!env.IsDevelopment())
{
app.UseSpaStaticFiles();
}
app.UseRouting();
app.UseCors(builder => builder
.AllowAnyOrigin()
.AllowAnyHeader()
.AllowAnyMethod());
app.UseIdentityServer();
app.UseAuthorization();
app.UseSwaggerUi3(settings =>
{
settings.OAuth2Client = new OAuth2ClientSettings {
ClientId = IdentityServerValues.DocumentationClientId, ClientSecret = IdentityServerValues.DocumentationClientSecret
};
settings.Path = "/docs";
settings.DocumentPath = "/docs/api-specification.json";
});
//app.UseCookiePolicy();
//app.UseAuthentication();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllerRoute("default", "{controller}/{action=Index}/{id?}");
endpoints.MapHealthChecks("/health");
});
app.UseSpa(spa =>
{
spa.Options.SourcePath = "ClientApp";
if (env.IsDevelopment())
{
//spa.UseAngularCliServer(npmScript: "serve");
//spa.Options.StartupTimeout = TimeSpan.FromSeconds(120); // Increase the timeout if angular app is taking longer to startup
spa.UseProxyToSpaDevelopmentServer("http://localhost:4200"); // Use this instead to use the angular cli server
}
});
}
/// <summary>
/// Attempts to get the build number from the mods folder.
/// </summary>
private static void GetModsDirectoryBuild()
{
ReadOnlySpan <char> storagePath = StoragePath.AsSpan();
ReadOnlySpan <char> lastDirectory = storagePath[(storagePath.LastIndexOf(Path.DirectorySeparatorChar) + 1)..];
private string GenerateBlobPath(QueueMessage message)
{
return(StoragePath.Combine("message", Guid.NewGuid().ToString()));
}
public static bool TryCreateFromDefaultConfigurationFile(out TelegramBotService telegramBotService)
{
string filename = StoragePath.WithFilename("TelegramBotConfiguration.json");
return(TryCreateFromConfigurationFile(filename, out telegramBotService));
}
protected override async Task <IReadOnlyCollection <Blob> > ListAtAsync(string path, ListOptions options, CancellationToken cancellationToken)
{
IEnumerable <ObjectInfo> objects = await _api.List(StoragePath.Normalize(path)).ConfigureAwait(false);
return(objects.Select(ToBlob).Where(b => b != null).ToList());
}
public void Get_parent_theory(string path, string expected)
{
Assert.Equal(expected, StoragePath.GetParent(path));
}
public void Normalize_theory(string path, string expected)
{
Assert.Equal(expected, StoragePath.Normalize(path));
}
public void Combine_theory(string expected, string[] parts)
{
Assert.Equal(expected, StoragePath.Combine(parts));
}
protected override void DeleteSameNodeTypeAtPathIfExisting(StoragePath destinationPath)
{
Storage.TryGetFileNodeAndThrowOnConflictingFolder(destinationPath)?.Delete();
}
private FileNode(FsDataStorage storage, StoragePath path, FolderNode parent)
: base(storage, path, parent)
{
Content = new FileContent(ownerFileNode: this);
Attributes = FileAttributes.Normal;
}
/// <summary>
/// GCP requires no trailing root
/// </summary>
private static string NormalisePath(string path)
{
path = StoragePath.Normalize(path);
return(path.Substring(1));
}
public NodeSettings(Arguments settings, DataNode section)
{
this.WebLogs = settings.GetBool("web.log", section.GetBool("web.logs"));
this.BlockTime = settings.GetInt("block.time", section.GetInt32("block.time"));
this.MinimumFee = settings.GetInt("minimum.fee", section.GetInt32("minimum.fee"));
this.MinimumPow = settings.GetInt("minimum.pow", section.GetInt32("minimum.pow"));
int maxPow = 5; // should be a constanct like MinimumBlockTime
if (this.MinimumPow < 0 || this.MinimumPow > maxPow)
{
throw new Exception("Proof-Of-Work difficulty has to be between 1 and 5");
}
this.ApiProxyUrl = settings.GetString("api.proxy.url", section.GetString("api.proxy.url"));
if (string.IsNullOrEmpty(this.ApiProxyUrl))
{
this.ApiProxyUrl = null;
}
this.Seeds = section.GetNode("seeds").Children.Select(p => p.Value).ToList();
this.Mode = settings.GetEnum <NodeMode>("node.mode", section.GetEnum <NodeMode>("node.mode", NodeMode.Invalid));
if (this.Mode == NodeMode.Invalid)
{
throw new Exception("Unknown node mode specified");
}
this.NexusName = settings.GetString("nexus.name", section.GetString("nexus.name"));
this.NodeWif = settings.GetString("node.wif", section.GetString("node.wif"));
this.StorageConversion = settings.GetBool("convert.storage", section.GetBool("convert.storage"));
this.ApiLog = settings.GetBool("api.log", section.GetBool("api.log"));
this.NodePort = settings.GetInt("node.port", section.GetInt32("node.port"));
this.NodeHost = settings.GetString("node.host", section.GetString("node.host", "localhost"));
this.ProfilerPath = settings.GetString("profiler.path", section.GetString("profiler.path"));
if (string.IsNullOrEmpty(this.ProfilerPath))
{
this.ProfilerPath = null;
}
this.HasSync = settings.GetBool("has.sync", section.GetBool("has.sync"));
this.HasMempool = settings.GetBool("has.mempool", section.GetBool("has.mempool"));
this.MempoolLog = settings.GetBool("mempool.log", section.GetBool("mempool.log"));
this.HasEvents = settings.GetBool("has.events", section.GetBool("has.events"));
this.HasRelay = settings.GetBool("has.relay", section.GetBool("has.relay"));
this.HasArchive = settings.GetBool("has.archive", section.GetBool("has.archive"));
this.HasRpc = settings.GetBool("has.rpc", section.GetBool("has.rpc"));
this.RpcPort = settings.GetInt("rpc.port", section.GetInt32("rpc.port"));
this.HasRest = settings.GetBool("has.rest", section.GetBool("has.rest"));
this.RestPort = settings.GetInt("rest.port", section.GetInt32("rest.port"));
this.NexusBootstrap = settings.GetBool("nexus.bootstrap", section.GetBool("nexus.bootstrap"));
this.GenesisTimestampUint = settings.GetUInt("genesis.timestamp", section.GetUInt32("genesis.timestamp"));
this.GenesisTimestamp = new Timestamp((this.GenesisTimestampUint == 0) ? Timestamp.Now.Value : this.GenesisTimestampUint);
this.ApiCache = settings.GetBool("api.cache", section.GetBool("api.cache"));
this.Readonly = settings.GetBool("readonly", section.GetBool("readonly"));
this.SenderHost = settings.GetString("sender.host", section.GetString("sender.host"));
this.SenderThreads = settings.GetUInt("sender.threads", section.GetUInt32("sender.threads"));
this.SenderAddressCount = settings.GetUInt("sender.address.count", section.GetUInt32("sender.address.count"));
var defaultStoragePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Storage/";
var defaultOraclePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/Oracle/";
this.StoragePath = settings.GetString("storage.path", section.GetString("storage.path"));
if (string.IsNullOrEmpty(this.StoragePath))
{
this.StoragePath = defaultStoragePath;
}
if (!StoragePath.EndsWith("" + Path.DirectorySeparatorChar))
{
StoragePath += Path.DirectorySeparatorChar;
}
StoragePath = Path.GetFullPath(StoragePath);
this.VerifyStoragePath = settings.GetString("verify.storage.path", section.GetString("verify.storage.path"));
if (string.IsNullOrEmpty(this.VerifyStoragePath))
{
this.VerifyStoragePath = defaultStoragePath;
}
this.OraclePath = settings.GetString("oracle.path", section.GetString("oracle.path"));
if (string.IsNullOrEmpty(this.OraclePath))
{
this.OraclePath = defaultOraclePath;
}
var backend = settings.GetString("storage.backend", section.GetString("storage.backend"));
if (!Enum.TryParse <StorageBackendType>(backend, true, out this.StorageBackend))
{
throw new Exception("Unknown storage backend: " + backend);
}
if (this.StorageConversion)
{
this.RandomSwapData = section.GetBool("random.Swap.data");
}
}
