private static BlobStorageContext GetBlobStorageContextPrivate(SqlDataReader reader, int fileId, int versionId, int propertyTypeId)
{
// this is a helper method to aid both the sync and
// async version of the GetBlobContext operation
var length = reader.GetSafeInt64(0);
var providerName = reader.GetSafeString(1);
var providerData = reader.GetSafeString(2);
var useFileStream = false;
FileStreamData fsData = null;
if (BlobStorage.FileStreamEnabled)
{
fsData = new FileStreamData
{
Path = reader.GetSafeString(3),
TransactionContext = reader.GetSqlBytes(4).Buffer
};
useFileStream = fsData.Path != null;
}
var provider = BlobStorageBase.GetProvider(providerName);
return(new BlobStorageContext(provider, providerData)
{
VersionId = versionId,
PropertyTypeId = propertyTypeId,
FileId = fileId,
Length = length,
UseFileStream = useFileStream,
BlobProviderData = provider == BlobStorageBase.BuiltInProvider
? new BuiltinBlobProviderData {
FileStreamData = fsData
}
: provider.ParseData(providerData)
});
}
protected internal override BinaryCacheEntity LoadBinaryCacheEntity(int nodeVersionId, int propertyTypeId, out FileStreamData fileStreamData)
{
WriteLog(MethodInfo.GetCurrentMethod(), nodeVersionId, propertyTypeId);
return(base.LoadBinaryCacheEntity(nodeVersionId, propertyTypeId, out fileStreamData));
}
internal static void AddStream(BlobStorageContext context, Stream stream)
{
FileStreamData fileStreamData = null;
var providerData = context.BlobProviderData as BuiltinBlobProviderData;
if (providerData != null)
{
fileStreamData = providerData.FileStreamData;
}
SqlProcedure cmd = null;
try
{
// if possible, write the stream using the special Filestream technology
if (BlobStorageBase.UseFileStream(stream.Length))
{
WriteSqlFileStream(stream, context.FileId, fileStreamData);
return;
}
// We have to work with an integer since SQL does not support
// binary values bigger than [Int32.MaxValue].
var streamSize = Convert.ToInt32(stream.Length);
cmd = new SqlProcedure {
CommandText = "proc_BinaryProperty_WriteStream"
};
cmd.Parameters.Add("@Id", SqlDbType.Int).Value = context.FileId;
var offsetParameter = cmd.Parameters.Add("@Offset", SqlDbType.Int);
var valueParameter = cmd.Parameters.Add("@Value", SqlDbType.VarBinary, streamSize);
if (BlobStorage.FileStreamEnabled)
{
var useFileStreamParameter = cmd.Parameters.Add("@UseFileStream", SqlDbType.TinyInt);
useFileStreamParameter.Value = false;
}
var offset = 0;
byte[] buffer = null;
stream.Seek(0, SeekOrigin.Begin);
// The 'while' loop is misleading here, because we write the whole
// stream at once. Bigger files should go to the Filestream
// column anyway.
while (offset < streamSize)
{
// Buffer size may be less at the end os the stream than the limit
var bufferSize = streamSize - offset;
if (buffer == null || buffer.Length != bufferSize)
{
buffer = new byte[bufferSize];
}
// Read bytes from the source
stream.Read(buffer, 0, bufferSize);
offsetParameter.Value = offset;
valueParameter.Value = buffer;
// Write full stream
cmd.ExecuteNonQuery();
offset += bufferSize;
}
}
finally
{
cmd?.Dispose();
}
}
private static void WriteBinaryStream(Stream stream, int binaryPropertyId, FileStreamData fileStreamData = null)
{
SqlProcedure cmd = null;
try
{
var longStreamSize = stream.Length;
var useFileStream = RepositoryConfiguration.FileStreamEnabled &&
longStreamSize > RepositoryConfiguration.MinimumSizeForFileStreamInBytes;
//If possible, write the stream using the special Filestream technology
if (useFileStream)
{
WriteSqlFileStream(stream, binaryPropertyId, fileStreamData);
return;
}
//We have to work with an integer since SQL does not support
//binary values bigger than [Int32.MaxValue].
var streamSize = Convert.ToInt32(stream.Length);
cmd = new SqlProcedure {
CommandText = "proc_BinaryProperty_WriteStream"
};
cmd.Parameters.Add("@Id", SqlDbType.Int).Value = binaryPropertyId;
var offsetParameter = cmd.Parameters.Add("@Offset", SqlDbType.Int);
var valueParameter = cmd.Parameters.Add("@Value", SqlDbType.VarBinary, streamSize);
if (RepositoryConfiguration.FileStreamEnabled)
{
var useFileStreamParameter = cmd.Parameters.Add("@UseFileStream", SqlDbType.TinyInt);
useFileStreamParameter.Value = useFileStream;
}
int offset = 0;
byte[] buffer = null;
stream.Seek(0, SeekOrigin.Begin);
//The 'while' loop is misleading here, because we write the whole
//stream at once. Bigger files should go to the Filestream
//column anyway.
while (offset < streamSize)
{
// Buffer size may be less at the end os the stream than the limit
int bufferSize = streamSize - offset;
if (buffer == null || buffer.Length != bufferSize)
{
buffer = new byte[bufferSize];
}
// Read bytes from the source
stream.Read(buffer, 0, bufferSize);
offsetParameter.Value = offset;
valueParameter.Value = buffer;
// Write full stream
cmd.ExecuteNonQuery();
offset += bufferSize;
}
}
finally
{
if (cmd != null)
{
cmd.Dispose();
}
}
}
public void UpdateBinaryProperty(int binaryDataId, BinaryDataValue value)
{
if (!RepositoryConfiguration.FileStreamEnabled)
{
// MS-SQL does not support stream size over [Int32.MaxValue],
// but check only if Filestream is not enabled
if (value.Stream != null && value.Stream.Length > Int32.MaxValue)
{
throw new NotSupportedException();
}
}
var isRepositoryStream = value.Stream is RepositoryStream || value.Stream is SenseNetSqlFileStream;
FileStreamData fileStreamData = null;
SqlProcedure cmd = null;
try
{
cmd = new SqlProcedure {
CommandText = "proc_BinaryProperty_Update"
};
cmd.Parameters.Add("@BinaryPropertyId", SqlDbType.Int).Value = binaryDataId;
cmd.Parameters.Add("@ContentType", SqlDbType.NVarChar, 450).Value = value.ContentType;
cmd.Parameters.Add("@FileNameWithoutExtension", SqlDbType.NVarChar, 450).Value = value.FileName.FileNameWithoutExtension == null ? (object)DBNull.Value : (object)value.FileName.FileNameWithoutExtension;
cmd.Parameters.Add("@Extension", SqlDbType.NVarChar, 50).Value = ValidateExtension(value.FileName.Extension);
cmd.Parameters.Add("@Size", SqlDbType.BigInt).Value = value.Size;
// Do not update the stream field in the database if it is not loaded (other change happened)
cmd.Parameters.Add("@IsStreamModified", SqlDbType.TinyInt).Value = isRepositoryStream ? 0 : 1;
cmd.Parameters.Add("@Checksum", SqlDbType.VarChar, 200).Value = (value.Checksum != null) ? (object)value.Checksum : DBNull.Value;
if (RepositoryConfiguration.FileStreamEnabled)
{
string path;
byte[] transactionContext;
//Update row and retrieve file path and
//transaction context for the Filestream column
using (var reader = cmd.ExecuteReader())
{
reader.Read();
path = reader.GetString(0);
transactionContext = reader.GetSqlBytes(1).Buffer;
}
if (!string.IsNullOrEmpty(path))
{
fileStreamData = new FileStreamData {
Path = path, TransactionContext = transactionContext
}
}
;
}
else
{
cmd.ExecuteNonQuery();
}
}
finally
{
if (cmd != null)
{
cmd.Dispose();
}
}
if (!isRepositoryStream && value.Stream != null && value.Stream.Length > 0)
{
// Stream exists and is loaded -> write it
WriteBinaryStream(value.Stream, binaryDataId, fileStreamData);
}
}
public async Task <ActionResult <IEnumerable <UFileReturnDto> > > UploadFiles([FromForm] UFilesReceiveDto filesReceiveDto)
{
if (filesReceiveDto.Files == null || filesReceiveDto.Files.Count == 0)
{
return(BadRequest("No files found"));
}
if (filesReceiveDto.Files.Count > _uploadSettings.Value.MaxFiles)
{
return(BadRequest($"Too many files. Max files is {_uploadSettings.Value.MaxFiles.ToString()}"));
}
// Serialize the info tab
var infoList = filesReceiveDto.Infos == null ? null :
JsonSerializer.Deserialize <List <MultiFileInfoDto> >(filesReceiveDto.Infos, new JsonSerializerOptions()
{
PropertyNameCaseInsensitive = true
});
if (infoList != null && infoList.Count != 0 && infoList.Count != filesReceiveDto.Files.Count)
{
return(BadRequest("Either leave infos empty or add the same amount of info entries as files!"));
}
// before we do any computation or allocation
foreach (var file in filesReceiveDto.Files)
{
if (file.Length > _uploadSettings.Value.MaxSize)
{
return(BadRequest(
$"File too big. Maximum file size allowed is {_uploadSettings.Value.MaxSize.ToString()} bytes"));
}
string extension = Path.GetExtension(file.FileName);
if (_uploadSettings.Value.BlockedExtensions.Any(
e => e.Equals(extension, StringComparison.OrdinalIgnoreCase)))
{
return(BadRequest($"File extension {extension} is not allowed"));
}
}
Album album = null;
// check if specified album exists
if (filesReceiveDto.AlbumId.HasValue)
{
album = await _albumRepo.GetAlbumById(filesReceiveDto.AlbumId.Value);
if (album == null)
{
return(BadRequest("Album doesn't exist."));
}
}
int userId = int.Parse(User.FindFirst(ClaimTypes.NameIdentifier).Value);
// Check if user exceeded upload limit
if (_limitSettings.Value.MaxFiles != 0 && await _fileRepo.GetFileCount(userId) >= _limitSettings.Value.MaxFiles)
{
return(BadRequest($"You have reached the maximum file limit of {_limitSettings.Value.MaxFiles.ToString()}"));
}
var userFromRepo = await _userRepo.GetUserById(userId);
List <FileStreamData> fileStreams = new List <FileStreamData>(filesReceiveDto.Files.Count);
List <UFile> dbFiles = new List <UFile>(filesReceiveDto.Files.Count);
try
{
int index = 0;
foreach (var file in filesReceiveDto.Files)
{
string extension = Path.GetExtension(file.FileName);
string fileId = Guid.NewGuid().ToString();
string path = _utilsService.GenerateFilePath(fileId, extension);
var fileData = new FileStreamData()
{
Path = path,
CopyTask = null,
FileStream = new FileStream(path, FileMode.Create)
};
// begin entire file download and saving
fileData.CopyTask = file.CopyToAsync(fileData.FileStream);
fileStreams.Add(fileData);
// create files for database
bool isPublic = true;
string filename = Path.GetFileNameWithoutExtension(file.FileName);
string name = filename;
if (infoList != null && infoList.Count != 0)
{
isPublic = infoList[index].IsPublic;
name = infoList[index].Name ?? filename;
}
if (string.IsNullOrWhiteSpace(name))
{
name = fileId;
}
var uFile = new UFile(fileId, isPublic, extension, file.ContentType,
userFromRepo.Id, name);
if (album != null)
{
uFile.AlbumId = album.Id;
}
dbFiles.Add(uFile);
index++;
}
// now await the files one after another
foreach (var file in fileStreams)
{
await file.CopyTask;
}
// now we know all files successfully downloaded and saved
// add to the DB
foreach (var dbFile in dbFiles)
{
_fileRepo.Add(dbFile);
}
if (!await _fileRepo.SaveAll())
{
await UndoChanges();
return(BadRequest("Failed saving the files"));
}
var filesToReturn = _mapper.Map <IEnumerable <UFileReturnDto> >(dbFiles);
return(Ok(filesToReturn));
}
catch (Exception e)
{
await UndoChanges();
_logger.Log(LogLevel.Error, e, "Failed to save Files!");
return(BadRequest("Something failed sorry :("));
}
finally
{
foreach (var data in fileStreams)
{
await data.FileStream.DisposeAsync();
data.CopyTask.Dispose();
}
}
async Task UndoChanges()
{
for (int i = 0; i < fileStreams.Count; i++)
{
var file = fileStreams[i];
var dbFile = dbFiles[i];
// wait in case its not finished yet
await file.CopyTask;
await file.FileStream.DisposeAsync();
file.CopyTask.Dispose();
// remove photo if it exists
if (System.IO.File.Exists(file.Path))
{
System.IO.File.Delete(file.Path);
}
// remove Db entries
var fileFromDb = await _fileRepo.GetFile(dbFile.PublicId);
if (fileFromDb != null)
{
_fileRepo.Delete(fileFromDb);
}
}
await _fileRepo.SaveAll();
}
}
internal static Stream GetBinaryStream2(int nodeId, int versionId, int propertyTypeId)
{
FileStreamData fileStreamData = null;
if (TransactionScope.IsActive)
{
// Aktív tranzakció hekk
var bce = DataProvider.Current.LoadBinaryCacheEntity(versionId, propertyTypeId, out fileStreamData);
//If we received the necessary information for serving file stream, use that. If filestream data
//is null, we MUST NOT use SqlFileStream here, because the data is in the database.
if (bce.UseFileStream && fileStreamData != null)
{
return(new ContentRepository.Storage.Data.SqlFileStream(bce.Length, bce.BinaryPropertyId, fileStreamData));
}
return(bce.RawData == null
? new RepositoryStream(bce.Length, bce.BinaryPropertyId)
: new RepositoryStream(bce.BinaryPropertyId, bce.Length, bce.RawData));
}
// Try to load cached binary entity
var cacheKey = BinaryCacheEntity.GetCacheKey(versionId, propertyTypeId);
var binaryCacheEntity = (BinaryCacheEntity)DistributedApplication.Cache.Get(cacheKey);
if (binaryCacheEntity == null)
{
//Not in cache, load it from the database
binaryCacheEntity = DataProvider.Current.LoadBinaryCacheEntity(versionId, propertyTypeId, out fileStreamData);
//insert the binary cache entity into the
//cache only if we know the node id
if (binaryCacheEntity != null && nodeId != 0)
{
if (!RepositoryConfiguration.WorkingMode.Populating)
{
DistributedApplication.Cache.Insert(cacheKey, binaryCacheEntity, new NodeIdDependency(nodeId));
}
}
}
//Not found even in the database
if (binaryCacheEntity == null)
{
return(null);
}
if (binaryCacheEntity.Length == -1)
{
return(null);
}
//We can use SqlFilestream to load the binary from the filesystem.
//Transaction is handled inside the SqlFileStream.
if (binaryCacheEntity.UseFileStream)
{
return(new ContentRepository.Storage.Data.SqlFileStream(binaryCacheEntity.Length, binaryCacheEntity.BinaryPropertyId));
}
var stream = binaryCacheEntity.RawData == null
? new RepositoryStream(binaryCacheEntity.Length, binaryCacheEntity.BinaryPropertyId)
: new RepositoryStream(binaryCacheEntity.BinaryPropertyId, binaryCacheEntity.Length, binaryCacheEntity.RawData);
return(stream);
}
/// <summary>
/// Loads a cache item into memory that either contains the raw binary (if its size fits into the limit) or
/// just the blob metadata pointing to the blob storage.
/// </summary>
/// <param name="versionId">Content version id.</param>
/// <param name="propertyTypeId">Binary property type id.</param>
public BinaryCacheEntity LoadBinaryCacheEntity(int versionId, int propertyTypeId)
{
var columnDefinitions = BlobStorage.FileStreamEnabled
? string.Format(LoadBinaryCacheentityColumnsFormatFilestreamScript, BlobStorage.BinaryCacheSize)
: string.Format(LoadBinaryCacheentityColumnsFormatScript, BlobStorage.BinaryCacheSize);
var commandText = string.Format(LoadBinaryCacheentityFormatScript, columnDefinitions);
using (var cmd = new SqlProcedure {
CommandText = commandText
})
{
cmd.Parameters.Add("@VersionId", SqlDbType.Int).Value = versionId;
cmd.Parameters.Add("@PropertyTypeId", SqlDbType.Int).Value = propertyTypeId;
cmd.CommandType = CommandType.Text;
using (var reader = cmd.ExecuteReader(CommandBehavior.SingleRow | CommandBehavior.SingleResult))
{
if (!reader.HasRows || !reader.Read())
{
return(null);
}
var length = reader.GetInt64(0);
var binaryPropertyId = reader.GetInt32(1);
var fileId = reader.GetInt32(2);
var providerName = reader.GetSafeString(3);
var providerTextData = reader.GetSafeString(4);
byte[] rawData;
if (reader.IsDBNull(5))
{
rawData = null;
}
else
{
rawData = (byte[])reader.GetValue(5);
}
var useFileStream = false;
FileStreamData fileStreamData = null;
if (BlobStorage.FileStreamEnabled)
{
useFileStream = reader.GetInt32(6) == 1;
if (useFileStream)
{
// fill Filestream info if we really need it
fileStreamData = new FileStreamData
{
Path = reader.GetSafeString(7),
TransactionContext = reader.GetSqlBytes(8).Buffer
};
}
}
var provider = BlobStorageBase.GetProvider(providerName);
var context = new BlobStorageContext(provider, providerTextData)
{
VersionId = versionId, PropertyTypeId = propertyTypeId, FileId = fileId, Length = length, UseFileStream = useFileStream
};
if (provider == BlobStorageBase.BuiltInProvider)
{
context.BlobProviderData = new BuiltinBlobProviderData {
FileStreamData = fileStreamData
}
}
;
return(new BinaryCacheEntity
{
Length = length,
RawData = rawData,
BinaryPropertyId = binaryPropertyId,
FileId = fileId,
Context = context
});
}
}
}
/// <summary>
/// Updates an existing binary property value in the database and the blob storage.
/// </summary>
/// <param name="blobProvider">Blob storage provider.</param>
/// <param name="value">Binary data to update.</param>
public void UpdateBinaryProperty(IBlobProvider blobProvider, BinaryDataValue value)
{
var streamLength = value.Stream?.Length ?? 0;
if (blobProvider != BlobStorageBase.BuiltInProvider && streamLength > 0)
{
var ctx = new BlobStorageContext(blobProvider, value.BlobProviderData)
{
VersionId = 0,
PropertyTypeId = 0,
FileId = value.FileId,
Length = streamLength,
UseFileStream = false
};
blobProvider.Allocate(ctx);
using (var stream = blobProvider.GetStreamForWrite(ctx))
value.Stream?.CopyTo(stream);
value.BlobProviderName = ctx.Provider.GetType().FullName;
value.BlobProviderData = BlobStorageContext.SerializeBlobProviderData(ctx.BlobProviderData);
}
else
{
value.BlobProviderName = null;
value.BlobProviderData = null;
}
if (blobProvider == BlobStorageBase.BuiltInProvider && !BlobStorage.FileStreamEnabled)
{
// MS-SQL does not support stream size over [Int32.MaxValue],
// but check only if Filestream is not enabled
if (streamLength > int.MaxValue)
{
throw new NotSupportedException();
}
}
var isRepositoryStream = value.Stream is RepositoryStream || value.Stream is SenseNetSqlFileStream;
var hasStream = isRepositoryStream || value.Stream is MemoryStream;
if (!hasStream)
{
// do not do any database operation if the stream is not modified
return;
}
FileStreamData fileStreamData = null;
SqlProcedure cmd = null;
try
{
string sql;
CommandType commandType;
if (blobProvider == BlobStorageBase.BuiltInProvider)
{
commandType = CommandType.StoredProcedure;
sql = "proc_BinaryProperty_Update";
}
else
{
commandType = CommandType.Text;
sql = BlobStorage.FileStreamEnabled
? UpdateBinarypropertyNewFilerowFilestreamScript
: UpdateBinarypropertyNewFilerowScript;
}
cmd = new SqlProcedure {
CommandText = sql, CommandType = commandType
};
cmd.Parameters.Add("@BinaryPropertyId", SqlDbType.Int).Value = value.Id;
cmd.Parameters.Add("@ContentType", SqlDbType.NVarChar, 450).Value = value.ContentType;
cmd.Parameters.Add("@FileNameWithoutExtension", SqlDbType.NVarChar, 450).Value = value.FileName.FileNameWithoutExtension == null ? DBNull.Value : (object)value.FileName.FileNameWithoutExtension;
cmd.Parameters.Add("@Extension", SqlDbType.NVarChar, 50).Value = ValidateExtension(value.FileName.Extension);
cmd.Parameters.Add("@Size", SqlDbType.BigInt).Value = value.Size;
cmd.Parameters.Add("@Checksum", SqlDbType.VarChar, 200).Value = value.Checksum != null ? (object)value.Checksum : DBNull.Value;
cmd.Parameters.Add("@BlobProvider", SqlDbType.NVarChar, 450).Value = value.BlobProviderName != null ? (object)value.BlobProviderName : DBNull.Value;
cmd.Parameters.Add("@BlobProviderData", SqlDbType.NVarChar, int.MaxValue).Value = value.BlobProviderData != null ? (object)value.BlobProviderData : DBNull.Value;
int fileId;
if (BlobStorage.FileStreamEnabled)
{
string path;
byte[] transactionContext;
// Update row and retrieve file path and
// transaction context for the Filestream column
using (var reader = cmd.ExecuteReader())
{
reader.Read();
fileId = reader.GetInt32(0);
path = reader.GetSafeString(1);
transactionContext = reader.IsDBNull(2) ? null : reader.GetSqlBytes(2).Buffer;
}
if (!string.IsNullOrEmpty(path))
{
fileStreamData = new FileStreamData {
Path = path, TransactionContext = transactionContext
}
}
;
}
else
{
fileId = (int)cmd.ExecuteScalar();
}
if (fileId > 0 && fileId != value.FileId)
{
value.FileId = fileId;
}
}
finally
{
cmd?.Dispose();
}
// ReSharper disable once InvertIf
if (blobProvider == BlobStorageBase.BuiltInProvider && !isRepositoryStream && streamLength > 0)
{
// Stream exists and is loaded -> write it
var ctx = new BlobStorageContext(blobProvider, value.BlobProviderData)
{
VersionId = 0,
PropertyTypeId = 0,
FileId = value.FileId,
Length = streamLength,
UseFileStream = fileStreamData != null,
BlobProviderData = new BuiltinBlobProviderData {
FileStreamData = fileStreamData
}
};
BuiltInBlobProvider.UpdateStream(ctx, value.Stream);
}
}
/// <summary>
/// Inserts a new binary property value into the metadata database and the blob storage,
/// removing the previous one if the content is not new.
/// </summary>
/// <param name="blobProvider">Blob storage provider.</param>
/// <param name="value">Binary data to insert.</param>
/// <param name="versionId">Content version id.</param>
/// <param name="propertyTypeId">Binary property type id.</param>
/// <param name="isNewNode">Whether this value belongs to a new or an existing node.</param>
public void InsertBinaryProperty(IBlobProvider blobProvider, BinaryDataValue value, int versionId, int propertyTypeId, bool isNewNode)
{
var streamLength = value.Stream?.Length ?? 0;
var useFileStream = BlobStorage.FileStreamEnabled &&
streamLength > Convert.ToInt64(BlobStorage.MinimumSizeForFileStreamInBytes);
var ctx = new BlobStorageContext(blobProvider)
{
VersionId = versionId, PropertyTypeId = propertyTypeId, FileId = 0, Length = streamLength, UseFileStream = useFileStream
};
// In case of an external provider allocate the place for bytes and
// write the stream beforehand and get the generated provider data.
// Note that the external provider does not need an existing record
// in the Files table to work, it just stores the bytes.
if (blobProvider != BlobStorageBase.BuiltInProvider && streamLength > 0)
{
blobProvider.Allocate(ctx);
using (var stream = blobProvider.GetStreamForWrite(ctx))
value.Stream?.CopyTo(stream);
value.BlobProviderName = ctx.Provider.GetType().FullName;
value.BlobProviderData = BlobStorageContext.SerializeBlobProviderData(ctx.BlobProviderData);
}
SqlProcedure cmd = null;
FileStreamData fileStreamData = null;
try
{
cmd = useFileStream
? new SqlProcedure {
CommandText = isNewNode ? InsertBinaryPropertyFilestreamScript : DeleteAndInsertBinaryPropertyFilestream, CommandType = CommandType.Text
}
: new SqlProcedure {
CommandText = isNewNode ? InsertBinaryPropertyScript : DeleteAndInsertBinaryProperty, CommandType = CommandType.Text
};
cmd.Parameters.Add("@VersionId", SqlDbType.Int).Value = versionId != 0 ? (object)versionId : DBNull.Value;
cmd.Parameters.Add("@PropertyTypeId", SqlDbType.Int).Value = propertyTypeId != 0 ? (object)propertyTypeId : DBNull.Value;
cmd.Parameters.Add("@ContentType", SqlDbType.NVarChar, 450).Value = value.ContentType;
cmd.Parameters.Add("@FileNameWithoutExtension", SqlDbType.NVarChar, 450).Value = value.FileName.FileNameWithoutExtension == null ? DBNull.Value : (object)value.FileName.FileNameWithoutExtension;
cmd.Parameters.Add("@Extension", SqlDbType.NVarChar, 50).Value = ValidateExtension(value.FileName.Extension);
cmd.Parameters.Add("@Size", SqlDbType.BigInt).Value = Math.Max(0, value.Size);
cmd.Parameters.Add("@BlobProvider", SqlDbType.NVarChar, 450).Value = value.BlobProviderName != null ? (object)value.BlobProviderName : DBNull.Value;
cmd.Parameters.Add("@BlobProviderData", SqlDbType.NVarChar, int.MaxValue).Value = value.BlobProviderData != null ? (object)value.BlobProviderData : DBNull.Value;
cmd.Parameters.Add("@Checksum", SqlDbType.VarChar, 200).Value = value.Checksum != null ? (object)value.Checksum : DBNull.Value;
// insert binary and file rows and retrieve file path and transaction context for the Filestream column
using (var reader = cmd.ExecuteReader())
{
reader.Read();
value.Id = Convert.ToInt32(reader[0]);
value.FileId = Convert.ToInt32(reader[1]);
value.Timestamp = Utility.Convert.BytesToLong((byte[])reader.GetValue(2));
if (useFileStream)
{
fileStreamData = new FileStreamData
{
Path = reader.GetString(3),
TransactionContext = reader.GetSqlBytes(4).Buffer
};
}
}
}
finally
{
cmd.Dispose();
}
// The BuiltIn blob provider saves the stream after the record
// was saved into the Files table, because simple varbinary
// and sql filestream columns must exist before we can write a
// stream into the record.
// ReSharper disable once InvertIf
if (blobProvider == BlobStorageBase.BuiltInProvider && value.Stream != null && value.Stream.Length > 0)
{
ctx.FileId = value.FileId;
ctx.BlobProviderData = new BuiltinBlobProviderData {
FileStreamData = fileStreamData
};
BuiltInBlobProvider.AddStream(ctx, value.Stream);
}
}
public async Task <object> UploadFile(IFormFile file, FileTypeEnum fileType, string domainId, string folderId)
{
try
{
var isImageFile = _pictureService.IsImageFile(file);
byte[] fileBinary;
using (var ms = new MemoryStream())
{
await file.CopyToAsync(ms);
fileBinary = ms.ToArray();
}
var originalName = file.FileName;
var fileEx = !string.IsNullOrEmpty(Path.GetExtension(file.FileName).ToLower()) ? Path.GetExtension(file.FileName).ToLower() : "." + _pictureService.GetImageFormat(fileBinary).ToString();
var uniqueId = UniqueIDHelper.GenarateRandomString(12, false);
var uniqueCode = Guid.NewGuid();
var fileName = $"{uniqueCode.ToString().Replace("-", "")}{fileEx}";
var seoName = _webHelper.GetSeName(file.FileName, true, false);
var isPortraitImage = false;
var pathRoot = _aqFileProvider.GetAbsolutePath(@"contents\");
var fullPath = string.Empty;
var pathSubRoot = string.Empty;
var pathThumb12 = string.Empty;
var pathThumb14 = string.Empty;
var pathThumb16 = string.Empty;
var pathThumb18 = string.Empty;
byte[] fileThumb12;
byte[] fileThumb14;
byte[] fileThumb16;
byte[] fileThumb18;
if (!_aqFileProvider.DirectoryExists(pathRoot))
{
_aqFileProvider.CreateDirectory(pathRoot);
}
if (isImageFile)
{
pathSubRoot = Path.Combine(pathRoot, $@"images\");
}
else
{
pathSubRoot = Path.Combine(pathRoot, $@"docs\");
}
if (!_aqFileProvider.DirectoryExists(pathSubRoot))
{
_aqFileProvider.CreateDirectory(pathSubRoot);
}
domainId = !string.IsNullOrEmpty(domainId.ToLower()) ? domainId.ToLower() : "others";
var pathDomain = Path.Combine(pathSubRoot, $@"{domainId}\");
if (!_aqFileProvider.DirectoryExists(pathDomain))
{
_aqFileProvider.CreateDirectory(pathDomain);
}
folderId = !string.IsNullOrEmpty(folderId.ToLower()) ? folderId.ToLower() : "others";
var saveFilePath = Path.Combine(pathDomain, $@"{folderId}\");
if (!_aqFileProvider.DirectoryExists(saveFilePath))
{
_aqFileProvider.CreateDirectory(saveFilePath);
}
fullPath = Path.Combine(saveFilePath, $"{fileName}");
if (isImageFile)
{
int quality = 100;
//in MB
double fileSize = Convert.ToDouble(fileBinary.Length.ToString()) / 1024 / 1024;
if (fileSize >= 4)
{
quality = 45;
}
else if (fileSize >= 2 && fileSize < 4)
{
quality = 65;
}
else if (fileSize > 0.5 && fileSize < 2)
{
quality = 85;
}
fileBinary = _pictureService.ValidatePicture(fileBinary, 2048, quality);
fileThumb12 = _pictureService.ResizePicture(fileBinary, (int)ThumbRatioEnum.half, quality);
fileThumb14 = _pictureService.ResizePicture(fileBinary, (int)ThumbRatioEnum.quarter, quality);
fileThumb16 = _pictureService.ResizePicture(fileBinary, (int)ThumbRatioEnum.oneSixth, quality);
fileThumb18 = _pictureService.ResizePicture(fileBinary, (int)ThumbRatioEnum.oneEighth, quality);
pathThumb12 = Path.Combine(saveFilePath, $"{uniqueCode.ToString().Replace("-", "")}_12{fileEx}");
pathThumb14 = Path.Combine(saveFilePath, $"{uniqueCode.ToString().Replace("-", "")}_14{fileEx}");
pathThumb16 = Path.Combine(saveFilePath, $"{uniqueCode.ToString().Replace("-", "")}_16{fileEx}");
pathThumb18 = Path.Combine(saveFilePath, $"{uniqueCode.ToString().Replace("-", "")}_18{fileEx}");
isPortraitImage = _pictureService.isPortraitImage(fileBinary);
if (!_aqFileProvider.FileExists(pathThumb12))
{
using (Stream f = File.OpenWrite(pathThumb12))
{
await f.WriteAsync(fileThumb12, 0, fileThumb12.Length);
};
}
if (!_aqFileProvider.FileExists(pathThumb14))
{
using (Stream f = File.OpenWrite(pathThumb14))
{
await f.WriteAsync(fileThumb14, 0, fileThumb14.Length);
};
}
if (!_aqFileProvider.FileExists(pathThumb16))
{
using (Stream f = File.OpenWrite(pathThumb16))
{
await f.WriteAsync(fileThumb16, 0, fileThumb16.Length);
};
}
if (!_aqFileProvider.FileExists(pathThumb18))
{
using (Stream f = File.OpenWrite(pathThumb18))
{
await f.WriteAsync(fileThumb18, 0, fileThumb18.Length);
};
}
}
var virtualPath = fullPath.Replace($@"{_hostingEnvironment.WebRootPath}\", "").Replace(@"\", "/");
var virtuaPathThumb12 = pathThumb12.Replace($@"{_hostingEnvironment.WebRootPath}\", "").Replace(@"\", "/");
var virtuaPathThumb14 = pathThumb14.Replace($@"{_hostingEnvironment.WebRootPath}\", "").Replace(@"\", "/");
var virtuaPathThumb16 = pathThumb16.Replace($@"{_hostingEnvironment.WebRootPath}\", "").Replace(@"\", "/");
var virtuaPathThumb18 = pathThumb18.Replace($@"{_hostingEnvironment.WebRootPath}\", "").Replace(@"\", "/");
if (!_aqFileProvider.DirectoryExists(fullPath))
{
using (Stream f = File.OpenWrite(fullPath))
{
await f.WriteAsync(fileBinary, 0, fileBinary.Length);
};
}
var fileInfo = new FileStreamInfo()
{
UniqueId = uniqueId,
UniqueCode = uniqueCode,
FileTypeFid = Convert.ToInt32(fileType),
OriginalName = originalName,
FileName = fileName,
Seoname = seoName,
FileExtentions = fileEx,
FileSize = fileBinary.Length,
Path = virtualPath,
PathThumb12 = virtuaPathThumb12,
PathThumb14 = virtuaPathThumb14,
PathThumb16 = virtuaPathThumb16,
PathThumb18 = virtuaPathThumb18,
UploadedDateUtc = DateTime.UtcNow,
UploadedBy = "",
Deleted = false,
IsNew = true,
IsPortraitImage = isPortraitImage
};
await InsertFileInfo(fileInfo);
var fileData = new FileStreamData()
{
FileId = fileInfo.FileId,
FileData = fileBinary
};
await InsertFileData(fileData);
return(new FileUploadResponse()
{
FileName = fileInfo.FileName,
FileSize = fileInfo.FileSize,
FileId = fileInfo.FileId
});
}
catch
{
throw;
}
}
public async Task <int> InsertFileData(FileStreamData fileData)
{
var res = await Task.Run(() => _dataContext.FileStreamData.Add(fileData));
return(await _dataContext.SaveChangesAsync());
}
private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
{
cts.Token.ThrowIfCancellationRequested();
var depot = depotFilesData.depotDownloadInfo;
var depotDownloadCounter = depotFilesData.depotCounter;
string chunkID = Util.EncodeHexString(chunk.ChunkID);
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
CDNClient.DepotChunk chunkData = null;
do
{
cts.Token.ThrowIfCancellationRequested();
CDNClient.Server connection = null;
try
{
connection = cdnPool.GetConnection(cts.Token);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
connection, cdnToken, depot.depotKey).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (SteamKitWebRequestException e)
{
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
}while (chunkData == null);
if (chunkData == null)
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();
try
{
await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false);
fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
}
finally
{
fileStreamData.fileLock.Release();
}
int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload);
if (remainingChunks == 0)
{
fileStreamData.fileStream.Dispose();
fileStreamData.fileLock.Dispose();
}
ulong sizeDownloaded = 0;
lock (depotDownloadCounter)
{
sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length;
depotDownloadCounter.SizeDownloaded = sizeDownloaded;
depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength;
depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength;
}
lock (downloadCounter)
{
downloadCounter.TotalBytesCompressed += chunk.CompressedLength;
downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength;
}
if (remainingChunks == 0)
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
}
}
private static void DownloadSteam3AsyncDepotFile(
CancellationTokenSource cts,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
ConcurrentQueue <Tuple <FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData> > networkChunkQueue)
{
cts.Token.ThrowIfCancellationRequested();
var depot = depotFilesData.depotDownloadInfo;
var stagingDir = depotFilesData.stagingDir;
var depotDownloadCounter = depotFilesData.depotCounter;
var oldProtoManifest = depotFilesData.previousManifest;
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{
File.Delete(fileStagingPath);
}
FileStream fs = null;
List <ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
Console.WriteLine("Pre-allocating {0}", fileFinalPath);
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List <ProtoManifest.ChunkData>();
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want
if (Config.VerifyAll)
{
Console.WriteLine("Validating {0}", fileFinalPath);
}
var matchingChunks = new List <ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
var orderedChunks = matchingChunks.OrderBy(x => x.OldChunk.Offset);
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create);
fs.SetLength((long)file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in orderedChunks)
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{
neededChunks.Add(match.NewChunk);
}
else
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
File.Delete(fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
Console.WriteLine("Validating {0}", fileFinalPath);
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
if (neededChunks.Count() == 0)
{
lock (depotDownloadCounter)
{
depotDownloadCounter.SizeDownloaded += (ulong)file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
}
if (fs != null)
{
fs.Dispose();
}
return;
}
else
{
var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
lock (depotDownloadCounter)
{
depotDownloadCounter.SizeDownloaded += sizeOnDisk;
}
}
}
FileStreamData fileStreamData = new FileStreamData
{
fileStream = fs,
fileLock = new SemaphoreSlim(1),
chunksToDownload = neededChunks.Count
};
foreach (var chunk in neededChunks)
{
networkChunkQueue.Enqueue(Tuple.Create(fileStreamData, file, chunk));
}
}
public int CreateIndex(string P_key, string P_value)
{
// Key Should contain ASCII 32-122 only
KeyBytes = ASCIIEncoding.UTF8.GetBytes(P_key);
uint RecordPointer = uint.MaxValue, NewRecordPointer = uint.MaxValue, IndexPointer = uint.MaxValue, NewIndexOffset = 0;
String LastKey = "";
NewRecordPointer = (uint)LastPosition; /// Start of Next Key
IndexOffset = 0;
ByteIndex = 0;
do
{
IndexPosition = (uint)(IndexOffset + (KeyBytes[ByteIndex] - 32) * 8);
count++;
try // Read next Branch (ASCII 32-122)
{
if (KeyBytes[ByteIndex] < 32 || KeyBytes[ByteIndex] > 122)
{
return(-52); // Validate Character Range
}
FileStreamIdx.Seek(IndexPosition, 0);
RecordPointer = BinaryReaderIdx.ReadUInt32();
IndexPointer = BinaryReaderIdx.ReadUInt32();
}
catch (Exception e)
{
StringLastError = e.Message;
return(-1);
}
if (IndexPointer == uint.MaxValue)
{
try
{
FileStreamIdx.Seek(IndexPosition, 0);
BinaryWriterIdx.Write(NewRecordPointer);
BinaryWriterIdx.Write(IndexPosition);
FileStreamIdx.Flush();
return(1); // Inserted new Branch successfully
}
catch (Exception e)
{
StringLastError = e.Message;
return(-1);
}
}
else
{
try
{
FileStreamData.Seek(RecordPointer, 0);
LastKey = BinaryReaderData.ReadString();
LastKeyBytes = ASCIIEncoding.UTF8.GetBytes(LastKey);
}
catch (Exception e)
{
StringLastError = e.Message;
return(-10);
}
Difference = GetDif(ref LastKeyBytes, ref KeyBytes);
if (Difference == 0)
{
if (BinaryReaderData.PeekChar() == 0)
{
return(-3);
}
else
{
FileStreamData.Seek(RecordPointer, 0);
BinaryWriterData.Write(P_key);
BinaryWriterData.Write(false);
BinaryWriterData.Write(P_value);
BinaryWriterData.Flush();
return(-4); /// if the record flag is deleted, update record.
}
}
else
{
if (Difference > 0)
{
if (IndexPointer == IndexPosition)
{
try
{
NewIndexOffset = (uint)FileStreamIdx.Length;
FileStreamIdx.Seek(IndexPosition + 4, 0);
BinaryWriterIdx.Write(NewIndexOffset);
ADDIndexBlock();
IndexPosition = (uint)(NewIndexOffset + (KeyBytes[ByteIndex + 1] - 32) * 8);
FileStreamIdx.Seek(IndexPosition, 0);
BinaryWriterIdx.Write(NewRecordPointer);
BinaryWriterIdx.Write(IndexPosition);
FileStreamIdx.Flush();
return(1); // Inserted new key successfully
}
catch (Exception e)
{
StringLastError = e.Message;
return(-16);
}
}
else
{
IndexOffset = IndexPointer;
}
}
else
{
if (IndexPointer == IndexPosition)
{
try
{
NewIndexOffset = (uint)FileStreamIdx.Length;
FileStreamIdx.Seek(IndexPosition, 0);
BinaryWriterIdx.Write(NewRecordPointer);
BinaryWriterIdx.Write(NewIndexOffset);
ADDIndexBlock();
NewRecordPointer = RecordPointer;
KeyBytes = ASCIIEncoding.UTF8.GetBytes(LastKey);
P_key = LastKey;
IndexOffset = NewIndexOffset;
IndexPosition = (uint)(NewIndexOffset + (KeyBytes[ByteIndex + 1] - 32) * 8);
FileStreamIdx.Seek(IndexPosition, 0);
BinaryWriterIdx.Write(NewRecordPointer);
BinaryWriterIdx.Write(IndexPosition);
FileStreamIdx.Flush();
return(1); // Inserted new key successfully
}
catch (Exception e)
{
StringLastError = e.Message;
return(-17);
}
}
else
{
try
{
FileStreamIdx.Seek(IndexPosition, 0);
BinaryWriterIdx.Write(NewRecordPointer);
NewRecordPointer = RecordPointer;
KeyBytes = ASCIIEncoding.UTF8.GetBytes(LastKey);
P_key = LastKey;
FileStreamIdx.Flush();
IndexOffset = IndexPointer;
}
catch (Exception e)
{
StringLastError = e.Message;
return(-18);
}
}
}
}
}
ByteIndex++;
} while (ByteIndex < KeyBytes.Length);
return(-20);
}
