/// <summary>
/// Initializes a new instance of the <see cref="T:Chattel.AssetServerWHIP"/> class by opening up a connection to the remote WHIP server.
/// </summary>
/// <param name="serverTitle">Server title. Used to provide a visuall handle for this server in the logs.</param>
/// <param name="host">Host.</param>
/// <param name="port">Port.</param>
/// <param name="password">Password.</param>
public AssetServerWHIP(string serverTitle, string host, int port, string password)
{
_serverHandle = serverTitle;
Host = host;
Port = port;
Password = password;
_provider = new RemoteServer(Host, (ushort)Port, Password);
_provider.Start(); // TODO: this needs to be started when needed, and shutdown after no usage for a period of time. Noted that the library doesn't like repeated start stops, it seems to not keep up the auth info, so the whole _whipServer instance would need to be scrapped and reinitialized.
var status = _provider.GetServerStatus();
LOG.Log(Logging.LogLevel.Info, () => $"[{_serverHandle}] WHIP connection prepared for host {Host}:{Port}\n'{status}'.");
}
public IEnumerable <LogMessage> GetLogMessagesByJobId(IStorageConnection connection, string jobId, int from = 1, int count = 10)
{
var logMessages = new List <LogMessage>();
try
{
int counterValue = GetCounterValue(connection, jobId);
int toValue = count > counterValue ? counterValue : count;
int fromValue = from <= 0 ? 1 : from;
foreach (int i in Enumerable.Range(fromValue, toValue))
{
var logMessageHash = connection.GetAllEntriesFromHash(Util.GetKeyName(i, jobId));
if (logMessageHash != null && logMessageHash.Any())
{
var logMessage = SerializationHelper
.Deserialize <LogMessage>(logMessageHash.FirstOrDefault().Value);
logMessages.Add(logMessage);
}
}
}
catch (Exception ex)
{
var logLine = $"Error Read Log Messages. Exception Message = {ex.Message}, StackTrace = {ex.ToString()}";
HangfireInternalLog.Log(Logging.LogLevel.Error, () => logLine);
Debug.WriteLine(logLine);
}
return(logMessages);
}
public static void Log(string jobId, LogLevel logLevel, string logMessage)
{
try
{
if (string.IsNullOrWhiteSpace(jobId))
{
return;
}
var item = Util.GetLoggerContextName(jobId);
if (JobsLoggerFilter.Loggers[item] is LoggerContext loggerContext &&
loggerContext.IsEnabled(logLevel))
{
var context = loggerContext.PfContext;
using (var connection = context.Storage.GetConnection())
{
var jobExpirationTimeout = context.Storage.JobExpirationTimeout;
loggerContext.SaveLogMessage(connection, jobId, jobExpirationTimeout, logLevel, logMessage);
}
}
}
catch (Exception ex)
{
var logLine = $"Error Write Log. Exception Message = {ex.Message}, StackTrace = {ex.ToString()}";
HangfireInternalLog.Log(Logging.LogLevel.Error, () => logLine);
Trace.WriteLine(logLine);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="T:Chattel.AssetServerCF"/> class by opening up and warming the connection to CF.
/// </summary>
/// <param name="serverTitle">Server title. Used to provide a visuall handle for this server in the logs.</param>
/// <param name="username">Username.</param>
/// <param name="apiKey">API key.</param>
/// <param name="defaultRegion">Default region.</param>
/// <param name="useInternalUrl">If set to <c>true</c> use internal URL.</param>
/// <param name="containerPrefix">Container prefix.</param>
public AssetServerCF(string serverTitle, string username, string apiKey, string defaultRegion, bool useInternalUrl, string containerPrefix)
{
_serverHandle = serverTitle;
Username = username;
APIKey = apiKey;
DefaultRegion = defaultRegion;
UseInternalURL = useInternalUrl;
ContainerPrefix = containerPrefix;
var identity = new CloudIdentity {
Username = Username, APIKey = APIKey
};
var restService = new InWorldz.Data.Assets.Stratus.CoreExt.ExtendedJsonRestServices(DEFAULT_READ_TIMEOUT, DEFAULT_WRITE_TIMEOUT);
_provider = new InWorldz.Data.Assets.Stratus.CoreExt.ExtendedCloudFilesProvider(identity, DefaultRegion, null, restService);
//warm up
_provider.GetAccountHeaders(useInternalUrl: UseInternalURL, region: DefaultRegion);
LOG.Log(Logging.LogLevel.Info, () => $"[{_serverHandle}] CF connection prepared for region '{DefaultRegion}' and prefix '{ContainerPrefix}' under user '{Username}'.");
}
/// <summary>
/// Initializes a new instance of the <see cref="T:ChattelConfiguration"/> class.
/// If the localStoragePath is null, empty, or references a folder that doesn't exist or doesn't have write access, the local storage of assets will be disabled.
/// The serialParallelServerConfigs parameter allows you to specify server groups that should be accessed serially with subgroups that should be accessed in parallel.
/// Eg. if you have a new server you want to be hit for all operations, but to fallback to whichever of two older servers returns first, then set up a pattern like [ [ primary ], [ second1, second2 ] ].
/// </summary>
/// <param name="localStoragePath">Local storage folder path. Folder must exist or caching will be disabled.</param>
/// <param name="writeCachePath">Path to the write cache file.</param>
/// <param name="writeCacheRecordCount">Number of entries in the write cache file to support.</param>
/// <param name="serialParallelServers">Serially-accessed parallel servers.</param>
public ChattelConfiguration(string localStoragePath, string writeCachePath, uint writeCacheRecordCount, IEnumerable <IEnumerable <IAssetServer> > serialParallelServers)
{
// Set up caching
if (string.IsNullOrWhiteSpace(localStoragePath))
{
LOG.Log(Logging.LogLevel.Info, () => $"Local storage path is empty, caching assets disabled.");
}
else if (!Directory.Exists(localStoragePath))
{
LOG.Log(Logging.LogLevel.Info, () => $"Local storage path folder does not exist, caching assets disabled.");
}
else
{
LocalStorageFolder = new DirectoryInfo(localStoragePath);
LOG.Log(Logging.LogLevel.Info, () => $"Local storage of assets enabled at {LocalStorageFolder.FullName}");
}
// Set up server handlers
var serialParallelAssetServers = new List <List <IAssetServer> >();
SerialParallelAssetServers = serialParallelAssetServers;
// Copy server handle lists so that the list cannot be changed from outside.
if (serialParallelServers != null && serialParallelServers.Any())
{
if (string.IsNullOrWhiteSpace(writeCachePath) || !LocalStorageEnabled || writeCacheRecordCount <= 0) // Write cache only makes sense when there's both a cache AND upstream servers.
{
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache file path is empty, write cache record count is zero, or local storage is disabled. Crash recovery will be compromised.");
}
else
{
WriteCacheFile = new FileInfo(writeCachePath);
WriteCacheRecordCount = writeCacheRecordCount;
LOG.Log(Logging.LogLevel.Info, () => $"Write cache enabled at {WriteCacheFile.FullName} with {WriteCacheRecordCount} records.");
}
foreach (var parallelServers in serialParallelServers)
{
var parallelServerConnectors = new List <IAssetServer>();
foreach (var serverConnector in parallelServers)
{
if (serverConnector != null)
{
parallelServerConnectors.Add(serverConnector);
}
}
if (parallelServerConnectors.Any())
{
serialParallelAssetServers.Add(parallelServerConnectors);
}
}
}
else
{
LOG.Log(Logging.LogLevel.Warn, () => "Servers empty or not specified. No asset servers connectors configured.");
}
}
/// <summary>
/// Opens or creates the write cache file. If there are entries in the file that are marked as not uploaded, then
/// this ctor loads those assets from the local storage and uploads them to the remotes passed in via the ChattelWriter instance.
/// </summary>
/// <param name="fileInfo">FileInfo instance for the path where to load or create the write cache file.</param>
/// <param name="recordCount">Record count to set the write cache to.</param>
/// <param name="writer">ChattelWriter instance for uploading un-finished assets to on load.</param>
/// <param name="localStorage">Local storage instace to load unfinished assets from.</param>
/// <exception cref="T:Chattel.ChattelConfigurationException">Thrown if there are assets marked as needing to be uploaded but the current configuration prevents uploading.</exception>
public WriteCache(FileInfo fileInfo, uint recordCount, ChattelWriter writer, IChattelLocalStorage localStorage)
{
_fileInfo = fileInfo ?? throw new ArgumentNullException(nameof(fileInfo));
if (recordCount < 2)
{
throw new ArgumentOutOfRangeException(nameof(recordCount), "Having less than two record makes no sense and causes errors.");
}
// If the file doesn't exist, create it and zero the needed records.
if (!_fileInfo.Exists)
{
LOG.Log(Logging.LogLevel.Info, () => $"Write cache file doesn't exist, creating and formatting file '{_fileInfo.FullName}'");
Initialize(recordCount);
_fileInfo.Refresh();
LOG.Log(Logging.LogLevel.Debug, () => $"Write cache formatting complete.");
}
var writeCacheFileRecordCount = (uint)((_fileInfo.Length - WRITE_CACHE_MAGIC_NUMBER.Length) / WriteCacheNode.BYTE_SIZE);
if (writeCacheFileRecordCount < recordCount)
{
// Expand the file.
Expand(recordCount - writeCacheFileRecordCount);
}
else if (writeCacheFileRecordCount > recordCount)
{
// For now, use the file size.
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache not able to be shrunk in this version of Chattel, continuing with old value of {writeCacheFileRecordCount} records instead of requested {recordCount} records.");
recordCount = writeCacheFileRecordCount;
// TODO: find a way to shrink the file without losing ANY of the records that have not yet been submitted to an upstream server.
// Could get difficult in the case of a full file...
}
LOG.Log(Logging.LogLevel.Info, () => $"Reading write cache from file '{_fileInfo.FullName}'. Expecting {recordCount} records, found {writeCacheFileRecordCount} records, choosing the larger.");
_writeCacheNodes = Read(out IEnumerable <WriteCacheNode> assetsToBeSentUpstream).ToArray();
LOG.Log(Logging.LogLevel.Debug, () => $"Reading write cache complete.");
if (assetsToBeSentUpstream.Any())
{
if (writer == null)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there is no asset writer!");
}
if (localStorage == null)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there no cache to read them from!");
}
if (!writer.HasUpstream)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there are no remote servers configured!");
}
}
// Send the assets to the remote server. Yes do this in the startup thread: if you can't access the servers, then why continue?
foreach (var assetCacheNode in assetsToBeSentUpstream)
{
LOG.Log(Logging.LogLevel.Debug, () => $"Attempting to remotely store {assetCacheNode.AssetId}.");
if (localStorage.TryGetAsset(assetCacheNode.AssetId, out var asset))
{
try {
writer.PutAssetSync(asset);
}
catch (AssetExistsException) {
// Ignore these.
LOG.Log(Logging.LogLevel.Info, () => $"Remote server reports that the asset with ID {assetCacheNode.AssetId} already exists.");
}
ClearNode(assetCacheNode);
}
else
{
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache indicates asset {assetCacheNode.AssetId} has not been sent upstream, but the cache reports that there's no such asset!.");
}
}
// Bootstrap the system.
GetNextAvailableNode();
}
/// <summary>
/// Gets the asset from the server.
/// </summary>
/// <returns>The asset.</returns>
/// <param name="assetId">Asset identifier.</param>
/// <param name="handler">Callback delegate to hand the asset to.</param>
/// <param name="cacheRule">Bitfield controlling how local storage is to be handled when used as a cache for remote servers.</param>
public void GetAssetAsync(Guid assetId, AssetHandler handler, CacheRule cacheRule)
{
// Ask for null, get null.
if (assetId == Guid.Empty)
{
handler(null);
}
// TODO: see if https://github.com/Reactive-Extensions/Rx.NET would do a better job, but they have to finish releasing 4.0 first.
// It might be beneficial to move the listener processsing to another thread, but then you potentially lose parallism across multiple asset IDs.
StratusAsset result = null;
while (true)
{
// Hit up the local storage first. If there's no upstream then ignore skipread.
if (!(cacheRule.HasFlag(CacheRule.SkipRead) && _config.SerialParallelAssetServers.Any()) && (_localStorage?.TryGetAsset(assetId, out result) ?? false))
{
handler(result);
return;
}
var listeners = new Queue <AssetHandler>();
listeners.Enqueue(handler); // Add myself to the new listeners list first thing, assuming, probably wrongly, that the following test is true. If wrong, meh: this queue gets dropped like an old potato.
if (_idsBeingFetched.TryAdd(assetId, listeners))
{
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
if (parallelServers.Count() == 1) // Optimization: no need to hit up the parallel stuff if there's only 1.
{
result = parallelServers.First().RequestAssetSync(assetId);
}
else
{
result = parallelServers.AsParallel().Select(server => server.RequestAssetSync(assetId)).FirstOrDefault(a => a != null);
}
if (result != null)
{
if (!cacheRule.HasFlag(CacheRule.SkipWrite))
{
_localStorage?.StoreAsset(result);
}
break;
}
}
// Now to process the listeners.
var exceptions = new ConcurrentQueue <Exception>();
lock (listeners) { // Prevent new listeners from being added.
Parallel.ForEach(listeners, waiting_handler => {
if (waiting_handler == null)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Attempted to process a handler for assetId {assetId} that was null!");
return;
}
try {
waiting_handler(result);
}
catch (Exception e) {
exceptions.Enqueue(e);
}
});
_idsBeingFetched.TryRemove(assetId, out var trash);
}
if (exceptions.Count > 0)
{
LOG.Log(Logging.LogLevel.Error, () => $"Exceptions ({exceptions.Count}) were thrown by handler(s) listening for asset {assetId}", new AggregateException(exceptions));
}
return; // We're done here.
}
// See if we can add ourselves to the listener list.
if (_idsBeingFetched.TryGetValue(assetId, out listeners))
{
// Skiplock: if the lock cannot be taken, move on to the retry because the list is already being emptied.
var lockTaken = false;
try {
Monitor.TryEnter(listeners, ref lockTaken);
if (lockTaken)
{
listeners.Enqueue(handler);
return;
}
}
finally {
if (lockTaken)
{
Monitor.Exit(listeners);
}
}
// lock was skipped, therefore that list is already being cleaned out.
}
// It's gone already, so let's try again as the asset should be in local storage or we should query the servers again.
Thread.Sleep(50);
}
}
/// <summary>
/// Sends the asset to the asset servers.
/// Throws AssetExistsException or AggregateException.
/// </summary>
/// <param name="asset">The asset to store.</param>
public void PutAssetSync(StratusAsset asset)
{
asset = asset ?? throw new ArgumentNullException(nameof(asset));
if (asset.Id == Guid.Empty)
{
throw new ArgumentException("Asset cannot have zero ID.", nameof(asset));
}
// Handle parallel calls with the same asset ID.
var firstLock = new ReaderWriterLockSlim();
try {
firstLock.EnterWriteLock();
var activeLock = _activeWriteLocks.GetOrAdd(asset.Id, firstLock);
if (firstLock != activeLock)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Another thread already storing asset with ID {asset.Id}, halting this call until the first completes, then just returning.");
// There's another thread currently adding this exact ID, so we need to wait on it so that we return when it's actually ready for a GET.
activeLock.EnterReadLock();
activeLock.ExitReadLock();
return;
}
// Hit up local storage first.
if (_localStorage?.TryGetAsset(asset.Id, out var result) ?? false)
{
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
// Lock is cleared in the finally clause.
throw new AssetExistsException(asset.Id);
}
var exceptions = new List <Exception>();
var success = false;
WriteCacheNode activeNode = null;
// First step: get it in local storage.
try {
_localStorage?.StoreAsset(asset);
if (HasUpstream)
{
// Write to writecache file. In this way if we crash after this point we can recover and send the asset to the servers.
activeNode = _writeCache.WriteNode(asset);
// If that fails, it'll throw.
}
else
{
// Set success if there're no upstream servers. This supports applications that act as asset servers.
success = true;
}
}
catch (WriteCacheFullException e) {
LOG.Log(Logging.LogLevel.Warn, () => $"Local cache is full, attempting remote servers before failing.", e);
exceptions.Add(e);
}
catch (Exception e) {
exceptions.Add(e);
}
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
// Remember each iteration of this loop is going through serially accessed blocks of parallel-access servers.
// Therefore any failure or problem in one of the blocks means to just continue to the next block.
try {
if (parallelServers.Count() == 1)
{
parallelServers.First().StoreAssetSync(asset);
}
else
{
var errorBag = new ConcurrentBag <Exception>();
Parallel.ForEach(parallelServers, server => {
try {
server.StoreAssetSync(asset);
}
catch (Exception e) {
errorBag.Add(e);
}
});
if (errorBag.Count >= parallelServers.Count())
{
// If all the servers choked, then pass the buck.
throw new AggregateException(errorBag);
}
}
if (activeNode != null)
{
_writeCache.ClearNode(activeNode);
}
success = true;
break; // It was successfully stored in the first bank of parallel servers, don't do the next bank.
}
catch (AssetException e) {
exceptions.Add(e);
}
catch (AggregateException e) {
// Unwind the aggregate one layer.
foreach (var ex in e.InnerExceptions)
{
exceptions.Add(ex);
}
}
catch (Exception e) {
exceptions.Add(e);
}
}
if (!success)
{
throw new AggregateException("Unable to store asset anywhere. See inner exceptions for details.", exceptions);
}
}
finally {
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
firstLock.ExitWriteLock();
}
}
/// <summary>
/// Requests that an asset be fetched from the folder tree.
/// </summary>
/// <returns><c>true</c>, if get asset was found, <c>false</c> otherwise.</returns>
/// <param name="assetId">Asset identifier.</param>
/// <param name="asset">The resulting asset.</param>
public bool TryGetAsset(Guid assetId, out StratusAsset asset)
{
if (!_config.LocalStorageEnabled)
{
asset = null;
return(false);
}
// Convert the UUID into a path.
var path = UuidToLocalPath(assetId);
if (_assetsBeingWritten.TryGetValue(path, out asset))
{
LOG.Log(Logging.LogLevel.Debug, () => $"Attempted to read an asset from local storage, but another thread is writing it. Shortcutting read of {path}");
// Asset is currently being pushed to disk, so might as well return it now since I have it in memory.
return(true);
}
// Attempt to read and return that file. This needs to handle happening from multiple threads in case a given asset is read from multiple threads at the same time.
var removeFile = false;
try {
using (var stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)) {
asset = Serializer.Deserialize <StratusAsset>(stream);
}
return(true);
}
catch (PathTooLongException e) {
_config.DisableLocalStorage();
LOG.Log(Logging.LogLevel.Error, () => "[ASSET_READER] Attempted to read a locally stored asset, but the path was too long for the filesystem. Disabling local storage.", e);
}
catch (DirectoryNotFoundException) {
// Kinda expected if that's an item that's not been stored locally.
}
catch (UnauthorizedAccessException e) {
_config.DisableLocalStorage();
LOG.Log(Logging.LogLevel.Error, () => "[ASSET_READER] Attempted to read a locally stored asset, but this user is not allowed access. Disabling local storage.", e);
}
catch (FileNotFoundException) {
// Kinda expected if that's an item that's not been stored locally.
}
catch (IOException e) {
// This could be temporary.
LOG.Log(Logging.LogLevel.Warn, () => "[ASSET_READER] Attempted to read a locally stored asset, but there was an IO error.", e);
}
catch (ProtoException e) {
LOG.Log(Logging.LogLevel.Warn, () => $"[ASSET_READER] Attempted to read a locally stored asset, but there was a protobuf decoding error. Removing the offending local storage file as it is either corrupt or from an older installation: {path}", e);
removeFile = true;
}
if (removeFile)
{
try {
File.Delete(path);
// TODO: at some point the folder tree should be checked for folders that should be removed.
}
#pragma warning disable RECS0022 // A catch clause that catches System.Exception and has an empty body
catch {
// If there's a delete failure it'll just keep trying as the asset is called for again.
}
#pragma warning restore RECS0022 // A catch clause that catches System.Exception and has an empty body
}
// Nope, no ability to get the asset.
asset = null;
return(false);
}