private Dictionary <string, string> GenerateStorageHeaders(StratusAsset asset, MemoryStream stream)
{
//the HTTP headers only accept letters and digits
StringBuilder fixedName = new StringBuilder();
bool appended = false;
foreach (char letter in asset.Name)
{
char c = (char)(0x000000ff & (uint)letter);
if (c == 127 || (c < ' ' && c != '\t'))
{
continue;
}
else
{
fixedName.Append(letter);
appended = true;
}
}
if (!appended)
{
fixedName.Append("empty");
}
Dictionary <string, string> headers = new Dictionary <string, string>
{
{ "ETag", OpenSim.Framework.Util.Md5Hash(stream) },
{ "X-Object-Meta-Temp", asset.Temporary ? "1" : "0" },
{ "X-Object-Meta-Local", asset.Local ? "1" : "0" },
{ "X-Object-Meta-Type", asset.Type.ToString() },
{ "X-Object-Meta-Name", fixedName.ToString() }
};
if (!Config.Settings.Instance.EnableCFOverwrite)
{
headers.Add("If-None-Match", "*");
}
stream.Position = 0;
return(headers);
}
internal MemoryStream StoreAsset(StratusAsset asset)
{
MemoryStream stream = new MemoryStream();
try
{
ProtoBuf.Serializer.Serialize <StratusAsset>(stream, asset);
stream.Position = 0;
string assetIdStr = asset.Id.ToString();
Dictionary <string, string> mheaders = this.GenerateStorageHeaders(asset, stream);
this.WarnIfLongOperation("CreateObject",
() => _provider.CreateObject(GenerateContainerName(assetIdStr), stream, GenerateAssetObjectName(assetIdStr),
"application/octet-stream", headers: mheaders, useInternalUrl: Config.Settings.Instance.CFUseInternalURL,
region: Config.Settings.Instance.CFDefaultRegion)
);
stream.Position = 0;
return(stream);
}
catch (ResponseException e)
{
stream.Dispose();
if (e.Response.StatusCode == System.Net.HttpStatusCode.PreconditionFailed)
{
throw new AssetAlreadyExistsException(String.Format("Asset {0} already exists and can not be overwritten", asset.Id));
}
else
{
throw;
}
}
catch
{
stream.Dispose();
throw;
}
}
public void StoreAsset(AssetBase asset)
{
if (asset == null)
{
throw new ArgumentNullException("asset cannot be null");
}
if (asset.FullID == OpenMetaverse.UUID.Zero)
{
throw new ArgumentException("assets must not have a null ID");
}
StratusAsset wireAsset = StratusAsset.FromAssetBase(asset);
//for now we're not going to use compression etc, so set to zero
wireAsset.StorageFlags = 0;
//attempt to cache the full (wire) asset in case the CF StoreAsset call takes a while or throws (or both)
// Call this with a null stream in order to pre-cache the asset before the StoreAsset delay.
// Makes the asset available immediately after upload in spite of CF write delays.
if (this.CacheAssetIfAppropriate(asset.FullID, null, wireAsset))
{
statPutCached++;
}
statTotal++;
statPut++;
if (Config.Settings.Instance.UseAsyncStore)
{
// Now do the actual CF storage asychronously so as to not block the caller.
_threadPool.QueueWorkItem(() => StoreCFAsset(asset, wireAsset));
}
else
{
// This is a blocking write, session/caller will be blocked awaiting completion.
StoreCFAsset(asset, wireAsset);
}
}
private void DoTimeout(AssetBase asset, StratusAsset wireAsset, Exception e)
{
if (!Config.Settings.Instance.DisableWritebackCache)
{
//eat the exception and write locally
m_log.ErrorFormat("[InWorldz.Stratus]: Timeout attempting to store asset {0}. Storing locally.", asset.FullID);
_diskWriteBack.StoreAsset(wireAsset);
}
else
{
m_log.ErrorFormat("[InWorldz.Stratus]: Timeout attempting to store asset {0}: {1}", asset.FullID, e);
throw new AssetServerException(String.Format("Timeout attempting to store asset {0}: {1}", asset.FullID, e.Message), e);
}
}
// Returns true if it added it to the cache
private bool CacheAssetIfAppropriate(OpenMetaverse.UUID assetId, System.IO.MemoryStream stream, StratusAsset asset)
{
if (!Config.Settings.Instance.CFUseCache) return false;
if (stream.Length > Config.Constants.MAX_CACHEABLE_ASSET_SIZE)
{
statBigStream++;
return false;
}
if (asset.Data.Length > Config.Constants.MAX_CACHEABLE_ASSET_SIZE)
{
statBigAsset++;
return false;
}
lock (_assetCache)
{
if (!_assetCache.HasAsset(assetId))
{
//we do not yet have this asset. we need to make a determination if caching the stream
//or caching the asset would be more beneficial
if (stream.Length > Config.Constants.MAX_STREAM_CACHE_SIZE)
{
//asset is too big for caching the stream to have any theoretical benefit.
//instead we cache the asset itself
_assetCache.CacheAssetData(assetId, asset);
}
else
{
//caching the stream should make for faster retrival and collection
_assetCache.CacheAssetData(assetId, stream);
}
return true; // now cached, in one form or the other
}
}
statDupUpdate++;
return false;
}
private void StoreCFAsset(AssetBase asset, StratusAsset wireAsset)
{
bool isRetry = false;
Util.Retry(2, new List <Type> {
typeof(AssetAlreadyExistsException), typeof(UnrecoverableAssetServerException)
}, () =>
{
CloudFilesAssetWorker worker;
System.IO.MemoryStream assetStream = null;
ulong start = Util.GetLongTickCount();
try
{
worker = _asyncAssetWorkers.LeaseObject();
}
catch (Exception e)
{
statPutInit++;
throw new UnrecoverableAssetServerException(e.Message, e);
}
try
{
if (Config.Settings.Instance.UnitTest_ThrowTimeout)
{
throw new System.Net.WebException("Timeout for unit testing", System.Net.WebExceptionStatus.Timeout);
}
using (assetStream = worker.StoreAsset(wireAsset))
{
this.CacheAssetIfAppropriate(asset.FullID, assetStream, wireAsset);
}
}
catch (AssetAlreadyExistsException)
{
if (!isRetry) //don't throw if this is a retry. this can happen if a write times out and then succeeds
{
statPutExists++;
throw;
}
}
catch (System.Net.WebException e)
{
if (e.Status == System.Net.WebExceptionStatus.Timeout || e.Status == System.Net.WebExceptionStatus.RequestCanceled)
{
statPutTO++;
DoTimeout(asset, wireAsset, e);
}
else
{
statPutExceptWeb++;
ReportThrowStorageError(asset, e);
}
}
catch (System.IO.IOException e)
{
//this sucks, i think timeouts on writes are causing .net to claim the connection
//was forcibly closed by the remote host.
if (e.Message.Contains("forcibly closed"))
{
statPutNTO++;
DoTimeout(asset, wireAsset, e);
}
else
{
statPutExceptIO++;
ReportThrowStorageError(asset, e);
}
}
catch (Exception e)
{
statPutExcept++;
m_log.ErrorFormat("[InWorldz.Stratus]: Unable to store asset {0}: {1}", asset.FullID, e);
throw new AssetServerException(String.Format("Unable to store asset {0}: {1}", asset.FullID, e.Message), e);
}
finally
{
ulong elapsed = Util.GetLongTickCount() - start;
statPuts.Add(elapsed);
isRetry = true;
_asyncAssetWorkers.ReturnObject(worker);
}
});
}
// Updated to allow a null to be passed for the stream so that we can add the asset to the cache before a write completes.
// (It may be called again with both once the write completes.)
// Returns true if it added it to the cache
private bool CacheAssetIfAppropriate(OpenMetaverse.UUID assetId, System.IO.MemoryStream stream, StratusAsset asset)
{
if (!Config.Settings.Instance.CFUseCache)
{
return(false);
}
if ((stream != null) && (stream.Length > Config.Constants.MAX_CACHEABLE_ASSET_SIZE))
{
statBigStream++;
return(false);
}
if (asset.Data.Length > Config.Constants.MAX_CACHEABLE_ASSET_SIZE)
{
if (stream == null)
{
statBigAsset++; // only increment if this is the followup call
}
return(false);
}
lock (_assetCache)
{
//we do not yet have this asset. we need to make a determination if caching the stream
//or caching the asset would be more beneficial
if ((stream == null) || (stream.Length > Config.Constants.MAX_STREAM_CACHE_SIZE))
{
//asset is too big for caching the stream to have any theoretical benefit.
//instead we cache the asset itself
_assetCache.CacheAssetData(assetId, asset);
}
else
{
//caching the stream should make for faster retrival and collection
_assetCache.CacheAssetData(assetId, stream);
}
}
return(true); // now cached, in one form or the other
}
private AssetBase GetAssetInternal(OpenMetaverse.UUID assetID)
{
// Quick exit for null ID case.
statTotal++;
statGet++;
if (assetID == OpenMetaverse.UUID.Zero)
{
statGetHit++;
return(null);
}
//cache?
Cache.CacheEntry cacheObject = null;
lock (_assetCache)
{
_assetCache.TryGetAsset(assetID, out cacheObject);
}
StratusAsset rawAsset = null;
if (cacheObject != null)
{
statGetHit++;
//stream cache or asset cache?
if (cacheObject.FullAsset != null)
{
rawAsset = cacheObject.FullAsset;
}
else
{
using (System.IO.MemoryStream stream = new System.IO.MemoryStream(cacheObject.Data, 0, cacheObject.Size))
{
rawAsset = DeserializeAssetFromStream(assetID, stream);
}
}
}
else
{
StratusAsset diskAsset = null;
if (!Config.Settings.Instance.DisableWritebackCache)
{
diskAsset = _diskWriteBack.GetAsset(assetID.Guid);
}
if (diskAsset != null)
{
rawAsset = diskAsset;
}
else
{
Util.Retry(2, new List <Type> {
typeof(UnrecoverableAssetServerException)
}, () =>
{
ulong start = Util.GetLongTickCount();
CloudFilesAssetWorker worker = null;
try
{
try
{
//nothing on the local disk, request from CF
worker = _asyncAssetWorkers.LeaseObject();
}
catch (Exception e)
{
//exception here is unrecoverable since this is construction
statGetInit++;
throw new UnrecoverableAssetServerException(e.Message, e);
}
using (System.IO.MemoryStream stream = worker.GetAsset(assetID))
{
statGetFetches++;
stream.Position = 0;
rawAsset = DeserializeAssetFromStream(assetID, stream);
//if we're using the cache, we need to put the raw data in there now
stream.Position = 0;
this.CacheAssetIfAppropriate(assetID, stream, rawAsset);
}
}
catch (net.openstack.Core.Exceptions.Response.ItemNotFoundException)
{
statGetNotFound++;
//not an exceptional case. this will happen
rawAsset = null;
}
finally
{
ulong elapsed = Util.GetLongTickCount() - start;
statGets.Add(elapsed);
if (worker != null)
{
_asyncAssetWorkers.ReturnObject(worker);
}
}
});
}
}
//nothing?
if (rawAsset == null)
{
return(null);
}
//convert
return(rawAsset.ToAssetBase());
}
public void StoreAsset(AssetBase asset)
{
if (asset == null)
{
throw new ArgumentNullException("asset cannot be null");
}
if (asset.FullID == OpenMetaverse.UUID.Zero)
{
throw new ArgumentException("assets must not have a null ID");
}
bool isRetry = false;
StratusAsset wireAsset = StratusAsset.FromAssetBase(asset);
//for now we're not going to use compression etc, so set to zero
wireAsset.StorageFlags = 0;
Util.Retry(2, new List <Type> {
typeof(AssetAlreadyExistsException), typeof(UnrecoverableAssetServerException)
}, () =>
{
CloudFilesAssetWorker worker;
try
{
worker = _asyncAssetWorkers.LeaseObject();
}
catch (Exception e)
{
throw new UnrecoverableAssetServerException(e.Message, e);
}
try
{
if (Config.Settings.Instance.UnitTest_ThrowTimeout)
{
throw new System.Net.WebException("Timeout for unit testing", System.Net.WebExceptionStatus.Timeout);
}
using (System.IO.MemoryStream assetStream = worker.StoreAsset(wireAsset))
{
//cache the stored asset to eliminate roudtripping when
//someone performs an upload
this.CacheAssetIfAppropriate(asset.FullID, assetStream, wireAsset);
}
}
catch (AssetAlreadyExistsException)
{
if (!isRetry) //don't throw if this is a retry. this can happen if a write times out and then succeeds
{
throw;
}
}
catch (System.Net.WebException e)
{
if (e.Status == System.Net.WebExceptionStatus.Timeout || e.Status == System.Net.WebExceptionStatus.RequestCanceled)
{
DoTimeout(asset, wireAsset, e);
}
else
{
ReportThrowStorageError(asset, e);
}
}
catch (System.IO.IOException e)
{
//this sucks, i think timeouts on writes are causing .net to claim the connection
//was forcibly closed by the remote host.
if (e.Message.Contains("forcibly closed"))
{
DoTimeout(asset, wireAsset, e);
}
else
{
ReportThrowStorageError(asset, e);
}
}
catch (Exception e)
{
m_log.ErrorFormat("[InWorldz.Stratus]: Unable to store asset {0}: {1}", asset.FullID, e);
throw new AssetServerException(String.Format("Unable to store asset {0}: {1}", asset.FullID, e.Message), e);
}
finally
{
isRetry = true;
_asyncAssetWorkers.ReturnObject(worker);
}
});
}
private void CacheAssetIfAppropriate(OpenMetaverse.UUID assetId, System.IO.MemoryStream stream, StratusAsset asset)
{
if (!Config.Settings.Instance.CFUseCache)
{
return;
}
if (stream.Length > Config.Constants.MAX_CACHEABLE_ASSET_SIZE)
{
return;
}
if (asset.Data.Length > Config.Constants.MAX_CACHEABLE_ASSET_SIZE)
{
return;
}
lock (_assetCache)
{
if (!_assetCache.HasAsset(assetId))
{
//we do not yet have this asset. we need to make a determination if caching the stream
//or caching the asset would be more beneficial
if (stream.Length > Config.Constants.MAX_STREAM_CACHE_SIZE)
{
//asset is too big for caching the stream to have any theoretical benefit.
//instead we cache the asset itself
_assetCache.CacheAssetData(assetId, asset);
}
else
{
//caching the stream should make for faster retrival and collection
_assetCache.CacheAssetData(assetId, stream);
}
}
}
}
private Dictionary<string, string> GenerateStorageHeaders(StratusAsset asset, MemoryStream stream)
{
//the HTTP headers only accept letters and digits
StringBuilder fixedName = new StringBuilder();
bool appended = false;
foreach (char letter in asset.Name)
{
char c = (char) (0x000000ff & (uint) letter);
if (c == 127 || (c < ' ' && c != '\t'))
{
continue;
}
else
{
fixedName.Append(letter);
appended = true;
}
}
if (!appended) fixedName.Append("empty");
Dictionary<string, string> headers = new Dictionary<string, string>
{
{"ETag", OpenSim.Framework.Util.Md5Hash(stream)},
{"X-Object-Meta-Temp", asset.Temporary ? "1" : "0"},
{"X-Object-Meta-Local", asset.Local ? "1" : "0"},
{"X-Object-Meta-Type", asset.Type.ToString()},
{"X-Object-Meta-Name", fixedName.ToString()}
};
if (!Config.Settings.Instance.EnableCFOverwrite)
{
headers.Add("If-None-Match", "*");
}
stream.Position = 0;
return headers;
}
internal MemoryStream StoreAsset(StratusAsset asset)
{
MemoryStream stream = new MemoryStream();
try
{
ProtoBuf.Serializer.Serialize<StratusAsset>(stream, asset);
stream.Position = 0;
string assetIdStr = asset.Id.ToString();
Dictionary<string, string> mheaders = this.GenerateStorageHeaders(asset, stream);
this.WarnIfLongOperation("CreateObject",
() => _provider.CreateObject(GenerateContainerName(assetIdStr), stream, GenerateAssetObjectName(assetIdStr),
"application/octet-stream", headers: mheaders, useInternalUrl: Config.Settings.Instance.CFUseInternalURL,
region: Config.Settings.Instance.CFDefaultRegion)
);
stream.Position = 0;
return stream;
}
catch (ResponseException e)
{
stream.Dispose();
if (e.Response.StatusCode == System.Net.HttpStatusCode.PreconditionFailed)
{
throw new AssetAlreadyExistsException(String.Format("Asset {0} already exists and can not be overwritten", asset.Id));
}
else
{
throw;
}
}
catch
{
stream.Dispose();
throw;
}
}
