/// <summary>
/// Store the data in the cache.
/// </summary>
/// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
/// <param name="data">The data as a byte array</param>
public void Store(string key, byte[] data)
{
LeanData.ParseKey(key, out var fileName, out var entryName);
// We only support writing to zips with this provider, we also need an entryName to write
// Return silently because RemoteFileSubscriptionStreamReader depends on this function not throwing.
if (!fileName.EndsWith(".zip", StringComparison.InvariantCulture) || entryName == null)
{
return;
}
// Only allow one thread at a time to modify our cache
lock (_zipFileCache)
{
// If its not in the cache, and can not be cached we need to create it
if (!_zipFileCache.TryGetValue(fileName, out var cachedZip) && !Cache(fileName, out cachedZip))
{
// Create the zip, if successful, cache it for later use
if (Compression.ZipCreateAppendData(fileName, entryName, data))
{
Cache(fileName, out _);
}
return;
}
cachedZip.WriteEntry(entryName, data);
}
}
/// <summary>
/// Fetch data from the cache
/// </summary>
/// <param name="key">A string representing the key of the cached data</param>
/// <returns>An <see cref="Stream"/> of the cached data</returns>
public Stream Fetch(string key)
{
LeanData.ParseKey(key, out var filePath, out var entryName);
var stream = _dataProvider.Fetch(filePath);
if (filePath.EndsWith(".zip") && stream != null)
{
// get the first entry from the zip file
try
{
var entryStream = Compression.UnzipStream(stream, out _zipFile, entryName);
// save the file stream so it can be disposed later
_zipFileStream = stream;
return(entryStream);
}
catch (ZipException exception)
{
Log.Error("SingleEntryDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception);
stream.DisposeSafely();
return(null);
}
}
return(stream);
}
/// <summary>
/// Store the data in the cache.
/// </summary>
/// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
/// <param name="data">The data as a byte array</param>
public void Store(string key, byte[] data)
{
LeanData.ParseKey(key, out var fileName, out var entryName);
// We only support writing to zips with this provider, we also need an entryName to write
// Return silently because RemoteFileSubscriptionStreamReader depends on this function not throwing.
if (!fileName.EndsWith(".zip", StringComparison.InvariantCulture) || entryName == null)
{
return;
}
// Only allow one thread at a time to modify our cache
lock (_zipFileCache)
{
// If its not in the cache, and can not be cached we need to create it
if (!_zipFileCache.TryGetValue(fileName, out var cachedZip) && !Cache(fileName, out cachedZip))
{
// Create the zip, if successful, cache it for later use
if (Compression.ZipCreateAppendData(fileName, entryName, data))
{
Cache(fileName, out _);
}
else
{
throw new InvalidOperationException($"Failed to store data {fileName}#{entryName}");
}
return;
}
lock (cachedZip)
{
if (cachedZip.Disposed)
{
// if disposed and we have the lock means it's not in the dictionary anymore, let's assert it
// but there is a window for another thread to add a **new/different** instance which is okay
// we will pick it up on the store call bellow
if (_zipFileCache.TryGetValue(fileName, out var existing) && ReferenceEquals(existing, cachedZip))
{
Log.Error($"ZipDataCacheProvider.Store(): unexpected cache state for {fileName}");
throw new InvalidOperationException(
"LEAN entered an unexpected state. Please contact [email protected] so we may debug this further.");
}
Store(key, data);
}
else
{
cachedZip.WriteEntry(entryName, data);
}
}
}
}
/// <summary>
/// Fetch data from the cache
/// </summary>
/// <param name="key">A string representing the key of the cached data</param>
/// <returns>An <see cref="Stream"/> of the cached data</returns>
public Stream Fetch(string key)
{
LeanData.ParseKey(key, out var filePath, out var entryName);
if (!File.Exists(filePath))
{
return(null);
}
try
{
using (var zip = ZipFile.Read(filePath))
{
ZipEntry entry;
if (entryName.IsNullOrEmpty())
{
// Return the first entry
entry = zip[0];
}
else
{
// Attempt to find our specific entry
if (!zip.ContainsEntry(entryName))
{
return(null);
}
entry = zip[entryName];
}
// Extract our entry and return it
var stream = new MemoryStream();
entry.Extract(stream);
stream.Position = 0;
return(stream);
}
}
catch (ZipException exception)
{
Log.Error("DiskDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception);
return(null);
}
}
/// <summary>
/// Does not attempt to retrieve any data
/// </summary>
public Stream Fetch(string key)
{
LeanData.ParseKey(key, out var filename, out var entryName);
// handles zip files
if (filename.EndsWith(".zip", StringComparison.InvariantCulture))
{
Stream stream = null;
try
{
CachedZipFile existingZip;
if (!_zipFileCache.TryGetValue(filename, out existingZip))
{
stream = CacheAndCreateEntryStream(filename, entryName);
}
else
{
try
{
lock (existingZip)
{
if (existingZip.Disposed)
{
// bad luck, thread race condition
// it was disposed and removed after we got it
// so lets create it again and add it
stream = CacheAndCreateEntryStream(filename, entryName);
}
else
{
existingZip.Refresh();
stream = CreateEntryStream(existingZip, entryName, filename);
}
}
}
catch (Exception exception)
{
if (exception is ZipException || exception is ZlibException)
{
Log.Error("ZipDataCacheProvider.Fetch(): Corrupt zip file/entry: " + filename + "#" + entryName + " Error: " + exception);
}
else
{
throw;
}
}
}
return(stream);
}
catch (Exception err)
{
Log.Error(err, "Inner try/catch");
stream?.DisposeSafely();
return(null);
}
}
else
{
// handles text files
return(_dataProvider.Fetch(filename));
}
}
/// <summary>
/// Store the data in the cache. Not implemented in this instance of the IDataCacheProvider
/// </summary>
/// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
/// <param name="data">The data as a byte array</param>
public void Store(string key, byte[] data)
{
LeanData.ParseKey(key, out var filePath, out var entryName);
Compression.ZipCreateAppendData(filePath, entryName, data, true);
}
