/// <summary> /// Store the data in the cache. /// </summary> /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param> /// <param name="data">The data as a byte array</param> public void Store(string key, byte[] data) { LeanData.ParseKey(key, out var fileName, out var entryName); // We only support writing to zips with this provider, we also need an entryName to write // Return silently because RemoteFileSubscriptionStreamReader depends on this function not throwing. if (!fileName.EndsWith(".zip", StringComparison.InvariantCulture) || entryName == null) { return; } // Only allow one thread at a time to modify our cache lock (_zipFileCache) { // If its not in the cache, and can not be cached we need to create it if (!_zipFileCache.TryGetValue(fileName, out var cachedZip) && !Cache(fileName, out cachedZip)) { // Create the zip, if successful, cache it for later use if (Compression.ZipCreateAppendData(fileName, entryName, data)) { Cache(fileName, out _); } return; } cachedZip.WriteEntry(entryName, data); } }
/// <summary> /// Fetch data from the cache /// </summary> /// <param name="key">A string representing the key of the cached data</param> /// <returns>An <see cref="Stream"/> of the cached data</returns> public Stream Fetch(string key) { LeanData.ParseKey(key, out var filePath, out var entryName); var stream = _dataProvider.Fetch(filePath); if (filePath.EndsWith(".zip") && stream != null) { // get the first entry from the zip file try { var entryStream = Compression.UnzipStream(stream, out _zipFile, entryName); // save the file stream so it can be disposed later _zipFileStream = stream; return(entryStream); } catch (ZipException exception) { Log.Error("SingleEntryDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception); stream.DisposeSafely(); return(null); } } return(stream); }
/// <summary> /// Store the data in the cache. /// </summary> /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param> /// <param name="data">The data as a byte array</param> public void Store(string key, byte[] data) { LeanData.ParseKey(key, out var fileName, out var entryName); // We only support writing to zips with this provider, we also need an entryName to write // Return silently because RemoteFileSubscriptionStreamReader depends on this function not throwing. if (!fileName.EndsWith(".zip", StringComparison.InvariantCulture) || entryName == null) { return; } // Only allow one thread at a time to modify our cache lock (_zipFileCache) { // If its not in the cache, and can not be cached we need to create it if (!_zipFileCache.TryGetValue(fileName, out var cachedZip) && !Cache(fileName, out cachedZip)) { // Create the zip, if successful, cache it for later use if (Compression.ZipCreateAppendData(fileName, entryName, data)) { Cache(fileName, out _); } else { throw new InvalidOperationException($"Failed to store data {fileName}#{entryName}"); } return; } lock (cachedZip) { if (cachedZip.Disposed) { // if disposed and we have the lock means it's not in the dictionary anymore, let's assert it // but there is a window for another thread to add a **new/different** instance which is okay // we will pick it up on the store call bellow if (_zipFileCache.TryGetValue(fileName, out var existing) && ReferenceEquals(existing, cachedZip)) { Log.Error($"ZipDataCacheProvider.Store(): unexpected cache state for {fileName}"); throw new InvalidOperationException( "LEAN entered an unexpected state. Please contact [email protected] so we may debug this further."); } Store(key, data); } else { cachedZip.WriteEntry(entryName, data); } } } }
/// <summary> /// Fetch data from the cache /// </summary> /// <param name="key">A string representing the key of the cached data</param> /// <returns>An <see cref="Stream"/> of the cached data</returns> public Stream Fetch(string key) { LeanData.ParseKey(key, out var filePath, out var entryName); if (!File.Exists(filePath)) { return(null); } try { using (var zip = ZipFile.Read(filePath)) { ZipEntry entry; if (entryName.IsNullOrEmpty()) { // Return the first entry entry = zip[0]; } else { // Attempt to find our specific entry if (!zip.ContainsEntry(entryName)) { return(null); } entry = zip[entryName]; } // Extract our entry and return it var stream = new MemoryStream(); entry.Extract(stream); stream.Position = 0; return(stream); } } catch (ZipException exception) { Log.Error("DiskDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception); return(null); } }
/// <summary> /// Does not attempt to retrieve any data /// </summary> public Stream Fetch(string key) { LeanData.ParseKey(key, out var filename, out var entryName); // handles zip files if (filename.EndsWith(".zip", StringComparison.InvariantCulture)) { Stream stream = null; try { CachedZipFile existingZip; if (!_zipFileCache.TryGetValue(filename, out existingZip)) { stream = CacheAndCreateEntryStream(filename, entryName); } else { try { lock (existingZip) { if (existingZip.Disposed) { // bad luck, thread race condition // it was disposed and removed after we got it // so lets create it again and add it stream = CacheAndCreateEntryStream(filename, entryName); } else { existingZip.Refresh(); stream = CreateEntryStream(existingZip, entryName, filename); } } } catch (Exception exception) { if (exception is ZipException || exception is ZlibException) { Log.Error("ZipDataCacheProvider.Fetch(): Corrupt zip file/entry: " + filename + "#" + entryName + " Error: " + exception); } else { throw; } } } return(stream); } catch (Exception err) { Log.Error(err, "Inner try/catch"); stream?.DisposeSafely(); return(null); } } else { // handles text files return(_dataProvider.Fetch(filename)); } }
/// <summary> /// Store the data in the cache. Not implemented in this instance of the IDataCacheProvider /// </summary> /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param> /// <param name="data">The data as a byte array</param> public void Store(string key, byte[] data) { LeanData.ParseKey(key, out var filePath, out var entryName); Compression.ZipCreateAppendData(filePath, entryName, data, true); }