/// <summary>
/// Download data from an HTTP URL and insert the result into the
/// cache. If the data is already in the cache, this returns
/// a cached value. An explicit key is provided rather than the URL itself.
/// </summary>
/// <param name="blobCache">The blob cache associated with the action.</param>
/// <param name="key">The key to store with.</param>
/// <param name="url">The URL to download.</param>
/// <param name="headers">An optional Dictionary containing the HTTP
/// request headers.</param>
/// <param name="fetchAlways">Force a web request to always be issued, skipping the cache.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
/// <returns>The data downloaded from the URL.</returns>
public IObservable <byte[]> DownloadUrl(IBlobCache blobCache, string key, string url, IDictionary <string, string> headers = null, bool fetchAlways = false, DateTimeOffset?absoluteExpiration = null)
{
if (blobCache is null)
{
throw new ArgumentNullException(nameof(blobCache));
}
var doFetch = MakeWebRequest(new Uri(url), headers).SelectMany(x => ProcessWebResponse(x, url, absoluteExpiration));
var fetchAndCache = doFetch.SelectMany(x => blobCache.Insert(key, x, absoluteExpiration).Select(_ => x));
IObservable <byte[]> ret;
if (!fetchAlways)
{
ret = blobCache.Get(key).Catch(fetchAndCache);
}
else
{
ret = fetchAndCache;
}
var conn = ret.PublishLast();
conn.Connect();
return(conn);
}
/// <summary>
/// This is the non-generic analog of JsonSerializationMixin.GetAndFetchLatest[1]
/// We shouldn't make modifications to this that alter its behavior from the generic
/// version. By having this we can keep our two GetAndRefresh methods extremely
/// similar and thus trust that what works in one will work in the other.
///
/// 1. https://github.com/akavache/Akavache/blob/1b19bb56d/Akavache/Portable/JsonSerializationMixin.cs#L202-L236
/// </summary>
static IObservable <byte[]> GetAndFetchLatestBytes(
this IBlobCache blobCache,
string key,
Func <IObservable <byte[]> > fetchFunc,
Func <DateTimeOffset, bool> fetchPredicate = null,
DateTimeOffset?absoluteExpiration = null)
{
var fetch = Observable.Defer(() => blobCache.GetCreatedAt(key))
.Select(x => fetchPredicate == null || x == null || fetchPredicate(x.Value))
.Where(predicateIsTrue => predicateIsTrue)
.SelectMany(_ =>
{
return(fetchFunc()
.SelectMany(x => blobCache.Invalidate(key).Select(__ => x))
.SelectMany(x => blobCache.Insert(key, x, absoluteExpiration).Select(__ => x)));
});
var result = blobCache.Get(key).Select(x => Tuple.Create(x, true))
.Catch(Observable.Return(new Tuple <byte[], bool>(null, false)));
return(result
.SelectMany(x => x.Item2
? Observable.Return(x.Item1)
: Observable.Empty <byte[]>())
.Concat(fetch)
.Replay()
.RefCount());
}
/// <summary>
/// Stores the artwork and returns a key to retrieve the artwork again.
/// </summary>
public async Task Store(string key, byte[] data)
{
if (data == null)
{
throw new ArgumentNullException(nameof(data));
}
await this.storageSemaphore.Wait(key);
if (await this.cache.GetCreatedAt(key) != null)
{
this.storageSemaphore.Release(key);
return;
}
this.Log().Debug($"Adding new artwork {key} to the BlobCache");
try
{
await queue.EnqueueObservableOperation(1, () => cache.Insert(key, data));
this.Log().Debug($"Added artwork {key} to the BlobCache");
}
finally
{
this.storageSemaphore.Release(key);
}
}
public async Task AddToCache <T>(string cacheName, T objectToAdd)
{
await cache.Insert
(
cacheName,
Encoding.UTF8.GetBytes(JsonSerializer.Serialize(objectToAdd)),
DateTimeOffset.Now.AddDays(1)
);
}
/// <summary>
/// Save all the data we pull off the internet in the cache for later user.
/// </summary>
/// <param name="file"></param>
/// <param name="serverModifiedType"></param>
/// <param name="filedata"></param>
/// <param name="cache"></param>
/// <returns></returns>
public static IObservable <Unit> SaveFileInCache(this IFile file, string serverModifiedType, byte[] filedata, IBlobCache cache)
{
var timeToDelete = DateTime.Now + Settings.CacheFilesTime;
var insertDate = cache.InsertObject(file.FileDateKey(), serverModifiedType, timeToDelete);
var insertData = cache.Insert(file.FileDataKey(), filedata, timeToDelete);
return(Observable.Concat(insertDate, insertData).Skip(1));
}
/// <summary>
/// Inserts a item into the cache.
/// </summary>
/// <param name="blobCache">The blob cache to insert the item into.</param>
/// <param name="key">The key to associate with the entry.</param>
/// <param name="data">The data for the entry.</param>
/// <param name="expiration">A timespan that will be added to the current DateTime.</param>
/// <returns>A observable which will signal when the item is added.</returns>
public static IObservable <Unit> Insert(this IBlobCache blobCache, string key, byte[] data, TimeSpan expiration)
{
if (blobCache is null)
{
throw new ArgumentNullException(nameof(blobCache));
}
return(blobCache.Insert(key, data, blobCache.Scheduler.Now + expiration));
}
/// <summary>
/// Insert an object into the cache, via the JSON serializer.
/// </summary>
/// <typeparam name="T">The type of item.</typeparam>
/// <param name="blobCache">The cache to insert the item.</param>
/// <param name="key">The key to associate with the object.</param>
/// <param name="value">The object to serialize.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
/// <returns>An observable which signals when the insertion has completed.</returns>
public static IObservable <Unit> InsertObject <T>(this IBlobCache blobCache, string key, T value, DateTimeOffset?absoluteExpiration = null)
{
if (blobCache is IObjectBlobCache objCache)
{
return(objCache.InsertObject(key, value, absoluteExpiration));
}
var bytes = SerializeObject(value);
return(blobCache.Insert(GetTypePrefixedKey(key, typeof(T)), bytes, absoluteExpiration));
}
/// <summary>
/// Inserts the specified key/value pairs into the blob.
/// </summary>
/// <param name="blobCache">The blob cache to insert the values to.</param>
/// <param name="keyValuePairs">The key/value to insert.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
/// <returns>A observable which signals when complete.</returns>
public static IObservable <Unit> Insert(this IBlobCache blobCache, IDictionary <string, byte[]> keyValuePairs, DateTimeOffset?absoluteExpiration = null)
{
if (blobCache is IBulkBlobCache bulkCache)
{
return(bulkCache.Insert(keyValuePairs, absoluteExpiration));
}
return(keyValuePairs.ToObservable()
.SelectMany(x => blobCache.Insert(x.Key, x.Value, absoluteExpiration))
.TakeLast(1));
}
/// <summary>
/// Insert an object into the cache, via the JSON serializer.
/// </summary>
/// <param name="key">The key to associate with the object.</param>
/// <param name="value">The object to serialize.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
public static IObservable <Unit> InsertObject <T>(this IBlobCache This, string key, T value, DateTimeOffset?absoluteExpiration = null)
{
var objCache = This as IObjectBlobCache;
if (objCache != null)
{
return(objCache.InsertObject(key, value, absoluteExpiration));
}
var bytes = SerializeObject(value);
return(This.Insert(key, bytes, absoluteExpiration));
}
public object this[string key, Type type]
{
set
{
var valueByteArray = serializationManager.SerializeToByteArray(value, type);
blobCache.Insert(key, valueByteArray).Wait();
}
}
static IObservable <Unit> VacuumIfNecessary(IBlobCache blobCache, bool force = false)
{
const string key = "__Vacuumed";
var vauumInterval = TimeSpan.FromDays(30);
return(Observable.Defer(() => blobCache.GetCreatedAt(key))
.Where(lastVacuum => force || !lastVacuum.HasValue || blobCache.Scheduler.Now - lastVacuum.Value > vauumInterval)
.SelectMany(Observable.Defer(blobCache.Vacuum))
.SelectMany(Observable.Defer(() => blobCache.Insert(key, new byte[] { 1 })))
.Catch <Unit, Exception>(ex =>
{
log.Error("Could not vacuum image cache", ex);
return Observable.Return(Unit.Default);
})
.AsCompletion());
}
static IObservable <byte[]> ProcessAndCacheWebResponse(WebResponse wr, string url, IBlobCache cache, DateTimeOffset?absoluteExpiration)
{
var hwr = (HttpWebResponse)wr;
if ((int)hwr.StatusCode >= 400)
{
return(Observable.Throw <byte[]>(new WebException(hwr.StatusDescription)));
}
var ms = new MemoryStream();
hwr.GetResponseStream().CopyTo(ms);
var ret = ms.ToArray();
cache.Insert(url, ret, absoluteExpiration);
return(Observable.Return(ret));
}
/// <summary>
/// Download data from an HTTP URL and insert the result into the
/// cache. If the data is already in the cache, this returns
/// a cached value. An explicit key is provided rather than the URL itself.
/// </summary>
/// <param name="key">The key to store with.</param>
/// <param name="url">The URL to download.</param>
/// <param name="headers">An optional Dictionary containing the HTTP
/// request headers.</param>
/// <param name="fetchAlways">Force a web request to always be issued, skipping the cache.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
/// <returns>The data downloaded from the URL.</returns>
public IObservable<byte[]> DownloadUrl(IBlobCache This, string key, string url, IDictionary<string, string> headers = null, bool fetchAlways = false, DateTimeOffset? absoluteExpiration = null)
{
var doFetch = MakeWebRequest(new Uri(url), headers).SelectMany(x => ProcessWebResponse(x, url, absoluteExpiration));
var fetchAndCache = doFetch.SelectMany(x => This.Insert(key, x, absoluteExpiration).Select(_ => x));
var ret = default(IObservable<byte[]>);
if (!fetchAlways)
{
ret = This.Get(key).Catch(fetchAndCache);
}
else
{
ret = fetchAndCache;
}
var conn = ret.PublishLast();
conn.Connect();
return conn;
}
public static async Task<List<string>> GenerateDatabase(IBlobCache targetCache, int size)
{
var ret = new List<string>();
// Write out in groups of 4096
while (size > 0)
{
var toWriteSize = Math.Min(4096, size);
var toWrite = GenerateRandomDatabaseContents(toWriteSize);
await targetCache.Insert(toWrite);
foreach (var k in toWrite.Keys) ret.Add(k);
size -= toWrite.Count;
Console.WriteLine(size);
}
return ret;
}
/// <summary>
/// Download data from an HTTP URL and insert the result into the
/// cache. If the data is already in the cache, this returns
/// a cached value. An explicit key is provided rather than the URL itself.
/// </summary>
/// <param name="key">The key to store with.</param>
/// <param name="url">The URL to download.</param>
/// <param name="headers">An optional Dictionary containing the HTTP
/// request headers.</param>
/// <param name="fetchAlways">Force a web request to always be issued, skipping the cache.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
/// <returns>The data downloaded from the URL.</returns>
public IObservable <byte[]> DownloadUrl(IBlobCache This, string key, string url, IDictionary <string, string> headers = null, bool fetchAlways = false, DateTimeOffset?absoluteExpiration = null)
{
var doFetch = MakeWebRequest(new Uri(url), headers).SelectMany(x => ProcessWebResponse(x, url, absoluteExpiration));
var fetchAndCache = doFetch.SelectMany(x => This.Insert(key, x, absoluteExpiration).Select(_ => x));
var ret = default(IObservable <byte[]>);
if (!fetchAlways)
{
ret = This.Get(key).Catch(fetchAndCache);
}
else
{
ret = fetchAndCache;
}
var conn = ret.PublishLast();
conn.Connect();
return(conn);
}
/// <summary>
/// Tests generating a database.
/// </summary>
/// <param name="targetCache">The target blob cache.</param>
/// <param name="size">The number of items to generate.</param>
/// <returns>A list of generated items.</returns>
public static async Task <List <string> > GenerateDatabase(IBlobCache targetCache, int size)
{
var ret = new List <string>();
// Write out in groups of 4096
while (size > 0)
{
var toWriteSize = Math.Min(4096, size);
var toWrite = GenerateRandomDatabaseContents(toWriteSize);
await targetCache.Insert(toWrite);
foreach (var k in toWrite.Keys)
{
ret.Add(k);
}
size -= toWrite.Count;
Console.WriteLine(size);
}
return(ret);
}
IObservable<byte[]> ProcessAndCacheWebResponse(WebResponse wr, string url, IBlobCache cache, DateTimeOffset? absoluteExpiration)
{
var hwr = (HttpWebResponse)wr;
Debug.Assert(hwr != null, "The Web Response is somehow null but shouldn't be.");
if ((int)hwr.StatusCode >= 400)
{
return Observable.Throw<byte[]>(new WebException(hwr.StatusDescription));
}
var ms = new MemoryStream();
using (var responseStream = hwr.GetResponseStream())
{
Debug.Assert(responseStream != null, "The response stream is somehow null");
responseStream.CopyTo(ms);
}
var ret = ms.ToArray();
cache.Insert(url, ret, absoluteExpiration);
return Observable.Return(ret);
}
public static IObservable <Unit> Insert(this IBlobCache This, string key, byte[] data, TimeSpan expiration)
{
return(This.Insert(key, data, This.Scheduler.Now + expiration));
}
/// <summary>
/// Insert an object into the cache, via the JSON serializer.
/// </summary>
/// <param name="key">The key to associate with the object.</param>
/// <param name="value">The object to serialize.</param>
/// <param name="absoluteExpiration">An optional expiration date.</param>
public static IObservable <Unit> InsertObject <T>(this IBlobCache This, string key, T value, DateTimeOffset?absoluteExpiration = null)
{
var bytes = SerializeObject(value);
return(This.Insert(GetTypePrefixedKey(key, typeof(T)), bytes, absoluteExpiration));
}
public static void Insert(this IBlobCache This, string key, byte[] data, TimeSpan expiration)
{
This.Insert(key, data, This.Scheduler.Now + expiration);
}
public IObservable <Unit> Insert(string key, byte[] data, DateTimeOffset?absoluteExpiration = null)
{
return(_inner.Insert(key, data, absoluteExpiration));
}
//
// Summary:
// Attempt to return data from the cache. If the item doesn't exist or
// returns an error, call a Func to return the latest version of an object and
// insert the result in the cache. For most Internet applications, this method
// is the best method to call to fetch static data (i.e. images) from the network.
//
// Parameters:
// key:
// The key to associate with the object.
//
// fetchFunc:
// A Func which will asynchronously return the latest value for the bytes should
// the cache not contain the key. Observable.Start is the most straightforward
// way (though not the most efficient!) to implement this Func.
//
// absoluteExpiration:
// An optional expiration date.
//
// Returns:
// A Future result representing the bytes from the cache.
public static IObservable <byte[]> GetOrFetch(this IBlobCache This, string key, Func <IObservable <byte[]> > fetchFunc, DateTimeOffset?absoluteExpiration = null)
{
return(This.Get(key)
.Catch <byte[], Exception>(_ => Observable.Defer(() => fetchFunc())
.SelectMany(value => This.Insert(key, value, absoluteExpiration).Select(dummy => value))));
}
