/// <summary>
/// Deserializes the cache index.
/// </summary>
/// <param name="serializeFileName">Name of the serialize file.</param>
/// <returns></returns>
public static CacheIndex DeserializeCacheIndex(string serializeFileName)
{
lock (staticLock)
{
CacheIndex cacheIndex = null;
ISerializer <CacheIndex> iSerializer = SerializerFactory.Create <CacheIndex>(SerializationFormat.XML);
try
{
cacheIndex = iSerializer.DeserializeObjectFromFile(serializeFileName);
if (cacheIndex != null)
{
cacheIndex.PreFetchIndexEnabled = true;
cacheIndex.CleanIndexEnabled = true;
}
}
catch (Exception cexc)
{
if (cexc.Message.Contains("Bad PKCS7 padding") || cexc.Message.Contains("Padding is invalid and cannot be removed"))
{
// attempt to deserialize file with no encryption.
cacheIndex = iSerializer.DeserializeObjectFromFile(serializeFileName, EncryptionMode.NoEncryption);
}
else
{
throw;
}
}
return(cacheIndex);
}
}
/// <summary>
/// Creates a new cache index using the specified base URI string and cache path.
/// </summary>
/// <param name="baseUriString">The base URI string.</param>
/// <param name="cachePath">The cache path.</param>
/// <returns></returns>
public static CacheIndex Create(string baseUriString, string cachePath)
{
if (string.IsNullOrEmpty(baseUriString))
{
throw new ArgumentNullException("baseUriString");
}
CacheIndex cacheIndex = new CacheIndex(baseUriString, cachePath);
// attempt to deserialize cache index.
cacheIndex = DeserializeCacheIndex(cacheIndex.SerializeFile);
if (cacheIndex == null)
{
cacheIndex = new CacheIndex(baseUriString, cachePath);
}
else
{
// To-Do: for some reason properties in this class aren't being serialized
// so for now, simply reapply base string, and use default values and SerializeFileName
cacheIndex.BaseUri = baseUriString;
cacheIndex.CachePath = cachePath;
}
cacheIndex.PreFetchIndexEnabled = true;
cacheIndex.CleanIndexEnabled = true;
return(cacheIndex);
}
/// <summary>
/// Removes cached file from file system and expires metadata.
/// </summary>
/// <param name="cacheIndexItem"></param>
public void RemoveCurrentCache(CacheIndexItem cacheIndexItem)
{
//For all expired items where PreFetch is false, delete the file and then remove the item from the index
//For all expired items where PreFetch is true, delete the file and set Downloaded to null
if (cacheIndexItem == null)
{
return;
}
// delete file if it exists.
string filePath = GetCachePath(cacheIndexItem);
if (Device.File.Exists(filePath))
{
Device.File.Delete(filePath);
}
lock (syncLock)
{
if (cacheIndexItem.PreFetch)
{
cacheIndexItem.Expire();
}
else
{
Remove(cacheIndexItem);
}
CacheIndex.SerializeCacheIndex(this);
}
}
/// <summary>
/// returns the full path name of the cached file for CacheIndexItem
/// </summary>
/// <returns></returns>
public override string GetResponseFileName()
{
MonoCross.NetworkResponse NetworkResponse;
string filename = CacheIndex.GetFileName(CacheIndexItem, NetworkResourceArguments, out NetworkResponse);
ReturnStatus = NetworkResponse;
return(filename);
}
/// <summary>
/// Derives a unique key for a cache index, i.e. cacheIndex.BaseUri
/// </summary>
/// <param name="cacheIndex"></param>
/// <returns></returns>
public static string GetKey(CacheIndex cacheIndex)
{
if (cacheIndex == null)
{
throw new ArgumentNullException("cacheIndex");
}
return(cacheIndex.BaseUri);
}
public NetworkResponse Fetch(CacheIndex cacheIndex, CacheIndexItem cacheIndexItem, int timeoutMilliseconds)
{
NetworkResourceArguments args = new NetworkResourceArguments()
{
Headers = Device.RequestInjectionHeaders,
TimeoutMilliseconds = timeoutMilliseconds,
};
return(Fetch(cacheIndex, cacheIndexItem, args));
}
/// <summary>
/// Removes the specified cache index.
/// </summary>
/// <param name="cacheIndex">Index of the cache.</param>
public static void Remove(CacheIndex cacheIndex)
{
if (cacheIndex == null)
{
throw new ArgumentNullException("cacheIndex", "Requested CacheIndex is null");
}
string key = CacheIndexMap.GetKey(cacheIndex);
CacheIndexMap.Remove(key);
}
/// <summary>
/// Serializes the cache index.
/// </summary>
/// <param name="cacheIndex">The cache index to serialize.</param>
public static void SerializeCacheIndex(CacheIndex cacheIndex)
{
// // execute serialize cache index off main thread.
// Device.Thread.QueueWorker(CacheIndex.SerializeCacheIndex, cacheIndex);
// Reduce frequency of actual serializations to match the serialization interval.
if (DateTime.Now.Subtract(cacheIndex.LastSerializationDate).TotalMilliseconds > SerializationInterval || cacheIndex.LastSerializationDate == DateTime.MinValue)
{
SerializeCacheIndexImmediate(cacheIndex);
}
}
/// <summary>
/// Adds the specified key.
/// </summary>
/// <param name="key">The key.</param>
public static void Add(string key)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException("key", "Requested CacheIndex key is null or empty");
}
lock (Map)
{
if (!Map.ContainsKey(key))
{
Map.Add(key, CacheIndex.Create(key));
}
}
}
/// <summary>
/// Adds the specified cache manifest.
/// </summary>
/// <param name="cacheManifest">The cache manifest.</param>
public static void Add(CacheManifest cacheManifest)
{
if (cacheManifest == null)
{
throw new ArgumentNullException("cacheManifest");
}
lock (Map)
{
// if the cacheIndex key doesn't already exist in the map then add it.
if (!Map.ContainsKey(cacheManifest.ManifestBaseUri))
{
CacheIndex cacheIndex = CacheIndex.Create(cacheManifest);
Map.Add(cacheManifest.ManifestBaseUri, cacheIndex);
}
}
}
/// <summary>
/// Gets the specified key.
/// </summary>
/// <param name="key">The key.</param>
/// <returns></returns>
public static CacheIndex Get(string key)
{
if (String.IsNullOrEmpty(key))
{
throw new ArgumentNullException("key", "Requested CacheIndex key is null or empty");
}
CacheIndex cacheIndex;
if (!Map.TryGetValue(key, out cacheIndex))
{
cacheIndex = CacheIndex.Create(key);
CacheIndexMap.Add(cacheIndex);
}
return(cacheIndex);
}
/// <summary>
/// Updates the specified cache index.
/// </summary>
/// <param name="cacheIndex">Index of the cache.</param>
public static void Update(CacheIndex cacheIndex)
{
if (cacheIndex == null)
{
throw new ArgumentNullException("cacheIndex");
}
string key = CacheIndexMap.GetKey(cacheIndex);
// if the key exists in the map then replace it with current parameter..
if (Map.ContainsKey(key))
{
CacheIndexMap.Remove(key);
}
CacheIndexMap.Add(cacheIndex);
}
/// <summary>
/// Adds the specified cache index.
/// </summary>
/// <param name="cacheIndex">The cache index to add to the map.</param>
public static void Add(CacheIndex cacheIndex)
{
if (cacheIndex == null)
{
throw new ArgumentNullException("cacheIndex");
}
string key = CacheIndexMap.GetKey(cacheIndex);
lock (Map)
{
// if the cacheIndex key doesn't already exist in the map then add it.
if (!Map.ContainsKey(key))
{
Map.Add(key, cacheIndex);
}
}
}
/// <summary>
/// Serializes the cache index.
/// </summary>
/// <param name="obj">The cache index to serialize.</param>
public static void SerializeCacheIndex(Object obj)
{
if (obj == null)
{
return;
}
CacheIndex cacheIndex = obj as CacheIndex;
if (cacheIndex.Count() <= 0)
{
return;
}
lock (staticLock)
{
ISerializer <CacheIndex> iSerializer = SerializerFactory.Create <CacheIndex>(SerializationFormat.XML);
iSerializer.SerializeObjectToFile(cacheIndex, cacheIndex.SerializeFile);
}
}
/// <summary>
/// Returns cache index that matches Uri
/// </summary>
/// <param name="uriString">Absolute Uri for resource represented by Uri</param>
/// <returns></returns>
public static CacheIndex GetFromUri(string uriString)
{
if (String.IsNullOrEmpty(uriString))
{
throw new ArgumentNullException("uriString");
}
string key = GetKeyFromUri(uriString);
// To-Do: should this really create a cache index item
CacheIndex cacheIndex;
if (!Map.TryGetValue(key, out cacheIndex))
{
cacheIndex = CacheIndex.Create(key);
CacheIndexMap.Add(cacheIndex);
}
return(cacheIndex);
}
/// <summary>
/// Serializes the cache index immediately.
/// </summary>
/// <param name="cacheIndex">The cache index to serialize.</param>
public static void SerializeCacheIndexImmediate(CacheIndex cacheIndex)
{
lock (staticLock)
{
try
{
cacheIndex.LastSerializationDate = DateTime.Now;
ISerializer <CacheIndex> iSerializer = SerializerFactory.Create <CacheIndex>(SerializationFormat.XML);
iSerializer.SerializeObjectToFile(cacheIndex, cacheIndex.SerializeFile);
}
catch (Exception e)
{
int count = -1;
if (cacheIndex != null)
{
count = cacheIndex.Count;
}
Device.Log.Error("Exception trying to serialize cache index immediately. Cache index count was: " + count, e);
}
}
}
// To-Do - make more efficient so 2 new statements aren't used
/// <summary>
/// Creates a new cache index using the specified cache manifest and cache path.
/// </summary>
/// <param name="cacheManifest">The cache manifest.</param>
/// <param name="cachePath">The cache path.</param>
/// <returns></returns>
public static CacheIndex Create(CacheManifest cacheManifest, string cachePath)
{
// derive base URI from cache manifest.
// base Uri = root folder of cache manifest file.
if (cacheManifest == null)
{
throw new ArgumentNullException("cacheManifest cannot be null.");
}
string baseUriString = cacheManifest.ManifestBaseUri;
CacheIndex cacheIndex = new CacheIndex(baseUriString, cachePath);
// attempt to deserialize cache index.
cacheIndex = DeserializeCacheIndex(cacheIndex.SerializeFile);
if (cacheIndex == null)
{
cacheIndex = new CacheIndex(baseUriString, cachePath);
}
else
{
// To-Do: for some reason properties in this class aren't being serialized
// so for now, simply reapply base string, and use default values and SerializeFileName
cacheIndex.BaseUri = baseUriString;
cacheIndex.CachePath = cachePath;
}
// since we're creating from CacheManifest, execute the update
cacheIndex.UpdateIndex(cacheManifest);
cacheIndex.PreFetchIndexEnabled = true;
cacheIndex.CleanIndexEnabled = true;
return(cacheIndex);
}
private static void ImportMetadata(string metadataFileName, CacheIndex cacheIndex)
{
DateTime dtMetric = DateTime.UtcNow;
ISerializer <CacheIndexItem> iSerializer = SerializerFactory.Create <CacheIndexItem>(SerializationFormat.XML);
List <CacheIndexItem> listCache = iSerializer.DeserializeListFromFile(metadataFileName);
for (int i = 0; i < listCache.Count; i++)
{
// add index item, don't save index until all Adds have been completed.
var cacheItem = listCache[i];
if (cacheItem != null)
{
cacheIndex.Add(cacheItem, false);
}
}
// Serialize Cache Index due to changes.
CacheIndex.SerializeCacheIndex((object)cacheIndex);
Device.File.Delete(metadataFileName);
Device.Log.Metric(string.Format("Import Metadata: file: {0} Time: {1} milliseconds", metadataFileName, DateTime.UtcNow.Subtract(dtMetric).TotalMilliseconds));
}
/// <summary>
/// Extracts a resoure into the NRL cache files and applies metadata.
/// </summary>
/// <param name="uri">The URI of the resource.</param>
/// <param name="metadataFileName">Name of the metadata file.</param>
/// <param name="headers">The headers for the request.</param>
/// <param name="timeout">The timeout for the requst in milliseconds.</param>
/// <param name="postObject">The object to post.</param>
/// <param name="serializationFormat">The serialization format.</param>
/// <returns></returns>
public NetworkResponse Extract(string uri, string metadataFileName, IDictionary <string, string> headers, int timeout,
object postObject, SerializationFormat serializationFormat)
{
// if cache is not supported then don't bother extracting.
if (Device.NetworkGetMethod == MonoCross.Utilities.Network.NetworkGetMethod.NoCache)
{
Device.Log.Debug("Cache is not supported, Zip file extraction request ignored.");
return(new NetworkResponse()
{
StatusCode = HttpStatusCode.OK,
Message = "Cache is not supported, Zip file extraction request ignored.",
URI = uri,
Verb = "GET"
});
}
// validate parameter
if (string.IsNullOrEmpty(uri))
{
throw new ArgumentNullException("uri");
}
if (!Uri.IsWellFormedUriString(uri, UriKind.Absolute))
{
throw new ArgumentNullException("Cache Zip location is not a valid Absolute URI " + uri, "uri");
}
DateTime dtMetric = DateTime.UtcNow;
CacheIndex cacheIndex = CacheIndexMap.GetFromUri(uri);
CacheIndexItem zipCacheIndexItem = cacheIndex.Get(uri);
zipCacheIndexItem.PreFetch = false;
NetworkResponse networkResponse = null;
bool idleQueueEnabled = Device.Thread.IdleQueueEnabled;
try
{
// temporarily deactivate idle queue thread.
if (idleQueueEnabled)
{
Device.Thread.IdleQueueEnabled = false;
}
// download zip file from uri
string zipFileName = cacheIndex.GetCachePath(zipCacheIndexItem);
networkResponse = DownloadZipFile(uri, zipFileName, headers, timeout, postObject); //, serializationFormat );
if (networkResponse.StatusCode != HttpStatusCode.OK)
{
return(networkResponse);
}
string cachePath = cacheIndex.CachePath.AppendPath(cacheIndex.BaseUriPath);
// extract zip file into NRL cache
networkResponse.Message = "Extract Zip File";
ExtractZipFile(zipFileName, cachePath);
// process metadata for extracted files.
networkResponse.Message = "Import Metadata";
metadataFileName = cacheIndex.CachePath.AppendPath(cacheIndex.BaseUriPath).AppendPath(metadataFileName);
ImportMetadata(metadataFileName, cacheIndex);
Device.Log.Metric(string.Format("Extract and process zip file: file: {0} Time: {1} milliseconds", zipFileName, DateTime.UtcNow.Subtract(dtMetric).TotalMilliseconds));
networkResponse.Message = "Complete";
}
//catch ( WebException ex )
//{
// if ( ex.Response != null )
// {
// networkResponse.StatusCode = ( (HttpWebResponse) ex.Response ).StatusCode;
// ex.Data["StatusDescription"] = ( (HttpWebResponse) ex.Response ).StatusDescription;
// }
// else
// {
// networkResponse.StatusCode = (HttpStatusCode) ( -2 );
// }
// networkResponse.WebExceptionStatusCode = ex.Status;
// Device.Log.Error( ex );
// networkResponse.Exception = ex;
//}
catch (Exception exc)
{
Device.Log.Error(exc);
networkResponse.Exception = exc;
networkResponse.StatusCode = (HttpStatusCode)(-1);
}
finally
{
// remove zip file after processing is complete. perhaps in a finally block
if (cacheIndex != null && zipCacheIndexItem != null)
{
cacheIndex.RemoveCurrentCache(zipCacheIndexItem);
// cacheIndex.Remove( zipCacheIndexItem );
}
// reactivate queue thread if it was previously active.
if (idleQueueEnabled)
{
Device.Thread.IdleQueueEnabled = true;
}
// Serialize Cache Index due to changes.
CacheIndex.SerializeCacheIndex(cacheIndex);
}
return(networkResponse);
}
/// <summary>
/// Ensures that the CacheIndexItem has a current resource in cache.
/// </summary>
public NetworkResponse EnsureCurrentCache(CacheIndexItem cacheIndexItem, NetworkResourceArguments args)
{
NetworkResponse networkResponse = new NetworkResponse();
if (cacheIndexItem == null)
{
networkResponse.Message = "Cache Index Item requested argument is null";
networkResponse.StatusCode = HttpStatusCode.BadRequest;
return(networkResponse);
}
string filePath = GetCachePath(cacheIndexItem);
bool itemInCache = Device.File.Exists(filePath);
// if item in cache and not expired and not stale, then existing cache is current and no need to download new file
if (itemInCache && !cacheIndexItem.IsExpired && !cacheIndexItem.IsStale)
{
networkResponse.Message = "Existing cache is current, no need to download.";
networkResponse.Expiration = cacheIndexItem.Expiration;
networkResponse.Downloaded = cacheIndexItem.Downloaded;
networkResponse.AttemptToRefresh = cacheIndexItem.AttemptToRefresh;
networkResponse.StatusCode = HttpStatusCode.OK;
return(networkResponse);
}
// if Stale, but not expired, then place separate request on queue to download later on separate process
if (!cacheIndexItem.IsExpired && cacheIndexItem.IsStale && args.CacheStaleMethod == CacheStaleMethod.Deferred)
{
Device.Thread.QueueIdle(this.EnsureCurrentCache, cacheIndexItem);
networkResponse.Message = "Existing cache is stale, placing refresh request onto idle queue";
networkResponse.Expiration = cacheIndexItem.Expiration;
networkResponse.Downloaded = cacheIndexItem.Downloaded;
networkResponse.AttemptToRefresh = cacheIndexItem.AttemptToRefresh;
networkResponse.StatusCode = HttpStatusCode.OK;
return(networkResponse);
}
// if we got here, then attempt to download file.
try
{
CacheFetcher cacheFetcher = new CacheFetcher();
networkResponse = cacheFetcher.Fetch(this, cacheIndexItem, args);
if (networkResponse.WebExceptionStatusCode != WebExceptionStatus.Success && //.NameResolutionFailure
args.CacheStaleMethod == CacheStaleMethod.Immediate &&
itemInCache &&
!cacheIndexItem.IsExpired &&
cacheIndexItem.IsStale)
{
// unable to refresh cache file from server but existing cache file is still good.
networkResponse.Exception = null;
networkResponse.Message = string.Empty;
networkResponse.StatusCode = HttpStatusCode.OK;
networkResponse.WebExceptionStatusCode = WebExceptionStatus.Success;
}
}
catch (NetworkResourceLibraryException nrlexc)
{
networkResponse.Downloaded = cacheIndexItem.Downloaded;
networkResponse.AttemptToRefresh = cacheIndexItem.AttemptToRefresh;
networkResponse.Expiration = cacheIndexItem.Expiration;
networkResponse.Message = nrlexc.Message;
networkResponse.Exception = nrlexc;
return(networkResponse);
}
finally
{
switch (networkResponse.StatusCode)
{
case HttpStatusCode.Unauthorized:
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.RequestTimeout:
case HttpStatusCode.BadGateway:
case HttpStatusCode.GatewayTimeout:
case (HttpStatusCode)(-1):
case (HttpStatusCode)(-2):
CacheIndexMap.PrefetchProcessEnabled = false;
break;
default:
CacheIndexMap.PrefetchProcessEnabled = true;
break;
}
CacheIndex.SerializeCacheIndex(this);
}
return(networkResponse);
}
/// <summary>
/// Fetches the specified cache index item from the cache index.
/// </summary>
/// <param name="cacheIndex">Index of the cache.</param>
/// <param name="cacheIndexItem">The cache index item.</param>
/// <param name="args">NetworkResourceArguments for the request</param>
/// <returns></returns>
public NetworkResponse Fetch(CacheIndex cacheIndex, CacheIndexItem cacheIndexItem, NetworkResourceArguments args) //IDictionary<string, string> headers, int timeoutMilliseconds)
{
PostNetworkResponse = new NetworkResponse();
var fetchParameters = new FetchParameters
{
CacheIndex = cacheIndex,
CacheIndexItem = cacheIndexItem,
Headers = args.Headers,
DefaultExpiration = args.Expiration,
};
int timeoutMilliseconds = args.TimeoutMilliseconds;
DateTime dtMetric = DateTime.UtcNow;
// set callback and error handler
OnDownloadComplete += CacheFetcher_OnDownloadComplete;
OnError += CacheFetcher_OnError;
Exception threadExc = null;
Device.Thread.QueueWorker(parameters =>
{
try
{
FetchAsynch(parameters, timeoutMilliseconds);
}
catch (Exception e)
{
// You could react or save the exception to an 'outside' variable
threadExc = e;
}
finally
{
_autoEvent.Set(); // if you're firing and not forgetting ;)
}
}, fetchParameters);
// WaitOne returns true if autoEvent were signaled (i.e. process completed before timeout expired)
// WaitOne returns false it the timeout expired before the process completed.
if (!_autoEvent.WaitOne(timeoutMilliseconds))
{
string message = "CacheFetcher call to FetchAsynch timed out. uri " + fetchParameters.CacheIndexItem.RelativeUri;
Device.Log.Metric(string.Format("CacheFetcher timed out: Uri: {0} Time: {1:F0} milliseconds ", fetchParameters.CacheIndexItem.RelativeUri, DateTime.UtcNow.Subtract(dtMetric).TotalMilliseconds));
var networkResponse = new NetworkResponse
{
Message = message,
URI = fetchParameters.CacheIndex.GetAbsouteUri(fetchParameters.CacheIndexItem),
StatusCode = HttpStatusCode.RequestTimeout,
WebExceptionStatusCode = WebExceptionStatus.RequestCanceled, // not using ConnectFailure because connection may have succeeded
ResponseString = string.Empty,
Expiration = DateTime.MinValue.ToUniversalTime(),
Downloaded = DateTime.MinValue.ToUniversalTime(),
AttemptToRefresh = DateTime.MinValue.ToUniversalTime(),
Exception = threadExc,
};
return(networkResponse);
}
else if (threadExc != null)
{
PostNetworkResponse.Exception = threadExc;
PostNetworkResponse.Message = "CacheFetcher.FetchAsync threw an exception";
PostNetworkResponse.StatusCode = (HttpStatusCode)(-1);
}
Device.Log.Metric(string.Format("CacheFetcher Completed: Uri: {0} Time: {1:F0} milliseconds ", PostNetworkResponse.URI, DateTime.UtcNow.Subtract(dtMetric).TotalMilliseconds));
return(PostNetworkResponse);
}
