public void Test102_Caching_AbsoluteExpiration_Test()
{
CachingMode mode = CachingMode.AbsoluteExpiration;
domain.PrepareCachingMode(mode);
domain.SetAbsoluteTimeout(3);
// get the first copy of the product
DataSet productDs1 = domain.GetProductByID(productId, mode);
decimal price1 = GetPrice(productDs1);
decimal newPrice1 = price1 + 0.01m;
domain.SetListPrice(newPrice1, productId);
// get the second copy of the product
DataSet productDs2 = domain.GetProductByID(productId, mode);
decimal price2 = GetPrice(productDs2);
Thread.Sleep(3000);
DataSet productDs3 = domain.GetProductByID(productId, mode);
decimal price3 = GetPrice(productDs3);
Assert.AreEqual(price1, price2,
"price1 and price2 should match due to caching");
Assert.AreNotEqual(newPrice1, price2,
"newPrice1 and price2 should not match due to caching");
Assert.AreEqual(newPrice1, price3,
"newPrice1 and price3 should match once the cache expires the item");
domain.CompleteCachingMode(mode);
}
public void SetPageMode(Vector3 addr, CachingMode mode)
{
if (mode == CachingMode.Cold)
{
AbstractPage page = m_map.GetPage(addr);
if (page == null)
{
throw new ArgumentException("No uncached page at this address.");
}
Cache(page);
}
else
{
try
{
AbstractPage coldpage = m_map.GetPage(addr);
AbstractPage page = Wake(addr);
page.cacheMode = mode;
if (coldpage != null)
{
m_map.RemovePage(coldpage);
}
m_map.AddPage(page, addr);
}
catch (System.IO.FileNotFoundException)
{
throw new ArgumentException("No cached page at this address.");
}
}
}
/// <summary>
/// Loads the HEAD of the requested URL, and returns the response URL value.
/// For a site that redirects some queries, this allows you to get the 'real' URL for a given short URL.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>
/// Returns the URL that the response headers say we requested.
/// </returns>
/// <exception cref="System.ArgumentNullException">url</exception>
/// <exception cref="System.ArgumentException">url</exception>
public async Task <string> GetRedirectUrl(string url, string shortDescrip,
CachingMode caching, ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token)
{
Uri responseUri = await GetRedirectedHeaderRequestUri(url, shortDescrip, suppressNotifications, token);
return(responseUri?.AbsoluteUri);
}
public void SetPageMode(AbstractPage page, CachingMode mode)
{
if (page.cacheMode == mode)
{
return;
}
SetPageMode(page.address, mode);
}
/// <summary>
/// Gets the cached content for the provided URL, if any, and if flagged to use caching.
/// </summary>
/// <param name="url">The URL to search for.</param>
/// <param name="caching">The caching mode.</param>
/// <returns>Returns a (bool,string) tuple of whether there was cached content found, and what it was if found.</returns>
private (bool found, string content) GetCachedContent(string url, CachingMode caching)
{
if (caching == CachingMode.UseCache)
{
return(Cache.Get(url));
}
return(false, string.Empty);
}
public void CompleteCachingMode(CachingMode mode)
{
if (mode == CachingMode.Polling)
{
SqlCacheDependencyAdmin.DisableNotifications(ConnectionString);
}
else if (mode == CachingMode.Notification)
{
SqlDependency.Stop(ConnectionString);
}
}
public void PrepareCachingMode(CachingMode mode)
{
if (mode == CachingMode.Polling)
{
SqlCacheDependencyAdmin.EnableNotifications(ConnectionString);
SqlCacheDependencyAdmin.EnableTableForNotifications(ConnectionString, "Production.Product");
}
else if (mode == CachingMode.Notification)
{
SqlDependency.Start(ConnectionString);
}
}
/// <summary>
/// Gets the XML page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>Returns an XML document, if it can be loaded.</returns>
public async Task <XDocument> GetXmlPage(string url, string shortDescrip, CachingMode caching, ShouldCache shouldCache,
SuppressNotifications suppressNotifications, CancellationToken token)
{
XDocument xmldoc = null;
string content = await GetPageContent(url, shortDescrip, caching, shouldCache, suppressNotifications, token).ConfigureAwait(false);
if (!string.IsNullOrEmpty(content))
{
xmldoc = XDocument.Parse(content);
}
return(xmldoc);
}
/// <summary>
/// Asynchronously load a specific web page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>
/// Returns an HTML document, if it can be loaded.
/// </returns>
public async Task <HtmlDocument> GetPage(string url, string shortDescrip, CachingMode caching, ShouldCache shouldCache,
SuppressNotifications suppressNotifications, CancellationToken token)
{
HtmlDocument htmldoc = null;
string content = await GetPageContent(url, shortDescrip, caching, shouldCache, suppressNotifications, token).ConfigureAwait(false);
if (!string.IsNullOrEmpty(content))
{
htmldoc = new HtmlDocument();
await Task.Run(() => htmldoc.LoadHtml(content), token).ConfigureAwait(false);
}
return(htmldoc);
}
/// <summary>
/// Gets the content of the requested page.
/// </summary>
/// <param name="url">The URL to load.</param>
/// <param name="shortDescrip">The short description of the page (for notifications).</param>
/// <param name="caching">The caching mode.</param>
/// <param name="shouldCache">Whether the requested page should be cached.</param>
/// <param name="suppressNotifications">Whether to suppress notifications.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>Returns the loaded resource string.</returns>
private async Task <string> GetPageContent(string url, string shortDescrip, CachingMode caching, ShouldCache shouldCache,
SuppressNotifications suppressNotifications, CancellationToken token)
{
var(uri, url2) = GetVerifiedUrl(url);
var(found, content) = GetCachedContent(url2, caching);
if (found)
{
NotifyStatusChange(PageRequestStatusType.LoadedFromCache, url2, shortDescrip, null, suppressNotifications);
}
else
{
content = await GetUrlContent(uri, url2, shortDescrip, shouldCache, suppressNotifications, token).ConfigureAwait(false) ?? string.Empty;
}
return(content);
}
public IDalSqlDataReader ExecuteReader(CommandBehavior iBehavior, CachingMode iCaching)
{
switch (iCaching)
{
case CachingMode.Disabled:
return(new NonCachedDalSqlDataReader(command.ExecuteReader(iBehavior)));
case CachingMode.Standard:
return(new CachedDalSqlDataReader(command.ExecuteReader(iBehavior)));
case CachingMode.Lazy:
return(new LazyCachedDalSqlDataReader(command.ExecuteReader(iBehavior)));
default:
//Stub
return(new NonCachedDalSqlDataReader(command.ExecuteReader(iBehavior)));
}
}
/// <summary>
/// Loads the HEAD of the requested URL, and returns the response URL value.
/// For a site that redirects some queries, this allows you to get the 'real' URL for a given short URL.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>
/// Returns the URL that the response headers say we requested.
/// </returns>
/// <exception cref="System.ArgumentNullException">url</exception>
/// <exception cref="System.ArgumentException">url</exception>
public async Task <string> GetRedirectUrlAsync(string url, string?shortDescrip,
CachingMode caching, ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token)
{
logger.LogInformation($"Requested URL redirect for \"{shortDescrip}\"");
Uri?responseUri = await GetRedirectedHeaderRequestUri(url, shortDescrip, suppressNotifications, token);
string result = responseUri?.AbsoluteUri ?? string.Empty;
if (string.IsNullOrEmpty(result))
{
logger.LogDebug("Redirect request failed for \"{shortDescrip}\".");
}
else
{
logger.LogDebug($"Redirect request succeeded. Using {result}");
}
return(result);
}
/// <summary>
/// Tries to get the cached version of the requested page.
/// </summary>
/// <param name="url">The URL.</param>
/// <param name="shortDescrip">The short descrip.</param>
/// <param name="caching">The caching.</param>
/// <param name="suppressNotifyMessages">if set to <c>true</c> [suppress notify messages].</param>
/// <returns>Returns whether it found the cached document, and the document, if found.</returns>
private async Task <Tuple <bool, HtmlDocument> > TryGetCachedPageAsync(string url, string shortDescrip,
CachingMode caching, SuppressNotifications suppressNotifications)
{
HtmlDocument htmldoc = null;
if (caching == CachingMode.UseCache)
{
htmldoc = await Cache.GetAsync(url).ConfigureAwait(false);
if (htmldoc != null)
{
NotifyStatusChange(PageRequestStatusType.LoadedFromCache, url, shortDescrip, null, suppressNotifications);
}
}
Tuple <bool, HtmlDocument> result = new Tuple <bool, HtmlDocument>(htmldoc != null, htmldoc);
return(result);
}
public void Test105_Caching_SqlDependency_Test()
{
CachingMode mode = CachingMode.SqlDependency;
domain.PrepareCachingMode(mode);
// get the first copy of the product
Console.WriteLine("Getting price1");
DataSet productDs1 = domain.GetProductByID(productId, mode);
Assert.IsNotNull(productDs1, "productDs1 cannot be null");
decimal price1 = GetPrice(productDs1);
decimal newPrice1 = price1 + 0.01m;
domain.SetListPrice(newPrice1, productId);
// sleep just long enough to allow the notification to work
Thread.Sleep(200);
// get the second copy of the product
Console.WriteLine("Getting price2");
DataSet productDs2 = domain.GetProductByID(productId, mode);
Assert.IsNotNull(productDs2, "productDs2 cannot be null");
decimal price2 = GetPrice(productDs2);
Console.WriteLine("Getting price3");
DataSet productDs3 = domain.GetProductByID(productId, mode);
Assert.IsNotNull(productDs3, "productDs3 cannot be null");
decimal price3 = GetPrice(productDs3);
Assert.AreNotEqual(price1, price2,
"price1 and price2 should not match due to the SqlDependency");
Assert.AreEqual(newPrice1, price2,
"newPrice1 and price2 should match due to the SqlDependency");
Assert.AreEqual(newPrice1, price3,
"newPrice1 and price3 should match");
domain.CompleteCachingMode(mode);
}
static void Main(string[] args)
{
// use simulator as the default action
string action = ChoseAction();
CachingMode mode = ChoseMode();
if ("simulator".Equals(action))
{
Simulator simulator = new Simulator(3, 10);
simulator.SetCachingMode(mode);
simulator.Run();
}
else if ("monitor".Equals(action))
{
Monitor monitor = new Monitor();
monitor.SetCachingMode(mode);
monitor.Run();
}
else
{
Console.WriteLine("Action not recognized: " + action);
}
}
public void Test101_Caching_Off_Test()
{
CachingMode mode = CachingMode.Off;
domain.PrepareCachingMode(mode);
// get the first copy of the product
DataSet productDs1 = domain.GetProductByID(productId, mode);
decimal oldPrice = GetPrice(productDs1);
decimal newPrice = oldPrice + 0.01m;
domain.SetListPrice(newPrice, productId);
// get the second copy of the product
DataSet productDs2 = domain.GetProductByID(productId, mode);
decimal updatedPrice = GetPrice(productDs2);
Assert.AreNotEqual(oldPrice, updatedPrice);
Assert.AreEqual(newPrice, updatedPrice);
domain.CompleteCachingMode(mode);
}
public float[][] Fit(float[][] X)
{
int exaggerationLength = (int)(MaxEpochs * ExaggerationRatio);
gpu = new GpuDevice();
cc = gpu.CreateConstantBuffer <TsneMapConstants>(0);
int N = X.Length;
cc.c.columns = X[0].Length;
cc.c.N = N;
cc.c.outDim = OutDim;
cc.c.metricType = MetricType;
#region Initialize Y
Buffer Y2Buf = null;
Buffer Y3Buf = null;
Buffer Y3StagingBuf = null;
Buffer Y2StagingBuf = null;
Buffer v2Buf = null;
Buffer v3Buf = null;
if (cc.c.outDim <= 2)
{
Y2Buf = gpu.CreateBufferRW(N, 8, 3);
Y2StagingBuf = gpu.CreateStagingBuffer(Y2Buf);
v2Buf = gpu.CreateBufferRW(N, 2 * 8, 5);
}
else
{
Y3Buf = gpu.CreateBufferRW(N, 12, 4);
Y3StagingBuf = gpu.CreateStagingBuffer(Y3Buf);
v3Buf = gpu.CreateBufferRW(N, 2 * 12, 6);
}
float rang = 0.05f;
Random rGenerator = new Random(435243);
if (cc.c.outDim <= 2)
{
using (var ws = gpu.NewWriteStream(v2Buf)) {
for (int row = 0; row < N; row++)
{
ws.Write <float>(0, 1, 0, 1);
}
}
using (var ws = gpu.NewWriteStream(Y2Buf)) {
for (int row = 0; row < N; row++)
{
for (int col = 0; col < cc.c.outDim; col++)
{
ws.Write((float)(rang * rGenerator.NextDouble() - rang / 2));
}
if (cc.c.outDim == 1)
{
ws.Write(0.0f);
}
}
}
}
else
{
using (var ws = gpu.NewWriteStream(v3Buf)) {
for (int row = 0; row < N; row++)
{
ws.Write <float>(0, 1, 0, 1, 0, 1);
}
}
using (var ws = gpu.NewWriteStream(Y3Buf)) {
for (int row = 0; row < N; row++)
{
for (int col = 0; col < cc.c.outDim; col++)
{
ws.Write((float)(rang * rGenerator.NextDouble() - rang / 2));
}
}
}
}
#endregion
#region Upload data table and initialize the distance matrix
// Used to aggregate values created by parallel threads.
// the size of of groupMaxBuf must be large enoght to hold a float value for each thread started in parallel.
// Notice: gpu.Run(k) will start k*GROUP_SIZE threads.
int gpSize = Math.Max(GpuGroupSize, MaxGroupNumber * GroupSize);
gpSize = Math.Max(gpSize, MaxGroupNumberHyp * GroupSizeHyp);
groupMaxBuf = gpu.CreateBufferRW(gpSize, 4, 7);
resultBuf = gpu.CreateBufferRW(3, 4, 2); // to receive the total changes.
resultStaging = gpu.CreateStagingBuffer(resultBuf);
tableBuf = gpu.CreateBufferRO(N * cc.c.columns, 4, 0);
if (MetricType == 1)
{
NormalizeTable(X);
}
gpu.WriteMarix(tableBuf, X, true);
const int MinCpuDimension = 100; // minimal dimension to trigger CPU caching.
const int MaxDimension = 64; // maximal dimension (table columns) for fast EuclideanNoCache shader. Must be the same as MAX_DIMENSION.
const int MaxDimensionS = 32; // maximal dimension (table columns) for fast EuclideanNoCache shader. Must be the same as MAX_DIMENSIONs.
if (N <= CacheLimit)
{
cachingMode = CachingMode.OnGpu;
}
else
{
if ((cc.c.columns > MinCpuDimension) && ((double)N * N * 4) < ((double)MaxCpuCacheSize * 1024.0 * 1024.0))
{
cachingMode = CachingMode.OnCpu;
}
else
{
if (cc.c.columns < MaxDimensionS)
{
cachingMode = CachingMode.OnFlySmS;
}
else if (cc.c.columns < MaxDimension)
{
cachingMode = CachingMode.OnFlySm;
}
else
{
cachingMode = CachingMode.OnFly;
}
}
}
#endregion
cc.c.targetH = (float)Math.Log(PerplexityRatio * N);
if (cachingMode == CachingMode.OnGpu)
{
CalculateP();
}
else if (cachingMode == CachingMode.OnCpu)
{
InitializePCpu();
}
else // (cachingMode == CachingMode.OnFly[Sm,SmS])
{
InitializeP();
}
using (var sd = gpu.LoadShader("TsneDx.CalculateSumQ.cso")) {
gpu.SetShader(sd);
cc.c.groupNumber = 256;
for (int i = 0; i < N; i += cc.c.groupNumber)
{
cc.c.blockIdx = i;
cc.Upload();
gpu.Run(cc.c.groupNumber);
}
cc.c.blockIdx = -1;
cc.Upload();
gpu.Run();
}
var sdNames = new Dictionary <CachingMode, string>()
{
{ CachingMode.OnGpu, "TsneDx.OneStep.cso" },
{ CachingMode.OnCpu, "TsneDx.OneStepCpuCache.cso" },
{ CachingMode.OnFly, "TsneDx.OneStepNoCache.cso" },
{ CachingMode.OnFlySm, "TsneDx.FastStep.cso" },
{ CachingMode.OnFlySmS, "TsneDx.FastStepS.cso" },
};
ComputeShader csOneStep = gpu.LoadShader(sdNames[cachingMode]);
ComputeShader csSumUp = gpu.LoadShader("TsneDx.OneStepSumUp.cso");
int stepCounter = 0;
while (true)
{
if (stepCounter < exaggerationLength)
{
if (ExaggerationSmoothen)
{
int len = (int)(0.9 * MaxEpochs);
if (stepCounter < len)
{
double t = (double)stepCounter / len;
t = Math.Sqrt(Math.Sqrt(t));
cc.c.PFactor = (float)((1 - t) * ExaggerationFactor + t);
}
else
{
cc.c.PFactor = 1.0f;
}
}
else
{
cc.c.PFactor = (float)ExaggerationFactor;
}
}
else
{
cc.c.PFactor = 1.0f;
}
gpu.SetShader(csOneStep);
if (cachingMode == CachingMode.OnGpu)
{
cc.c.groupNumber = MaxGroupNumber;
// Notice: cc.c.groupNumber*GroupSize must fit into groupMax[].
for (int bIdx = 0; bIdx < N; bIdx += cc.c.groupNumber * GroupSize)
{
cc.c.blockIdx = bIdx;
cc.Upload();
gpu.Run(cc.c.groupNumber);
}
cc.c.groupNumber = MaxGroupNumber * GroupSize;
}
else if (cachingMode == CachingMode.OnCpu)
{
int bSize = MaxGroupNumberHyp * GroupSizeHyp;
cc.c.groupNumber = MaxGroupNumberHyp;
for (int bIdx = 0; bIdx < N; bIdx += bSize)
{
gpu.WriteArray(cpuP, bIdx, Math.Min(N, bIdx + bSize), P2Buf);
cc.c.blockIdx = bIdx;
cc.Upload();
gpu.Run(cc.c.groupNumber);
}
cc.c.groupNumber = Math.Min(N, bSize);
}
else if ((cachingMode == CachingMode.OnFlySm) || (cachingMode == CachingMode.OnFlySmS))
{
const int GrSize = 64; // This value must match that of GR_SIZE in TsneMap.hlsl.
cc.c.groupNumber = MaxGroupNumber;
for (int bIdx = 0; bIdx < N; bIdx += cc.c.groupNumber * GrSize)
{
cc.c.blockIdx = bIdx;
cc.Upload();
gpu.Run(cc.c.groupNumber);
}
cc.c.groupNumber = cc.c.groupNumber * GrSize;
}
else // cachingMode==CachingMode.OnFly
{
cc.c.groupNumber = 128;
for (int bIdx = 0; bIdx < N; bIdx += cc.c.groupNumber)
{
cc.c.blockIdx = bIdx;
cc.Upload();
gpu.Run(cc.c.groupNumber);
}
}
//Notice: cc.c.groupNumber must be number of partial sumQ_next, which add up to sumQ for the next step.
gpu.SetShader(csSumUp);
cc.Upload();
gpu.Run();
currentVariation = gpu.ReadRange <float>(resultStaging, resultBuf, 3)[2] / N;
cc.c.mom = (float)((stepCounter < (MaxEpochs * momentumSwitch)) ? momentum : finalMomentum);
stepCounter++;
if (stepCounter % 10 == 0)
{
Console.Write('.');
}
if (stepCounter % 500 == 0)
{
Console.WriteLine();
}
if ((stepCounter >= MaxEpochs) || ((stepCounter >= (2 + exaggerationLength)) && (currentVariation < stopVariation)))
{
break;
}
}
Console.WriteLine();
float[][] Y = new float[N][];
using (var rs = gpu.NewReadStream((cc.c.outDim == 3) ? Y3StagingBuf : Y2StagingBuf, (cc.c.outDim == 3) ? Y3Buf : Y2Buf)) {
int outVDim = (cc.c.outDim == 3) ? 3 : 2;
for (int row = 0; row < N; row++)
{
Y[row] = rs.ReadRange <float>(outVDim);
}
}
if (cc.c.outDim == 1)
{
for (int i = 0; i < N; i++)
{
Y[i] = new float[] { Y[i][0] }
}
}
;
TsneDx.SafeDispose(csSumUp, csOneStep, PBuf, P2Buf, distanceBuf, tableBuf, resultBuf,
resultStaging, groupMaxBuf, Y3Buf, Y3StagingBuf, v3Buf, Y2Buf, Y2StagingBuf, v2Buf, cc, gpu);
return(AutoNormalize ? PcaNormalize.DoNormalize(Y) : Y);
}
public DataSet GetProductByID(int productId, CachingMode mode)
{
Cache cache = HttpRuntime.Cache;
string cacheKey = "Product-" + productId;
if (mode != CachingMode.Off)
{
if (cache[cacheKey] != null)
{
return(cache[cacheKey] as DataSet);
}
}
DataSet ds = null;
using (SqlConnection conn = new SqlConnection(ConnectionString))
{
// Construct the command to get any new ProductReview rows from the database along with
// the corresponding product name from the Product table.
SqlCommand cmd = new SqlCommand("SELECT ProductID, [Name], ListPrice " +
"FROM dbo.SimpleProduct WHERE ProductID = @ProductID", conn);
cmd.Parameters.Add("@ProductID", SqlDbType.Int);
cmd.Parameters[0].Value = productId;
//Note: We do not need to close the reader as the "using" statement
// will dispose of the connection.
CacheDependency cacheDependency = null;
if (mode == CachingMode.Polling)
{
// use Polling
cacheDependency = new SqlCacheDependency("aw", "dbo.SimpleProduct");
}
else if (mode == CachingMode.Notification)
{
// use Notifications
Console.WriteLine("Adding SqlCacheDependency");
//cacheDependency = new SqlCacheDependency(cmd);
//Console.WriteLine("Adding SqlDependency");
// Create a dependency on this query
//dependency = new SqlDependency(cmd);
// Register the event handler
//dependency.OnChange += new OnChangeEventHandler(Dependency_OnChange);
}
conn.Open();
SqlDataAdapter adapter = new SqlDataAdapter(cmd);
// Get any new rows
ds = new DataSet();
adapter.Fill(ds);
if (cacheDependency == null)
{
Console.WriteLine("-- using absolute timeout");
cache.Insert(cacheKey, ds, null,
DateTime.Now.AddSeconds(120), Cache.NoSlidingExpiration,
CacheItemPriority.Normal, RemovedCallback);
}
else
{
Console.WriteLine("-- using cacheDependency");
// just rely on the cacheDependency
cache.Insert(cacheKey, ds, cacheDependency,
Cache.NoAbsoluteExpiration, Cache.NoSlidingExpiration,
CacheItemPriority.Normal, RemovedCallback);
}
}
return(ds);
}
public IFluentSqlCommandFactory SetDefaultCachingMode(CachingMode iCachingMode)
{
DefaultCachingMode = iCachingMode;
return(this);
}
/// <summary>
/// Gets the cached content for the provided URL, if any, and if flagged to use caching.
/// </summary>
/// <param name="url">The URL to search for.</param>
/// <param name="caching">The caching mode.</param>
/// <returns>Returns a (bool,string) tuple of whether there was cached content found, and what it was if found.</returns>
private async Task <(bool found, string content)> GetCachedContent(string url, CachingMode caching)
{
if (caching == CachingMode.UseCache)
{
return(await Cache.GetAsync(url).ConfigureAwait(false));
}
return(false, string.Empty);
}
public void SetPageMode(Vector3 addr, CachingMode mode)
{
if (mode == CachingMode.Cold)
{
AbstractPage page = m_map.GetPage(addr);
if (page == null) throw new ArgumentException("No uncached page at this address.");
Cache(page);
}
else
{
try
{
AbstractPage coldpage = m_map.GetPage(addr);
AbstractPage page = Wake(addr);
page.cacheMode = mode;
if (coldpage != null)
{
m_map.RemovePage(coldpage);
}
m_map.AddPage(page, addr);
}
catch (System.IO.FileNotFoundException)
{
throw new ArgumentException("No cached page at this address.");
}
}
}
public void SetPageMode(AbstractPage page, CachingMode mode)
{
if (page.cacheMode == mode) return;
SetPageMode(page.address, mode);
}
public DataSet GetProductByID(int productId, CachingMode mode)
{
if (mode == CachingMode.SqlDependency)
{
lock (this)
{
if (products.ContainsKey(productId))
{
WriteMessage("Using cached copy");
OnDomainItemChanged(CacheActivity.AlreadyExists);
return(products[productId]);
}
}
}
Cache cache = HttpRuntime.Cache;
string cacheKey = "Product-" + productId;
if (mode != CachingMode.Off)
{
lock (this)
{
if (cache[cacheKey] != null)
{
WriteMessage("Using cached copy");
OnDomainItemChanged(CacheActivity.AlreadyExists);
return(cache[cacheKey] as DataSet);
}
}
}
DataSet ds = new DataSet();
try
{
using (DbCommand dbCmd = db.GetStoredProcCommand("dbo.chpt05_GetProductByID"))
{
db.AddInParameter(dbCmd, "@ProductID", DbType.Int16, productId);
CacheDependency cacheDependency = null;
if (mode == CachingMode.Polling)
{
// use Polling
WriteMessage("Using SqlCacheDependency (Polling)");
cacheDependency = new SqlCacheDependency("aw", "Production.Product");
}
else if (mode == CachingMode.Notification)
{
// use Notifications
WriteMessage("Using SqlCacheDependency (Notification)");
SqlCommand sqlCmd = dbCmd as SqlCommand;
if (sqlCmd != null)
{
cacheDependency = new SqlCacheDependency(sqlCmd);
}
}
else if (mode == CachingMode.SqlDependency)
{
SqlCommand sqlCmd = dbCmd as SqlCommand;
if (sqlCmd != null)
{
WriteMessage("Using SqlDependency");
SqlDependency sqlDependency = new SqlDependency(sqlCmd);
OnChangeEventHandler onChangeHandler =
delegate(object sender, SqlNotificationEventArgs e)
{
WriteSqlDependencyChange(sqlDependency, e, productId);
OnDomainItemChanged(CacheActivity.Removed);
products.Remove(productId);
};
sqlDependency.OnChange += onChangeHandler;
}
}
// get the data
ds = db.ExecuteDataSet(dbCmd);
CacheItemRemovedCallback removedCallback = delegate(
string key, object value, CacheItemRemovedReason reason)
{
switch (reason)
{
case CacheItemRemovedReason.Underused:
OnDomainItemChanged(CacheActivity.Underused);
break;
case CacheItemRemovedReason.Removed:
OnDomainItemChanged(CacheActivity.Removed);
break;
case CacheItemRemovedReason.Expired:
OnDomainItemChanged(CacheActivity.Expired);
break;
case CacheItemRemovedReason.DependencyChanged:
OnDomainItemChanged(CacheActivity.DependencyChanged);
break;
}
WriteCacheItemRemoved(key, value, reason);
};
if (mode == CachingMode.SqlDependency)
{
products[productId] = ds;
}
else if (mode == CachingMode.Polling || mode == CachingMode.Notification)
{
cache.Insert(cacheKey, ds, cacheDependency,
DateTime.Now.AddSeconds(absoluteTimeout), Cache.NoSlidingExpiration,
CacheItemPriority.Normal, removedCallback);
}
else if (mode == CachingMode.AbsoluteExpiration)
{
// shorten the absolute timeout from the default in this mode
if (absoluteTimeout == 120)
{
absoluteTimeout = 10;
}
WriteMessage("Using AbsoluteExpiration (" + absoluteTimeout + " seconds)");
cache.Insert(cacheKey, ds, null,
DateTime.Now.AddSeconds(absoluteTimeout), Cache.NoSlidingExpiration,
CacheItemPriority.Normal, removedCallback);
}
}
}
catch (Exception ex)
{
// TODO Log Error
throw;
}
//return the results
return(ds);
}
/// <summary>
/// Specifies the caching mode for the context instance.
/// </summary>
/// <param name="cachingMode">The caching mode to use.</param>
/// <returns>The current builder instance.</returns>
public Builder Caching(CachingMode cachingMode)
{
CachingMode = cachingMode;
return(this);
}
public void SetCachingMode(CachingMode mode)
{
cachingMode = mode;
}
/// <summary>
/// Asynchronously load a specific page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="token">Cancellation token.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <returns>Returns an HTML document, if it can be loaded.</returns>
/// <exception cref="ArgumentNullException">If url is null or empty.</exception>
/// <exception cref="ArgumentException">If url is not a valid absolute url.</exception>
private async Task <string> GetUrlContent(Uri uri, string url, string shortDescrip,
CachingMode caching, ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token)
{
string result = null;
int tries = 0;
DateTime expires = CacheInfo.DefaultExpiration;
NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications);
// Limit to no more than N parallel requests
await ss.WaitAsync(token).ConfigureAwait(false);
try
{
Cookie cookie = ForumCookies.GetCookie(uri);
if (cookie != null)
{
ClientHandler.CookieContainer.Add(uri, cookie);
}
HttpResponseMessage response;
Task <HttpResponseMessage> getResponseTask = null;
do
{
token.ThrowIfCancellationRequested();
if (tries > 0)
{
// Delay any additional attempts after the first.
await Task.Delay(retryDelay, token).ConfigureAwait(false);
// Notify the user if we're re-trying to load the page.
NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications);
}
tries++;
try
{
getResponseTask = client.GetAsync(uri, token).TimeoutAfter(timeout, token);
Debug.WriteLine($"Get URI {uri} task ID: {getResponseTask.Id}");
using (response = await getResponseTask.ConfigureAwait(false))
{
if (response.IsSuccessStatusCode)
{
result = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
// Get expires value
// Cannot get Expires value until we move to .NET Standard 2.0.
// If we get a successful result, we're done.
break;
}
else if (PageLoadFailed(response))
{
NotifyStatusChange(PageRequestStatusType.Failed, url,
GetFailureMessage(response, shortDescrip, url), null, suppressNotifications);
return(null);
}
else if (PageWasMoved(response))
{
url = response.Content.Headers.ContentLocation.AbsoluteUri;
uri = new Uri(url);
}
}
}
catch (OperationCanceledException)
{
if (token.IsCancellationRequested)
{
// user request
throw;
}
else
{
// timeout via cancellation
Debug.WriteLine($"Attempt to load {shortDescrip} timed out/self-cancelled (TA). Tries={tries}");
}
}
catch (TimeoutException)
{
Debug.WriteLine($"Attempt to load {shortDescrip} timed out. Tries={tries}");
}
catch (HttpRequestException e)
{
NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications);
throw;
}
} while (tries < retryLimit);
Debug.WriteLine($"Finished getting URI {uri} task ID: {getResponseTask.Id}");
if (result == null && tries >= retryLimit)
{
client.CancelPendingRequests();
}
}
catch (OperationCanceledException)
{
// If it's not a user-requested cancellation, generate a failure message.
if (!token.IsCancellationRequested)
{
NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications);
}
throw;
}
finally
{
ss.Release();
}
token.ThrowIfCancellationRequested();
if (result == null)
{
NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications);
return(null);
}
if (shouldCache == ShouldCache.Yes)
{
await Cache.AddAsync(url, result, expires);
}
NotifyStatusChange(PageRequestStatusType.Loaded, url, shortDescrip, null, suppressNotifications);
return(result);
}
public IReaderFluentSqlCommand <T> SetCachingMode(CachingMode iCachingMode)
{
Caching = iCachingMode;
return(this);
}
/// <summary>
/// Gets the XML page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>Returns an XML document, if it can be loaded.</returns>
public async Task <XDocument?> GetXmlDocumentAsync(string url, string shortDescrip, CachingMode caching, ShouldCache shouldCache,
SuppressNotifications suppressNotifications, CancellationToken token)
{
logger.LogInformation($"Requested XML document \"{shortDescrip}\"");
XDocument?xmldoc = null;
string content = await GetPageContent(url, shortDescrip, caching, shouldCache, suppressNotifications, token).ConfigureAwait(false);
if (!string.IsNullOrEmpty(content))
{
logger.LogInformation($"\"{shortDescrip}\" successfully loaded.");
xmldoc = XDocument.Parse(content);
logger.LogDebug($"\"{shortDescrip}\" successfully parsed into XDocument.");
}
return(xmldoc);
}
/// <summary>
/// Loads the HEAD of the requested URL, and returns the response URL value.
/// For a site that redirects some queries, this allows you to get the 'real' URL for a given short URL.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>
/// Returns the URL that the response headers say we requested.
/// </returns>
/// <exception cref="System.ArgumentNullException">url</exception>
/// <exception cref="System.ArgumentException">url</exception>
public async Task <string> GetHeaderUrl(string url, string shortDescrip,
CachingMode caching, ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException(nameof(url));
}
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
throw new ArgumentException($"Url is not valid: {url}", nameof(url));
}
Uri uri = new Uri(url);
NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications);
// Limit to no more than N parallel requests
await ss.WaitAsync(token).ConfigureAwait(false);
try
{
Cookie cookie = ForumCookies.GetCookie(uri);
if (cookie != null)
{
ClientHandler.CookieContainer.Add(uri, cookie);
}
int tries = 0;
HttpResponseMessage response;
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Head, uri);
while (tries < retryLimit && token.IsCancellationRequested == false)
{
if (tries > 0)
{
// If we have to retry loading the page, give it a short delay.
await Task.Delay(TimeSpan.FromSeconds(4)).ConfigureAwait(false);
NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications);
}
tries++;
try
{
// As long as we got a response (whether 200 or 404), we can extract what
// the server thinks the URL should be.
using (response = await client.SendAsync(request, token).ConfigureAwait(false))
{
return(response.RequestMessage.RequestUri.AbsoluteUri);
}
}
catch (HttpRequestException e)
{
NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications);
throw;
}
}
}
finally
{
ss.Release();
}
NotifyStatusChange(PageRequestStatusType.Loaded, url, shortDescrip, null, suppressNotifications);
return(null);
}
/// <summary>
/// Asynchronously load a specific web page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <param name="suppressNotifications">Indicates whether notification messages should be sent to output.</param>
/// <param name="token">Cancellation token.</param>
/// <returns>
/// Returns an HTML document, if it can be loaded.
/// </returns>
public async Task <HtmlDocument?> GetHtmlDocumentAsync(string url, string shortDescrip, CachingMode caching, ShouldCache shouldCache,
SuppressNotifications suppressNotifications, CancellationToken token)
{
logger.LogInformation($"Requested HTML document \"{shortDescrip}\"");
HtmlDocument?htmldoc = null;
string content = await GetPageContent(url, shortDescrip, caching, shouldCache, suppressNotifications, token).ConfigureAwait(false);
if (!string.IsNullOrEmpty(content))
{
logger.LogInformation($"\"{shortDescrip}\" successfully loaded from web.");
htmldoc = new HtmlDocument();
await Task.Run(() => htmldoc.LoadHtml(content), token).ConfigureAwait(false);
logger.LogDebug($"\"{shortDescrip}\" successfully parsed into HtmlDocument.");
}
return(htmldoc);
}
/// <summary>
/// Asynchronously load a specific page.
/// </summary>
/// <param name="url">The URL of the page to load. Cannot be null.</param>
/// <param name="shortDescrip">A short description that can be used in status updates. If null, no update will be given.</param>
/// <param name="caching">Indicator of whether to query the cache for the requested page.</param>
/// <param name="token">Cancellation token.</param>
/// <param name="shouldCache">Indicates whether the result of this page load should be cached.</param>
/// <returns>Returns an HTML document, if it can be loaded.</returns>
/// <exception cref="ArgumentNullException">If url is null or empty.</exception>
/// <exception cref="ArgumentException">If url is not a valid absolute url.</exception>
public async Task <HtmlDocument> GetPage(string url, string shortDescrip,
CachingMode caching, ShouldCache shouldCache, SuppressNotifications suppressNotifications, CancellationToken token)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException(nameof(url));
}
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
throw new ArgumentException($"Url is not valid: {url}", nameof(url));
}
Uri uri = new Uri(url);
url = Uri.UnescapeDataString(url);
HtmlDocument htmldoc = null;
string result = null;
int tries = 0;
// Try to load from cache first, if allowed.
if (caching == CachingMode.UseCache)
{
htmldoc = await Cache.GetAsync(url).ConfigureAwait(false);
if (htmldoc != null)
{
NotifyStatusChange(PageRequestStatusType.LoadedFromCache, url, shortDescrip, null, suppressNotifications);
return(htmldoc);
}
}
NotifyStatusChange(PageRequestStatusType.Requested, url, shortDescrip, null, suppressNotifications);
// Limit to no more than N parallel requests
await ss.WaitAsync(token).ConfigureAwait(false);
try
{
Cookie cookie = ForumCookies.GetCookie(uri);
if (cookie != null)
{
ClientHandler.CookieContainer.Add(uri, cookie);
}
HttpResponseMessage response;
Task <HttpResponseMessage> getResponseTask = null;
do
{
token.ThrowIfCancellationRequested();
if (tries > 0)
{
// Delay any additional attempts after the first.
await Task.Delay(retryDelay, token).ConfigureAwait(false);
// Notify the user if we're re-trying to load the page.
NotifyStatusChange(PageRequestStatusType.Retry, url, shortDescrip, null, suppressNotifications);
}
tries++;
try
{
getResponseTask = client.GetAsync(uri, token).TimeoutAfter(timeout, token);
Debug.WriteLine($"Get URI {uri} task ID: {getResponseTask.Id}");
using (response = await getResponseTask.ConfigureAwait(false))
{
if (response.IsSuccessStatusCode)
{
result = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
// If we get a successful result, we're done.
break;
}
else if (PageLoadFailed(response))
{
NotifyStatusChange(PageRequestStatusType.Failed, url,
GetFailureMessage(response, shortDescrip, url), null, suppressNotifications);
return(null);
}
else if (PageWasMoved(response))
{
url = response.Content.Headers.ContentLocation.AbsoluteUri;
uri = new Uri(url);
}
}
}
catch (OperationCanceledException)
{
if (token.IsCancellationRequested)
{
// user request
throw;
}
else
{
// timeout via cancellation
Debug.WriteLine($"Attempt to load {shortDescrip} timed out/self-cancelled (TA). Tries={tries}");
}
}
catch (TimeoutException)
{
Debug.WriteLine($"Attempt to load {shortDescrip} timed out. Tries={tries}");
}
catch (HttpRequestException e)
{
NotifyStatusChange(PageRequestStatusType.Error, url, shortDescrip, e, suppressNotifications);
throw;
}
} while (tries < retryLimit);
Debug.WriteLine($"Finished getting URI {uri} task ID: {getResponseTask.Id}");
if (result == null && tries >= retryLimit)
{
client.CancelPendingRequests();
}
}
catch (OperationCanceledException)
{
// If it's not a user-requested cancellation, generate a failure message.
if (!token.IsCancellationRequested)
{
NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications);
}
throw;
}
finally
{
ss.Release();
}
token.ThrowIfCancellationRequested();
if (result == null)
{
NotifyStatusChange(PageRequestStatusType.Failed, url, shortDescrip, null, suppressNotifications);
return(null);
}
if (shouldCache == ShouldCache.Yes)
{
Cache.Add(url, result);
}
htmldoc = new HtmlDocument();
await Task.Run(() => htmldoc.LoadHtml(result)).ConfigureAwait(false);
NotifyStatusChange(PageRequestStatusType.Loaded, url, shortDescrip, null, suppressNotifications);
return(htmldoc);
}
