// Helper function to allow Storage Client 1.7 (Microsoft.WindowsAzure.StorageClient) to utilize this class.
// Remove this function if only using Storage Client 2.0 (Microsoft.WindowsAzure.Storage).
public void DownloadBlobAsync(Microsoft.WindowsAzure.StorageClient.CloudBlob blob, string LocalFile)
{
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey account = blob.ServiceClient.Credentials as Microsoft.WindowsAzure.StorageCredentialsAccountAndKey;
ICloudBlob blob2 = new CloudBlockBlob(blob.Attributes.Uri, new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(blob.ServiceClient.Credentials.AccountName, account.Credentials.ExportBase64EncodedKey()));
DownloadBlobAsync(blob2, LocalFile);
}
public void optimizeIndex(string azureContainerName)
{
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(new Utility().getConfigXML());
XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']"));
string azureAccount = xNode.Attributes["account"].Value;
string azureEndpoint = xNode.Attributes["endpoint"].Value;
string azureSharedKey = xNode.Attributes["accountSharedKey"].Value;
string blobStorage = xNode.Attributes["endpoint"].Value;
xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']"));
string fragmentLocation = xNode.Attributes["value"].Value;
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
bool findexExists = false;
try
{
findexExists = IndexReader.IndexExists(azureDirectory);
if ((findexExists) && IndexWriter.IsLocked(azureDirectory))
azureDirectory.ClearLock("write.lock");
}
catch (Exception e)
{
Trace.WriteLine(e.ToString());
return;
}
IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024));
idxW.SetRAMBufferSizeMB(10.0);
idxW.SetUseCompoundFile(false);
idxW.SetMaxMergeDocs(10000);
idxW.SetMergeFactor(100);
idxW.Optimize();
}
public void processRequest(string queryName, NameValueCollection htKeys)
{
string retVal = string.Empty;
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(new Utility().getConfigXML());
XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']"));
string azureAccount = xNode.Attributes["account"].Value;
string azureEndpoint = xNode.Attributes["endpoint"].Value;
string azureSharedKey = xNode.Attributes["accountSharedKey"].Value;
string blobStorage = xNode.Attributes["endpoint"].Value;
xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']"));
string fragmentLocation = xNode.Attributes["value"].Value;
try
{
AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey");
azureResults ar = new azureResults();
// Get the page name and replace the .q extension with .xml
if (!queryName.ToLower().EndsWith(".xml"))
queryName += ".xml";
byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, "");
if (!ar.Succeeded)
{
NotifyError(new Exception(ar.StatusCode.ToString()));
}
else
{
xdoc = new XmlDocument();
System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
xdoc.LoadXml(enc.GetString(xmlFragment));
/*
* http://azure-architect.com/portals/16/MOOPData.xsd
*/
XmlNode xn = xdoc.SelectSingleNode("//storedProcedure[1]");
string storedProcedureName = xn.Attributes["procedureName"].Value;
string connectionStringName = xn.Attributes["connectionName"].Value;
SqlCommand cmd = new SqlCommand(storedProcedureName, new SqlConnection(new Utility().ResolveDataConnection(connectionStringName)));
cmd.CommandType = CommandType.StoredProcedure;
XmlNodeList xnl = xdoc.SelectNodes("/MOOPData/luceneData/field");
Field.Store[] fieldStore = new Field.Store[xnl.Count];
Field.Index[] indexType = new Field.Index[xnl.Count];
string[] luceneName = new string[xnl.Count];
string[] dataName = new string[xnl.Count];
bool[] isIncludedInOlioSearchFlag = new bool[xnl.Count];
bool[] isKeyFieldFlag = new bool[xnl.Count];
string olioSearchFieldName = string.Empty;
string azureContainerName = string.Empty;
olioSearchFieldName = xdoc.SelectSingleNode("//MOOPData/luceneData/olioSearchFieldName[1]").InnerText;
azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneData/azureContainer[1]").InnerText;
for (int i = 0; i < xnl.Count; i++)
{
XmlNode node = xnl[i];
switch (node.Attributes["store"].Value.ToLower())
{
case "compress": fieldStore[i] = Field.Store.COMPRESS; break;
case "no": fieldStore[i] = Field.Store.NO; break;
case "yes": fieldStore[i] = Field.Store.YES; break;
default: fieldStore[i] = Field.Store.NO; break;
}
switch (node.Attributes["index"].Value.ToLower())
{
case "analyzed": indexType[i] = Field.Index.ANALYZED; break;
case "analyzed_no_norms": indexType[i] = Field.Index.ANALYZED_NO_NORMS; break;
case "no": indexType[i] = Field.Index.NO; break;
case "no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break;
case "not_analyzed": indexType[i] = Field.Index.NOT_ANALYZED; break;
case "not_analyzed_no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break;
case "tokenized": indexType[i] = Field.Index.ANALYZED; break;
case "un_tokenized": indexType[i] = Field.Index.NOT_ANALYZED; break;
default: indexType[i] = Field.Index.NO; break;
}
dataName[i] = node.Attributes["dataName"].Value;
luceneName[i] = node.Attributes["luceneName"].Value;
isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true";
isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true";
isIncludedInOlioSearchFlag[i] = node.Attributes["isIncludedInOlioSearch"].Value == "true";
}
xnl = xdoc.SelectNodes("//parameter");
foreach (XmlNode node in xnl)
{
string parameterName = node.Attributes["parameterName"].Value;
string urlParameterName = node.Attributes["urlParameterName"].Value;
string dataType = node.Attributes["dataType"].Value;
string dataLength = node.Attributes["dataLength"].Value;
string defaultValue = node.Attributes["defaultValue"].Value;
if (!parameterName.StartsWith("@"))
parameterName = "@" + parameterName;
SqlParameter sp = new SqlParameter();
sp.ParameterName = parameterName;
switch (dataType)
{
case "bigint": sp.SqlDbType = SqlDbType.BigInt; break;
case "binary": sp.SqlDbType = SqlDbType.Binary; break;
case "bit": sp.SqlDbType = SqlDbType.Bit; break;
case "char": sp.SqlDbType = SqlDbType.Char; break;
case "date": sp.SqlDbType = SqlDbType.Date; break;
case "datetime": sp.SqlDbType = SqlDbType.DateTime; break;
case "datetime2": sp.SqlDbType = SqlDbType.DateTime2; break;
case "datetimeoffset": sp.SqlDbType = SqlDbType.DateTimeOffset; break;
case "decimal": sp.SqlDbType = SqlDbType.Decimal; break;
case "float": sp.SqlDbType = SqlDbType.Float; break;
case "geography": sp.SqlDbType = SqlDbType.Structured; break;
case "geometry": sp.SqlDbType = SqlDbType.Structured; break;
case "hierarchyid": sp.SqlDbType = SqlDbType.Structured; break;
case "image": sp.SqlDbType = SqlDbType.Image; break;
case "int": sp.SqlDbType = SqlDbType.Int; break;
case "money": sp.SqlDbType = SqlDbType.Money; break;
case "nchar": sp.SqlDbType = SqlDbType.NChar; break;
case "ntext": sp.SqlDbType = SqlDbType.NText; break;
case "nvarchar": sp.SqlDbType = SqlDbType.NVarChar; break;
case "real": sp.SqlDbType = SqlDbType.Real; break;
case "smalldatetime": sp.SqlDbType = SqlDbType.SmallDateTime; break;
case "smallint": sp.SqlDbType = SqlDbType.SmallInt; break;
case "smallmoney": sp.SqlDbType = SqlDbType.SmallMoney; break;
case "sql_variant": sp.SqlDbType = SqlDbType.Variant; break;
case "text": sp.SqlDbType = SqlDbType.Text; break;
case "time": sp.SqlDbType = SqlDbType.Time; break;
case "timestamp": sp.SqlDbType = SqlDbType.Timestamp; break;
case "tinyint": sp.SqlDbType = SqlDbType.TinyInt; break;
case "uniqueidentifier": sp.SqlDbType = SqlDbType.UniqueIdentifier; break;
case "varbinary": sp.SqlDbType = SqlDbType.VarBinary; break;
case "varchar": sp.SqlDbType = SqlDbType.VarChar; break;
case "xml": sp.SqlDbType = SqlDbType.Xml; break;
default: sp.SqlDbType = SqlDbType.Variant; break;
}
switch (urlParameterName.ToLower())
{
case "ipaddress": sp.Value = "127.0.0.1"; break;
case "domainname": sp.Value = ""; break;
default: if (htKeys[urlParameterName] != null)
sp.Value = htKeys[urlParameterName];
else
sp.Value = (defaultValue.ToLower() == "dbnull" ? DBNull.Value
: (object)defaultValue);
break;
}
cmd.Parameters.Add(sp);
}
cmd.Connection.Open();
SqlDataReader dr = cmd.ExecuteReader();
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
bool findexExists = false;
try
{
findexExists = IndexReader.IndexExists(azureDirectory);
if ((findexExists) && IndexWriter.IsLocked(azureDirectory))
azureDirectory.ClearLock("write.lock");
}
catch (Exception e)
{
Trace.WriteLine(e.ToString());
return;
}
IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024));
idxW.SetRAMBufferSizeMB(10.0);
idxW.SetUseCompoundFile(false);
idxW.SetMaxMergeDocs(10000);
idxW.SetMergeFactor(100);
while (dr.Read())
{
StringBuilder olioSearch = new StringBuilder();
Document doc = new Document();
for (int i = 0; i <= dataName.GetUpperBound(0); i++)
{
if (isKeyFieldFlag[i])
{
NotifyCaller(string.Format("Processing {0}", dr[dataName[i]].ToString().ToLower()));
idxW.DeleteDocuments(new Term(luceneName[i], dr[dataName[i]].ToString().ToLower()));
doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString().ToLower(), Field.Store.YES, Field.Index.NOT_ANALYZED));
}
else
try
{
doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString(), fieldStore[i], indexType[i]));
if (isIncludedInOlioSearchFlag[i])
olioSearch.AppendFormat("\r\n{0}", dr[dataName[i]].ToString());
}
catch (Exception ex)
{
NotifyError(ex);
}
}
if (olioSearch.ToString() != string.Empty && olioSearchFieldName != string.Empty)
doc.Add(new Field(olioSearchFieldName, olioSearch.ToString(), Field.Store.NO, Field.Index.ANALYZED));
idxW.AddDocument(doc);
}
idxW.Commit();
idxW.Close();
}
}
catch (Exception ex)
{
MOOPFramework.FrameworkUtility u = new MOOPFramework.FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection"));
u.LogData("localhost", "quoteSearchLoader", "testing", string.Empty, string.Empty, "", "QueryError", ex.ToString(),
u.nvc2XML(htKeys));
//retVal = string.Format("<!-- {0} -->", ex.ToString());
NotifyError(new Exception("An error occured but it was logged for later review"));
}
finally { if (retVal == string.Empty) retVal = "<root />"; }
}
public void ProcessRequest(HttpContext context)
{
Int64 expirationSeconds = 1;
HttpCacheability hc = HttpCacheability.NoCache;
try
{
string retVal = string.Empty;
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(new Utility().getConfigXML());
XmlNode xNode = xdoc.SelectSingleNode("//blobData[@name='default']");
string azureAccount = xNode.SelectSingleNode("Setting[@name='account']").Attributes["value"].Value;
string azureEndpoint = xNode.SelectSingleNode("Setting[@name='endpoint']").Attributes["value"].Value;
string azureSharedKey = xNode.SelectSingleNode("Setting[@name='accountSharedKey']").Attributes["value"].Value;
string blobStorage = azureEndpoint;
xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='luceneFragments']"));
string fragmentLocation = xNode.Attributes["value"].Value ;
string queryName = string.Empty;
SetCompression(context);
try
{
AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey");
azureResults ar = new azureResults();
// Get the page name and replace the .q extension with .xml
queryName = context.Request.Path;
queryName = queryName.Substring(queryName.LastIndexOf("/") + 1);
queryName = queryName.Substring(0, queryName.LastIndexOf(".")) + ".xml";
byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, "");
if (!ar.Succeeded)
{
context.Response.StatusCode = (int)ar.StatusCode;
}
else
{
xdoc = new XmlDocument();
System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
xdoc.LoadXml(enc.GetString(xmlFragment));
/*
* http://azure-architect.com/portals/16/MOOPData.xsd
*/
string azureContainerName = string.Empty;
azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneSearch/azureContainer[1]").InnerText;
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
AzureDirectory azDir = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
// IndexSearcher searcher = new IndexSearcher("azure-lucene-search");
try
{
var q = new BooleanQuery();
XmlNode xn = xdoc.SelectSingleNode("//luceneSearch[1]");
XmlNodeList xnl = xn.SelectNodes("//query");
Query[] qArray = new Query[xnl.Count];
bool hasReplaceableSearchValues = false;
bool hasReplacedAtLeastOneValue = false;
bool requirementsMet = true;
for (int i = 0; i < xnl.Count; i++)
{
XmlNode node = xnl[i];
string term = string.Empty;
term = node.Attributes["term"].Value;
string termValue = node.Attributes["termValue"].Value;
string variableValue = node.Attributes["variableValue"].Value;
string parmValue = string.Empty;
string requiredField = node.Attributes["required"].Value;
if (variableValue == "true") // See if there is a replacement attempt
{
if (requiredField == "true" && !context.Request.Params.AllKeys.Contains(termValue))
requirementsMet = false;
hasReplaceableSearchValues = true; // Set the flag to say we are attempting to replace the value
if (context.Request.Params.AllKeys.Contains(termValue)) // The parameter exists
{
hasReplacedAtLeastOneValue = true;
parmValue = context.Request.Params[termValue].Replace("+", " ").Replace("%20", " ");
}
}
if (node.Attributes["useAnalyzer"].Value == "true") // Should we use the analyzer
{
if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value
qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(parmValue);
if (variableValue != "true")
qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(termValue);
}
else // Using a Boolean
{
if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value
qArray[i] = new TermQuery(new Term(term, parmValue));
if (variableValue != "true")
qArray[i] = new TermQuery(new Term(term, termValue));
}
if (qArray[i] != null)
{
switch (node.Attributes["booleanClauseOccurs"].Value.ToLower())
{
case "must": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break;
case "must_not": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST_NOT)); break;
case "should": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.SHOULD)); break;
default: q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break;
}
}
}
IndexSearcher searcher2 = new IndexSearcher(azDir, true);
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true);
// If we have either no replaceable values or have replaceable values and at least one was provided
if (!hasReplaceableSearchValues || (hasReplaceableSearchValues && hasReplacedAtLeastOneValue))
if (requirementsMet)
searcher2.Search(q, collector);
int indexID = 1;
ScoreDoc[] hits = collector.TopDocs().ScoreDocs;
StringBuilder xmlOutput = new StringBuilder();
xmlOutput.AppendFormat("<?xml version=\"1.0\"?><root>");
for (int i = 0; i < hits.Length; ++i)
{
xmlOutput.AppendFormat("<hit>");
int docId = hits[i].Doc;
Document d = searcher2.Doc(docId);
xmlOutput.AppendFormat("<score>{0}</score><docID>{1}</docID>", hits[i].Score, indexID ++);
foreach (Field f in d.GetFields())
{
if (f.StringValue() == null)
xmlOutput.AppendFormat("<{0} />", f.Name());
else
xmlOutput.AppendFormat("<{0}>{1}</{0}>", f.Name(), System.Security.SecurityElement.Escape(f.StringValue()));
}
xmlOutput.AppendFormat("</hit>");
}
xmlOutput.AppendFormat("</root>");
retVal = xmlOutput.ToString();
}
catch (Exception ex)
{
retVal = "<root />";
}
MessageFormatOptions defaultOptions = new MessageFormatOptions();
string preferredContentType = string.Empty;
if (context.Request.HttpMethod == "GET" || context.Request.HttpMethod == "DELETE")
{
preferredContentType = (context.Request.AcceptTypes ?? new string[0])
.Select(m => m.Split(';')[0])
.FirstOrDefault(m => defaultOptions.MimeInputTypes.ContainsKey(m))
?? defaultOptions.DefaultContentType;
if (preferredContentType.Trim() == string.Empty)
preferredContentType = context.Request.Headers["Content-Type"];
}
else preferredContentType = context.Request.Headers["Content-Type"];
if (preferredContentType == "application/json")
{
context.Response.ContentType = preferredContentType;
XmlDocument doc = new XmlDocument();
doc.LoadXml(retVal);
retVal = JsonConvert.SerializeXmlNode(doc);
}
else
{
string luceneTransform = string.Empty;
luceneTransform = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").InnerText;
if (luceneTransform != string.Empty && luceneTransform != null)
{
xmlFragment = abs.GetBlob(fragmentLocation, luceneTransform, "", ref ar, "");
if (ar.Succeeded)
{
XsltUtil xslu = new XsltUtil();
retVal = XsltUtil.TransformXml(retVal, System.Text.ASCIIEncoding.ASCII.GetString(xmlFragment));
string transformContentType = "text/html";
try { transformContentType = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").Attributes["contentType"].Value; }
catch { }
context.Response.ContentType = transformContentType;
}
}
else
{
context.Response.ContentType = "text/xml";
}
}
}
}
catch (Exception ex)
{
FrameworkUtility u = new FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection"));
FrameworkUtility.ParsedURI pUri = new FrameworkUtility.ParsedURI(context.Request.RawUrl);
u.LogData(pUri.domainName, pUri.folderName, pUri.pageName, context.Request.HttpMethod, context.Request.UserHostAddress, "", "QueryError", ex.ToString(),
u.nvc2XML(context.Request.Headers));
//retVal = string.Format("<!-- {0} -->", ex.ToString());
context.Response.StatusDescription = "An error occured but it was logged for later review";
context.Response.StatusCode = 400;
}
finally { if (retVal == string.Empty) retVal = "<root />"; }
context.Response.Cache.SetExpires(DateTime.Now.AddSeconds(expirationSeconds));
context.Response.Cache.SetCacheability(hc);
context.Response.Write(retVal);
}
catch (Exception ex)
{
context.Response.StatusCode = 404;
context.Response.Close();
}
}