// Helper function to allow Storage Client 1.7 (Microsoft.WindowsAzure.StorageClient) to utilize this class. // Remove this function if only using Storage Client 2.0 (Microsoft.WindowsAzure.Storage). public void DownloadBlobAsync(Microsoft.WindowsAzure.StorageClient.CloudBlob blob, string LocalFile) { Microsoft.WindowsAzure.StorageCredentialsAccountAndKey account = blob.ServiceClient.Credentials as Microsoft.WindowsAzure.StorageCredentialsAccountAndKey; ICloudBlob blob2 = new CloudBlockBlob(blob.Attributes.Uri, new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(blob.ServiceClient.Credentials.AccountName, account.Credentials.ExportBase64EncodedKey())); DownloadBlobAsync(blob2, LocalFile); }
public void optimizeIndex(string azureContainerName) { XmlDocument xdoc = new XmlDocument(); xdoc.LoadXml(new Utility().getConfigXML()); XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']")); string azureAccount = xNode.Attributes["account"].Value; string azureEndpoint = xNode.Attributes["endpoint"].Value; string azureSharedKey = xNode.Attributes["accountSharedKey"].Value; string blobStorage = xNode.Attributes["endpoint"].Value; xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']")); string fragmentLocation = xNode.Attributes["value"].Value; Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey); Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false); AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory()); bool findexExists = false; try { findexExists = IndexReader.IndexExists(azureDirectory); if ((findexExists) && IndexWriter.IsLocked(azureDirectory)) azureDirectory.ClearLock("write.lock"); } catch (Exception e) { Trace.WriteLine(e.ToString()); return; } IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024)); idxW.SetRAMBufferSizeMB(10.0); idxW.SetUseCompoundFile(false); idxW.SetMaxMergeDocs(10000); idxW.SetMergeFactor(100); idxW.Optimize(); }
public void processRequest(string queryName, NameValueCollection htKeys) { string retVal = string.Empty; XmlDocument xdoc = new XmlDocument(); xdoc.LoadXml(new Utility().getConfigXML()); XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']")); string azureAccount = xNode.Attributes["account"].Value; string azureEndpoint = xNode.Attributes["endpoint"].Value; string azureSharedKey = xNode.Attributes["accountSharedKey"].Value; string blobStorage = xNode.Attributes["endpoint"].Value; xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']")); string fragmentLocation = xNode.Attributes["value"].Value; try { AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey"); azureResults ar = new azureResults(); // Get the page name and replace the .q extension with .xml if (!queryName.ToLower().EndsWith(".xml")) queryName += ".xml"; byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, ""); if (!ar.Succeeded) { NotifyError(new Exception(ar.StatusCode.ToString())); } else { xdoc = new XmlDocument(); System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); xdoc.LoadXml(enc.GetString(xmlFragment)); /* * http://azure-architect.com/portals/16/MOOPData.xsd */ XmlNode xn = xdoc.SelectSingleNode("//storedProcedure[1]"); string storedProcedureName = xn.Attributes["procedureName"].Value; string connectionStringName = xn.Attributes["connectionName"].Value; SqlCommand cmd = new SqlCommand(storedProcedureName, new SqlConnection(new Utility().ResolveDataConnection(connectionStringName))); cmd.CommandType = CommandType.StoredProcedure; XmlNodeList xnl = xdoc.SelectNodes("/MOOPData/luceneData/field"); Field.Store[] fieldStore = new Field.Store[xnl.Count]; Field.Index[] indexType = new Field.Index[xnl.Count]; string[] luceneName = new string[xnl.Count]; string[] dataName = new string[xnl.Count]; bool[] isIncludedInOlioSearchFlag = new bool[xnl.Count]; bool[] isKeyFieldFlag = new bool[xnl.Count]; string olioSearchFieldName = string.Empty; string azureContainerName = string.Empty; olioSearchFieldName = xdoc.SelectSingleNode("//MOOPData/luceneData/olioSearchFieldName[1]").InnerText; azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneData/azureContainer[1]").InnerText; for (int i = 0; i < xnl.Count; i++) { XmlNode node = xnl[i]; switch (node.Attributes["store"].Value.ToLower()) { case "compress": fieldStore[i] = Field.Store.COMPRESS; break; case "no": fieldStore[i] = Field.Store.NO; break; case "yes": fieldStore[i] = Field.Store.YES; break; default: fieldStore[i] = Field.Store.NO; break; } switch (node.Attributes["index"].Value.ToLower()) { case "analyzed": indexType[i] = Field.Index.ANALYZED; break; case "analyzed_no_norms": indexType[i] = Field.Index.ANALYZED_NO_NORMS; break; case "no": indexType[i] = Field.Index.NO; break; case "no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break; case "not_analyzed": indexType[i] = Field.Index.NOT_ANALYZED; break; case "not_analyzed_no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break; case "tokenized": indexType[i] = Field.Index.ANALYZED; break; case "un_tokenized": indexType[i] = Field.Index.NOT_ANALYZED; break; default: indexType[i] = Field.Index.NO; break; } dataName[i] = node.Attributes["dataName"].Value; luceneName[i] = node.Attributes["luceneName"].Value; isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true"; isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true"; isIncludedInOlioSearchFlag[i] = node.Attributes["isIncludedInOlioSearch"].Value == "true"; } xnl = xdoc.SelectNodes("//parameter"); foreach (XmlNode node in xnl) { string parameterName = node.Attributes["parameterName"].Value; string urlParameterName = node.Attributes["urlParameterName"].Value; string dataType = node.Attributes["dataType"].Value; string dataLength = node.Attributes["dataLength"].Value; string defaultValue = node.Attributes["defaultValue"].Value; if (!parameterName.StartsWith("@")) parameterName = "@" + parameterName; SqlParameter sp = new SqlParameter(); sp.ParameterName = parameterName; switch (dataType) { case "bigint": sp.SqlDbType = SqlDbType.BigInt; break; case "binary": sp.SqlDbType = SqlDbType.Binary; break; case "bit": sp.SqlDbType = SqlDbType.Bit; break; case "char": sp.SqlDbType = SqlDbType.Char; break; case "date": sp.SqlDbType = SqlDbType.Date; break; case "datetime": sp.SqlDbType = SqlDbType.DateTime; break; case "datetime2": sp.SqlDbType = SqlDbType.DateTime2; break; case "datetimeoffset": sp.SqlDbType = SqlDbType.DateTimeOffset; break; case "decimal": sp.SqlDbType = SqlDbType.Decimal; break; case "float": sp.SqlDbType = SqlDbType.Float; break; case "geography": sp.SqlDbType = SqlDbType.Structured; break; case "geometry": sp.SqlDbType = SqlDbType.Structured; break; case "hierarchyid": sp.SqlDbType = SqlDbType.Structured; break; case "image": sp.SqlDbType = SqlDbType.Image; break; case "int": sp.SqlDbType = SqlDbType.Int; break; case "money": sp.SqlDbType = SqlDbType.Money; break; case "nchar": sp.SqlDbType = SqlDbType.NChar; break; case "ntext": sp.SqlDbType = SqlDbType.NText; break; case "nvarchar": sp.SqlDbType = SqlDbType.NVarChar; break; case "real": sp.SqlDbType = SqlDbType.Real; break; case "smalldatetime": sp.SqlDbType = SqlDbType.SmallDateTime; break; case "smallint": sp.SqlDbType = SqlDbType.SmallInt; break; case "smallmoney": sp.SqlDbType = SqlDbType.SmallMoney; break; case "sql_variant": sp.SqlDbType = SqlDbType.Variant; break; case "text": sp.SqlDbType = SqlDbType.Text; break; case "time": sp.SqlDbType = SqlDbType.Time; break; case "timestamp": sp.SqlDbType = SqlDbType.Timestamp; break; case "tinyint": sp.SqlDbType = SqlDbType.TinyInt; break; case "uniqueidentifier": sp.SqlDbType = SqlDbType.UniqueIdentifier; break; case "varbinary": sp.SqlDbType = SqlDbType.VarBinary; break; case "varchar": sp.SqlDbType = SqlDbType.VarChar; break; case "xml": sp.SqlDbType = SqlDbType.Xml; break; default: sp.SqlDbType = SqlDbType.Variant; break; } switch (urlParameterName.ToLower()) { case "ipaddress": sp.Value = "127.0.0.1"; break; case "domainname": sp.Value = ""; break; default: if (htKeys[urlParameterName] != null) sp.Value = htKeys[urlParameterName]; else sp.Value = (defaultValue.ToLower() == "dbnull" ? DBNull.Value : (object)defaultValue); break; } cmd.Parameters.Add(sp); } cmd.Connection.Open(); SqlDataReader dr = cmd.ExecuteReader(); Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey); Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false); AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory()); bool findexExists = false; try { findexExists = IndexReader.IndexExists(azureDirectory); if ((findexExists) && IndexWriter.IsLocked(azureDirectory)) azureDirectory.ClearLock("write.lock"); } catch (Exception e) { Trace.WriteLine(e.ToString()); return; } IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024)); idxW.SetRAMBufferSizeMB(10.0); idxW.SetUseCompoundFile(false); idxW.SetMaxMergeDocs(10000); idxW.SetMergeFactor(100); while (dr.Read()) { StringBuilder olioSearch = new StringBuilder(); Document doc = new Document(); for (int i = 0; i <= dataName.GetUpperBound(0); i++) { if (isKeyFieldFlag[i]) { NotifyCaller(string.Format("Processing {0}", dr[dataName[i]].ToString().ToLower())); idxW.DeleteDocuments(new Term(luceneName[i], dr[dataName[i]].ToString().ToLower())); doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString().ToLower(), Field.Store.YES, Field.Index.NOT_ANALYZED)); } else try { doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString(), fieldStore[i], indexType[i])); if (isIncludedInOlioSearchFlag[i]) olioSearch.AppendFormat("\r\n{0}", dr[dataName[i]].ToString()); } catch (Exception ex) { NotifyError(ex); } } if (olioSearch.ToString() != string.Empty && olioSearchFieldName != string.Empty) doc.Add(new Field(olioSearchFieldName, olioSearch.ToString(), Field.Store.NO, Field.Index.ANALYZED)); idxW.AddDocument(doc); } idxW.Commit(); idxW.Close(); } } catch (Exception ex) { MOOPFramework.FrameworkUtility u = new MOOPFramework.FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection")); u.LogData("localhost", "quoteSearchLoader", "testing", string.Empty, string.Empty, "", "QueryError", ex.ToString(), u.nvc2XML(htKeys)); //retVal = string.Format("<!-- {0} -->", ex.ToString()); NotifyError(new Exception("An error occured but it was logged for later review")); } finally { if (retVal == string.Empty) retVal = "<root />"; } }
public void ProcessRequest(HttpContext context) { Int64 expirationSeconds = 1; HttpCacheability hc = HttpCacheability.NoCache; try { string retVal = string.Empty; XmlDocument xdoc = new XmlDocument(); xdoc.LoadXml(new Utility().getConfigXML()); XmlNode xNode = xdoc.SelectSingleNode("//blobData[@name='default']"); string azureAccount = xNode.SelectSingleNode("Setting[@name='account']").Attributes["value"].Value; string azureEndpoint = xNode.SelectSingleNode("Setting[@name='endpoint']").Attributes["value"].Value; string azureSharedKey = xNode.SelectSingleNode("Setting[@name='accountSharedKey']").Attributes["value"].Value; string blobStorage = azureEndpoint; xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='luceneFragments']")); string fragmentLocation = xNode.Attributes["value"].Value ; string queryName = string.Empty; SetCompression(context); try { AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey"); azureResults ar = new azureResults(); // Get the page name and replace the .q extension with .xml queryName = context.Request.Path; queryName = queryName.Substring(queryName.LastIndexOf("/") + 1); queryName = queryName.Substring(0, queryName.LastIndexOf(".")) + ".xml"; byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, ""); if (!ar.Succeeded) { context.Response.StatusCode = (int)ar.StatusCode; } else { xdoc = new XmlDocument(); System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); xdoc.LoadXml(enc.GetString(xmlFragment)); /* * http://azure-architect.com/portals/16/MOOPData.xsd */ string azureContainerName = string.Empty; azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneSearch/azureContainer[1]").InnerText; Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey); Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false); AzureDirectory azDir = new AzureDirectory(csa, azureContainerName, new RAMDirectory()); // IndexSearcher searcher = new IndexSearcher("azure-lucene-search"); try { var q = new BooleanQuery(); XmlNode xn = xdoc.SelectSingleNode("//luceneSearch[1]"); XmlNodeList xnl = xn.SelectNodes("//query"); Query[] qArray = new Query[xnl.Count]; bool hasReplaceableSearchValues = false; bool hasReplacedAtLeastOneValue = false; bool requirementsMet = true; for (int i = 0; i < xnl.Count; i++) { XmlNode node = xnl[i]; string term = string.Empty; term = node.Attributes["term"].Value; string termValue = node.Attributes["termValue"].Value; string variableValue = node.Attributes["variableValue"].Value; string parmValue = string.Empty; string requiredField = node.Attributes["required"].Value; if (variableValue == "true") // See if there is a replacement attempt { if (requiredField == "true" && !context.Request.Params.AllKeys.Contains(termValue)) requirementsMet = false; hasReplaceableSearchValues = true; // Set the flag to say we are attempting to replace the value if (context.Request.Params.AllKeys.Contains(termValue)) // The parameter exists { hasReplacedAtLeastOneValue = true; parmValue = context.Request.Params[termValue].Replace("+", " ").Replace("%20", " "); } } if (node.Attributes["useAnalyzer"].Value == "true") // Should we use the analyzer { if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(parmValue); if (variableValue != "true") qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(termValue); } else // Using a Boolean { if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value qArray[i] = new TermQuery(new Term(term, parmValue)); if (variableValue != "true") qArray[i] = new TermQuery(new Term(term, termValue)); } if (qArray[i] != null) { switch (node.Attributes["booleanClauseOccurs"].Value.ToLower()) { case "must": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break; case "must_not": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST_NOT)); break; case "should": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.SHOULD)); break; default: q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break; } } } IndexSearcher searcher2 = new IndexSearcher(azDir, true); TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true); // If we have either no replaceable values or have replaceable values and at least one was provided if (!hasReplaceableSearchValues || (hasReplaceableSearchValues && hasReplacedAtLeastOneValue)) if (requirementsMet) searcher2.Search(q, collector); int indexID = 1; ScoreDoc[] hits = collector.TopDocs().ScoreDocs; StringBuilder xmlOutput = new StringBuilder(); xmlOutput.AppendFormat("<?xml version=\"1.0\"?><root>"); for (int i = 0; i < hits.Length; ++i) { xmlOutput.AppendFormat("<hit>"); int docId = hits[i].Doc; Document d = searcher2.Doc(docId); xmlOutput.AppendFormat("<score>{0}</score><docID>{1}</docID>", hits[i].Score, indexID ++); foreach (Field f in d.GetFields()) { if (f.StringValue() == null) xmlOutput.AppendFormat("<{0} />", f.Name()); else xmlOutput.AppendFormat("<{0}>{1}</{0}>", f.Name(), System.Security.SecurityElement.Escape(f.StringValue())); } xmlOutput.AppendFormat("</hit>"); } xmlOutput.AppendFormat("</root>"); retVal = xmlOutput.ToString(); } catch (Exception ex) { retVal = "<root />"; } MessageFormatOptions defaultOptions = new MessageFormatOptions(); string preferredContentType = string.Empty; if (context.Request.HttpMethod == "GET" || context.Request.HttpMethod == "DELETE") { preferredContentType = (context.Request.AcceptTypes ?? new string[0]) .Select(m => m.Split(';')[0]) .FirstOrDefault(m => defaultOptions.MimeInputTypes.ContainsKey(m)) ?? defaultOptions.DefaultContentType; if (preferredContentType.Trim() == string.Empty) preferredContentType = context.Request.Headers["Content-Type"]; } else preferredContentType = context.Request.Headers["Content-Type"]; if (preferredContentType == "application/json") { context.Response.ContentType = preferredContentType; XmlDocument doc = new XmlDocument(); doc.LoadXml(retVal); retVal = JsonConvert.SerializeXmlNode(doc); } else { string luceneTransform = string.Empty; luceneTransform = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").InnerText; if (luceneTransform != string.Empty && luceneTransform != null) { xmlFragment = abs.GetBlob(fragmentLocation, luceneTransform, "", ref ar, ""); if (ar.Succeeded) { XsltUtil xslu = new XsltUtil(); retVal = XsltUtil.TransformXml(retVal, System.Text.ASCIIEncoding.ASCII.GetString(xmlFragment)); string transformContentType = "text/html"; try { transformContentType = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").Attributes["contentType"].Value; } catch { } context.Response.ContentType = transformContentType; } } else { context.Response.ContentType = "text/xml"; } } } } catch (Exception ex) { FrameworkUtility u = new FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection")); FrameworkUtility.ParsedURI pUri = new FrameworkUtility.ParsedURI(context.Request.RawUrl); u.LogData(pUri.domainName, pUri.folderName, pUri.pageName, context.Request.HttpMethod, context.Request.UserHostAddress, "", "QueryError", ex.ToString(), u.nvc2XML(context.Request.Headers)); //retVal = string.Format("<!-- {0} -->", ex.ToString()); context.Response.StatusDescription = "An error occured but it was logged for later review"; context.Response.StatusCode = 400; } finally { if (retVal == string.Empty) retVal = "<root />"; } context.Response.Cache.SetExpires(DateTime.Now.AddSeconds(expirationSeconds)); context.Response.Cache.SetCacheability(hc); context.Response.Write(retVal); } catch (Exception ex) { context.Response.StatusCode = 404; context.Response.Close(); } }