示例#1
0
        // Helper function to allow Storage Client 1.7 (Microsoft.WindowsAzure.StorageClient) to utilize this class.
        // Remove this function if only using Storage Client 2.0 (Microsoft.WindowsAzure.Storage).
        public void DownloadBlobAsync(Microsoft.WindowsAzure.StorageClient.CloudBlob blob, string LocalFile)
        {
            Microsoft.WindowsAzure.StorageCredentialsAccountAndKey account = blob.ServiceClient.Credentials as Microsoft.WindowsAzure.StorageCredentialsAccountAndKey;
            ICloudBlob blob2 = new CloudBlockBlob(blob.Attributes.Uri, new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(blob.ServiceClient.Credentials.AccountName, account.Credentials.ExportBase64EncodedKey()));

            DownloadBlobAsync(blob2, LocalFile);
        }
        public void optimizeIndex(string azureContainerName)
        {
            XmlDocument xdoc = new XmlDocument();
              xdoc.LoadXml(new Utility().getConfigXML());
              XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']"));

              string azureAccount = xNode.Attributes["account"].Value;
              string azureEndpoint = xNode.Attributes["endpoint"].Value;
              string azureSharedKey = xNode.Attributes["accountSharedKey"].Value;
              string blobStorage = xNode.Attributes["endpoint"].Value;

              xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']"));
              string fragmentLocation = xNode.Attributes["value"].Value;

              Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
              Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
              AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
              bool findexExists = false;
              try
              {
            findexExists = IndexReader.IndexExists(azureDirectory);
            if ((findexExists) && IndexWriter.IsLocked(azureDirectory))
              azureDirectory.ClearLock("write.lock");
              }
              catch (Exception e)
              {
            Trace.WriteLine(e.ToString());
            return;
              }

              IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024));
              idxW.SetRAMBufferSizeMB(10.0);
              idxW.SetUseCompoundFile(false);
              idxW.SetMaxMergeDocs(10000);
              idxW.SetMergeFactor(100);
              idxW.Optimize();
        }
        public void processRequest(string queryName, NameValueCollection htKeys)
        {
            string retVal = string.Empty;
              XmlDocument xdoc = new XmlDocument();
              xdoc.LoadXml(new Utility().getConfigXML());
              XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']"));

              string azureAccount = xNode.Attributes["account"].Value;
              string azureEndpoint = xNode.Attributes["endpoint"].Value;
              string azureSharedKey = xNode.Attributes["accountSharedKey"].Value;
              string blobStorage = xNode.Attributes["endpoint"].Value;

              xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']"));
              string fragmentLocation = xNode.Attributes["value"].Value;
              try
              {
            AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey");
            azureResults ar = new azureResults();
            // Get the page name and replace the .q extension with .xml
            if (!queryName.ToLower().EndsWith(".xml"))
              queryName += ".xml";
            byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, "");
            if (!ar.Succeeded)
            {
              NotifyError(new Exception(ar.StatusCode.ToString()));
            }
            else
            {

              xdoc = new XmlDocument();
              System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
              xdoc.LoadXml(enc.GetString(xmlFragment));
              /*
               * http://azure-architect.com/portals/16/MOOPData.xsd

               */
              XmlNode xn = xdoc.SelectSingleNode("//storedProcedure[1]");
              string storedProcedureName = xn.Attributes["procedureName"].Value;
              string connectionStringName = xn.Attributes["connectionName"].Value;

              SqlCommand cmd = new SqlCommand(storedProcedureName, new SqlConnection(new Utility().ResolveDataConnection(connectionStringName)));
              cmd.CommandType = CommandType.StoredProcedure;
              XmlNodeList xnl = xdoc.SelectNodes("/MOOPData/luceneData/field");
              Field.Store[] fieldStore = new Field.Store[xnl.Count];
              Field.Index[] indexType = new Field.Index[xnl.Count];
              string[] luceneName = new string[xnl.Count];
              string[] dataName = new string[xnl.Count];
              bool[] isIncludedInOlioSearchFlag = new bool[xnl.Count];
              bool[] isKeyFieldFlag = new bool[xnl.Count];
              string olioSearchFieldName = string.Empty;
              string azureContainerName = string.Empty;
              olioSearchFieldName = xdoc.SelectSingleNode("//MOOPData/luceneData/olioSearchFieldName[1]").InnerText;
              azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneData/azureContainer[1]").InnerText;
              for (int i = 0; i < xnl.Count; i++)
              {
            XmlNode node = xnl[i];
            switch (node.Attributes["store"].Value.ToLower())
            {
              case "compress": fieldStore[i] = Field.Store.COMPRESS; break;
              case "no": fieldStore[i] = Field.Store.NO; break;
              case "yes": fieldStore[i] = Field.Store.YES; break;
              default: fieldStore[i] = Field.Store.NO; break;
            }

            switch (node.Attributes["index"].Value.ToLower())
            {
              case "analyzed": indexType[i] = Field.Index.ANALYZED; break;
              case "analyzed_no_norms": indexType[i] = Field.Index.ANALYZED_NO_NORMS; break;
              case "no": indexType[i] = Field.Index.NO; break;
              case "no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break;
              case "not_analyzed": indexType[i] = Field.Index.NOT_ANALYZED; break;
              case "not_analyzed_no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break;
              case "tokenized": indexType[i] = Field.Index.ANALYZED; break;
              case "un_tokenized": indexType[i] = Field.Index.NOT_ANALYZED; break;
              default: indexType[i] = Field.Index.NO; break;
            }
            dataName[i] = node.Attributes["dataName"].Value;
            luceneName[i] = node.Attributes["luceneName"].Value;
            isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true";
            isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true";
            isIncludedInOlioSearchFlag[i] = node.Attributes["isIncludedInOlioSearch"].Value == "true";
              }

              xnl = xdoc.SelectNodes("//parameter");
              foreach (XmlNode node in xnl)
              {
            string parameterName = node.Attributes["parameterName"].Value;
            string urlParameterName = node.Attributes["urlParameterName"].Value;
            string dataType = node.Attributes["dataType"].Value;
            string dataLength = node.Attributes["dataLength"].Value;
            string defaultValue = node.Attributes["defaultValue"].Value;
            if (!parameterName.StartsWith("@"))
              parameterName = "@" + parameterName;
            SqlParameter sp = new SqlParameter();
            sp.ParameterName = parameterName;
            switch (dataType)
            {
              case "bigint": sp.SqlDbType = SqlDbType.BigInt; break;
              case "binary": sp.SqlDbType = SqlDbType.Binary; break;
              case "bit": sp.SqlDbType = SqlDbType.Bit; break;
              case "char": sp.SqlDbType = SqlDbType.Char; break;
              case "date": sp.SqlDbType = SqlDbType.Date; break;
              case "datetime": sp.SqlDbType = SqlDbType.DateTime; break;
              case "datetime2": sp.SqlDbType = SqlDbType.DateTime2; break;
              case "datetimeoffset": sp.SqlDbType = SqlDbType.DateTimeOffset; break;
              case "decimal": sp.SqlDbType = SqlDbType.Decimal; break;
              case "float": sp.SqlDbType = SqlDbType.Float; break;
              case "geography": sp.SqlDbType = SqlDbType.Structured; break;
              case "geometry": sp.SqlDbType = SqlDbType.Structured; break;
              case "hierarchyid": sp.SqlDbType = SqlDbType.Structured; break;
              case "image": sp.SqlDbType = SqlDbType.Image; break;
              case "int": sp.SqlDbType = SqlDbType.Int; break;
              case "money": sp.SqlDbType = SqlDbType.Money; break;
              case "nchar": sp.SqlDbType = SqlDbType.NChar; break;
              case "ntext": sp.SqlDbType = SqlDbType.NText; break;
              case "nvarchar": sp.SqlDbType = SqlDbType.NVarChar; break;
              case "real": sp.SqlDbType = SqlDbType.Real; break;
              case "smalldatetime": sp.SqlDbType = SqlDbType.SmallDateTime; break;
              case "smallint": sp.SqlDbType = SqlDbType.SmallInt; break;
              case "smallmoney": sp.SqlDbType = SqlDbType.SmallMoney; break;
              case "sql_variant": sp.SqlDbType = SqlDbType.Variant; break;
              case "text": sp.SqlDbType = SqlDbType.Text; break;
              case "time": sp.SqlDbType = SqlDbType.Time; break;
              case "timestamp": sp.SqlDbType = SqlDbType.Timestamp; break;
              case "tinyint": sp.SqlDbType = SqlDbType.TinyInt; break;
              case "uniqueidentifier": sp.SqlDbType = SqlDbType.UniqueIdentifier; break;
              case "varbinary": sp.SqlDbType = SqlDbType.VarBinary; break;
              case "varchar": sp.SqlDbType = SqlDbType.VarChar; break;
              case "xml": sp.SqlDbType = SqlDbType.Xml; break;
              default: sp.SqlDbType = SqlDbType.Variant; break;
            }
            switch (urlParameterName.ToLower())
            {
              case "ipaddress": sp.Value = "127.0.0.1"; break;
              case "domainname": sp.Value = ""; break;
              default: if (htKeys[urlParameterName] != null)
                  sp.Value = htKeys[urlParameterName];
                else
                  sp.Value = (defaultValue.ToLower() == "dbnull" ? DBNull.Value
                  : (object)defaultValue);
                break;
            }
            cmd.Parameters.Add(sp);
              }

              cmd.Connection.Open();
              SqlDataReader dr = cmd.ExecuteReader();
              Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
              Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
              AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
              bool findexExists = false;
              try
              {
            findexExists = IndexReader.IndexExists(azureDirectory);
            if ((findexExists) && IndexWriter.IsLocked(azureDirectory))
              azureDirectory.ClearLock("write.lock");
              }
              catch (Exception e)
              {
            Trace.WriteLine(e.ToString());
            return;
              }

              IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024));
              idxW.SetRAMBufferSizeMB(10.0);
              idxW.SetUseCompoundFile(false);
              idxW.SetMaxMergeDocs(10000);
              idxW.SetMergeFactor(100);
              while (dr.Read())
              {
            StringBuilder olioSearch = new StringBuilder();
            Document doc = new Document();
            for (int i = 0; i <= dataName.GetUpperBound(0); i++)
            {

              if (isKeyFieldFlag[i])
              {

                NotifyCaller(string.Format("Processing {0}", dr[dataName[i]].ToString().ToLower()));
                idxW.DeleteDocuments(new Term(luceneName[i], dr[dataName[i]].ToString().ToLower()));
                doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString().ToLower(), Field.Store.YES, Field.Index.NOT_ANALYZED));
              }
              else
                try
                {
                  doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString(), fieldStore[i], indexType[i]));

                  if (isIncludedInOlioSearchFlag[i])
                    olioSearch.AppendFormat("\r\n{0}", dr[dataName[i]].ToString());
                }
                catch (Exception ex)
                {
                  NotifyError(ex);
                }
            }
            if (olioSearch.ToString() != string.Empty && olioSearchFieldName != string.Empty)
              doc.Add(new Field(olioSearchFieldName, olioSearch.ToString(), Field.Store.NO, Field.Index.ANALYZED));
            idxW.AddDocument(doc);
              }
              idxW.Commit();
              idxW.Close();

            }
              }

              catch (Exception ex)
              {
            MOOPFramework.FrameworkUtility u = new MOOPFramework.FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection"));
            u.LogData("localhost", "quoteSearchLoader", "testing", string.Empty, string.Empty, "", "QueryError", ex.ToString(),
              u.nvc2XML(htKeys));
            //retVal = string.Format("<!-- {0} -->", ex.ToString());
            NotifyError(new Exception("An error occured but it was logged for later review"));
              }
              finally { if (retVal == string.Empty) retVal = "<root />"; }
        }
        public void ProcessRequest(HttpContext context)
        {
            Int64 expirationSeconds = 1;
              HttpCacheability hc = HttpCacheability.NoCache;

              try
              {
            string retVal = string.Empty;
            XmlDocument xdoc = new XmlDocument();
            xdoc.LoadXml(new Utility().getConfigXML());
            XmlNode xNode = xdoc.SelectSingleNode("//blobData[@name='default']");

            string azureAccount = xNode.SelectSingleNode("Setting[@name='account']").Attributes["value"].Value;
            string azureEndpoint = xNode.SelectSingleNode("Setting[@name='endpoint']").Attributes["value"].Value;
            string azureSharedKey = xNode.SelectSingleNode("Setting[@name='accountSharedKey']").Attributes["value"].Value;
            string blobStorage = azureEndpoint;

            xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='luceneFragments']"));
            string fragmentLocation = xNode.Attributes["value"].Value ;

            string queryName = string.Empty;

            SetCompression(context);

            try
            {
              AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey");
              azureResults ar = new azureResults();
              // Get the page name and replace the .q extension with .xml
              queryName = context.Request.Path;
              queryName = queryName.Substring(queryName.LastIndexOf("/") + 1);
              queryName = queryName.Substring(0, queryName.LastIndexOf(".")) + ".xml";
              byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, "");
              if (!ar.Succeeded)
              {
            context.Response.StatusCode = (int)ar.StatusCode;
              }
              else
              {
            xdoc = new XmlDocument();
            System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
            xdoc.LoadXml(enc.GetString(xmlFragment));
            /*
             * http://azure-architect.com/portals/16/MOOPData.xsd

             */
            string azureContainerName = string.Empty;
            azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneSearch/azureContainer[1]").InnerText;
            Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
            Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
            AzureDirectory azDir = new AzureDirectory(csa, azureContainerName, new RAMDirectory());

            //            IndexSearcher searcher = new IndexSearcher("azure-lucene-search");
            try
            {
              var q = new BooleanQuery();
              XmlNode xn = xdoc.SelectSingleNode("//luceneSearch[1]");
              XmlNodeList xnl = xn.SelectNodes("//query");
              Query[] qArray = new Query[xnl.Count];
              bool hasReplaceableSearchValues = false;
              bool hasReplacedAtLeastOneValue = false;
              bool requirementsMet = true;
              for (int i = 0; i < xnl.Count; i++)
              {
                XmlNode node = xnl[i];
                string term = string.Empty;
                term = node.Attributes["term"].Value;
                string termValue = node.Attributes["termValue"].Value;
                string variableValue = node.Attributes["variableValue"].Value;
                string parmValue = string.Empty;
                string requiredField = node.Attributes["required"].Value;
                if (variableValue == "true") // See if there is a replacement attempt
                {
                  if (requiredField == "true" && !context.Request.Params.AllKeys.Contains(termValue))
                    requirementsMet = false;
                  hasReplaceableSearchValues = true; // Set the flag to say we are attempting to replace the value
                  if (context.Request.Params.AllKeys.Contains(termValue)) // The parameter exists
                  {
                    hasReplacedAtLeastOneValue = true;
                    parmValue = context.Request.Params[termValue].Replace("+", " ").Replace("%20", " ");
                  }
                }
                if (node.Attributes["useAnalyzer"].Value == "true") // Should we use the analyzer
                {

                  if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value
                    qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(parmValue);
                   if (variableValue != "true")
                    qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(termValue);
                }
                else // Using a Boolean
                {
                  if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value
                    qArray[i] = new TermQuery(new Term(term, parmValue));
                  if (variableValue != "true")
                    qArray[i] = new TermQuery(new Term(term, termValue));
                }
                if (qArray[i] != null)
                {
                  switch (node.Attributes["booleanClauseOccurs"].Value.ToLower())
                  {
                    case "must": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break;
                    case "must_not": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST_NOT)); break;
                    case "should": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.SHOULD)); break;
                    default: q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break;
                  }
                }
              }
              IndexSearcher searcher2 = new IndexSearcher(azDir, true);
              TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true);
              // If we have either no replaceable values or have replaceable values and at least one was provided
              if (!hasReplaceableSearchValues || (hasReplaceableSearchValues && hasReplacedAtLeastOneValue))
                if (requirementsMet)
                  searcher2.Search(q, collector);
              int indexID = 1;
              ScoreDoc[] hits = collector.TopDocs().ScoreDocs;
              StringBuilder xmlOutput = new StringBuilder();
              xmlOutput.AppendFormat("<?xml version=\"1.0\"?><root>");
              for (int i = 0; i < hits.Length; ++i)
              {
                xmlOutput.AppendFormat("<hit>");
                int docId = hits[i].Doc;
                Document d = searcher2.Doc(docId);
                xmlOutput.AppendFormat("<score>{0}</score><docID>{1}</docID>", hits[i].Score, indexID ++);
                foreach (Field f in d.GetFields())
                {
                  if (f.StringValue() == null)
                    xmlOutput.AppendFormat("<{0} />", f.Name());
                  else
                    xmlOutput.AppendFormat("<{0}>{1}</{0}>", f.Name(), System.Security.SecurityElement.Escape(f.StringValue()));
                }
                xmlOutput.AppendFormat("</hit>");
              }
              xmlOutput.AppendFormat("</root>");
              retVal = xmlOutput.ToString();

            }
            catch (Exception ex)
            {
              retVal = "<root />";
            }
            MessageFormatOptions defaultOptions = new MessageFormatOptions();
            string preferredContentType = string.Empty;
            if (context.Request.HttpMethod == "GET" || context.Request.HttpMethod == "DELETE")
            {
                preferredContentType = (context.Request.AcceptTypes ?? new string[0])
                          .Select(m => m.Split(';')[0])
                          .FirstOrDefault(m => defaultOptions.MimeInputTypes.ContainsKey(m))
                          ?? defaultOptions.DefaultContentType;
                if (preferredContentType.Trim() == string.Empty)
                    preferredContentType = context.Request.Headers["Content-Type"];
            }
            else preferredContentType = context.Request.Headers["Content-Type"];
            if (preferredContentType == "application/json")
            {
                context.Response.ContentType = preferredContentType;
                XmlDocument doc = new XmlDocument();
                doc.LoadXml(retVal);
                retVal = JsonConvert.SerializeXmlNode(doc);
            }
            else
            {
                string luceneTransform = string.Empty;
                luceneTransform = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").InnerText;
                if (luceneTransform != string.Empty && luceneTransform != null)
                {
                    xmlFragment = abs.GetBlob(fragmentLocation, luceneTransform, "", ref ar, "");
                    if (ar.Succeeded)
                    {
                       XsltUtil xslu = new XsltUtil();
                        retVal = XsltUtil.TransformXml(retVal, System.Text.ASCIIEncoding.ASCII.GetString(xmlFragment));
                        string transformContentType = "text/html";
                        try { transformContentType = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").Attributes["contentType"].Value; }
                        catch { }
                        context.Response.ContentType = transformContentType;
                    }
                }

                else
                {
                    context.Response.ContentType = "text/xml";
                }
            }
              }
            }
            catch (Exception ex)
            {
              FrameworkUtility u = new FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection"));
              FrameworkUtility.ParsedURI pUri = new FrameworkUtility.ParsedURI(context.Request.RawUrl);
              u.LogData(pUri.domainName, pUri.folderName, pUri.pageName, context.Request.HttpMethod, context.Request.UserHostAddress, "", "QueryError", ex.ToString(),
            u.nvc2XML(context.Request.Headers));
              //retVal = string.Format("<!-- {0} -->", ex.ToString());
              context.Response.StatusDescription = "An error occured but it was logged for later review";
              context.Response.StatusCode = 400;
            }
            finally { if (retVal == string.Empty) retVal = "<root />"; }

            context.Response.Cache.SetExpires(DateTime.Now.AddSeconds(expirationSeconds));
            context.Response.Cache.SetCacheability(hc);
            context.Response.Write(retVal);
              }
              catch (Exception ex)
              {
            context.Response.StatusCode = 404;
            context.Response.Close();
              }
        }