public static void CreateIndex() {
try
{
var cloudAccount = Azure.GetStorageAccount();
using (var cacheDirectory = new RAMDirectory())
{
using (var azureDirectory = new AzureDirectory(cloudAccount, Azure.StorageContainerName, cacheDirectory))
{
using (Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30))
{
using (var indexWriter = new IndexWriter(azureDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED))
{
AddDocuments(indexWriter);
indexWriter.Commit();
}
}
}
}
}
catch (StorageException ex)
{
Trace.TraceError(ex.Message);
}
}
private void GenerateLuceneIndex(List<Entry> entries)
{
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=diningsearchstorage;AccountKey=xeYMzXThFxrU7SsAMGbSWLdy9psLFRMk5NI8x0bx24xtEg9MPIstf/xwPdjvDm6HpHZaCPxxVFCv/7DDd5wymA==");
AzureDirectory azureDirectory = new AzureDirectory(storageAccount, "diningsearchindex");
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
IndexWriter indexWriter = new IndexWriter(azureDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
////
int c = 0;
foreach (var entry in entries)
{
c++;
var item = new Document();
item.Add(new Field("Id", c.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
item.Add(new Field("Dish", entry.DishName, Field.Store.YES, Field.Index.ANALYZED));
item.Add(new Field("Cafe/Restaurant", string.Format("{0}/{1}", entry.CafeName, entry.RestaurantName), Field.Store.YES, Field.Index.ANALYZED));
item.Add(new Field("URL", entry.CafeUrl, Field.Store.YES, Field.Index.NOT_ANALYZED));
item.Add(new Field("Description", entry.Description ?? string.Empty, Field.Store.YES, Field.Index.ANALYZED));
item.Add(new Field("Price", entry.Price, Field.Store.YES, Field.Index.NOT_ANALYZED));
indexWriter.AddDocument(item);
}
////
indexWriter.Dispose();
azureDirectory.Dispose();
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
: base(blob.Name)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
string fileName = _name;
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine("GET {0} RETREIVED {1} bytes", _name, fileStream.Length);
fileStream.Close();
// and open it as an input
_indexInput = CacheDirectory.openInput(fileName, IOContext.DEFAULT);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public void TestReadAndWrite()
{
var cloudStorageAccount = CloudStorageAccount.Parse(connectionString);
const string containerName = "testcatalog2";
var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorageAccount, "temp", containerName: containerName);
var indexWriterConfig = new IndexWriterConfig(
Lucene.Net.Util.LuceneVersion.LUCENE_48,
new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48));
int dog = 0, cat = 0, car = 0;
using (var indexWriter = new IndexWriter(azureDirectory, indexWriterConfig))
{
for (var iDoc = 0; iDoc < 10000; iDoc++)
{
var bodyText = GeneratePhrase(40);
var doc = new Document {
new TextField("id", DateTime.Now.ToFileTimeUtc() + "-" + iDoc, Field.Store.YES),
new TextField("Title", GeneratePhrase(10), Field.Store.YES),
new TextField("Body", bodyText, Field.Store.YES)
};
dog += bodyText.Contains(" dog ") ? 1 : 0;
cat += bodyText.Contains(" cat ") ? 1 : 0;
car += bodyText.Contains(" car ") ? 1 : 0;
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}, {1} dog, {2} cat, {3} car", indexWriter.NumDocs, dog, cat, car);
}
try
{
var ireader = DirectoryReader.Open(azureDirectory);
for (var i = 0; i < 100; i++)
{
var searcher = new IndexSearcher(ireader);
var searchForPhrase = SearchForPhrase(searcher, "dog");
Assert.AreEqual(dog, searchForPhrase);
searchForPhrase = SearchForPhrase(searcher, "cat");
Assert.AreEqual(cat, searchForPhrase);
searchForPhrase = SearchForPhrase(searcher, "car");
Assert.AreEqual(car, searchForPhrase);
}
Console.WriteLine("Tests passsed");
}
catch (Exception x)
{
Console.WriteLine("Tests failed:\n{0}", x);
}
finally
{
// check the container exists, and delete it
var blobClient = cloudStorageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(containerName);
Assert.IsTrue(container.Exists()); // check the container exists
container.Delete();
}
}
private static void UnidirectionalSync(AzureDirectory sourceDirectory, Directory destinationDirectory)
{
var sourceFiles = sourceDirectory.ListAll();
var fileNameFilter = IndexFileNameFilter.Filter;
byte[] buffer = new byte[16384];
foreach (string sourceFile in sourceFiles)
{
// only copy file if it is accepted by Lucene's default filter
// and it does not already exist (except for segment map files, we always want those)
if (fileNameFilter.Accept(null, sourceFile) && (!destinationDirectory.FileExists(sourceFile) || sourceFile.StartsWith("segment")))
{
IndexOutput indexOutput = null;
IndexInput indexInput = null;
try
{
indexOutput = destinationDirectory.CreateOutput(sourceFile);
indexInput = sourceDirectory.OpenInput(sourceFile);
long length = indexInput.Length();
long position = 0;
while (position < length)
{
int bytesToRead = position + 16384L > length ? (int)(length - position) : 16384;
indexInput.ReadBytes(buffer, 0, bytesToRead);
indexOutput.WriteBytes(buffer, bytesToRead);
position += bytesToRead;
}
}
finally
{
try
{
indexOutput?.Close();
}
finally
{
indexInput?.Close();
}
}
}
}
// we'll remove old files from both AzureDirectory's cache directory, as well as our destination directory
// (only when older than 45 minutes - old files may still have active searches on them so we need a margin)
var referenceTimestamp = LuceneTimestampFromDateTime(DateTime.UtcNow.AddMinutes(-45));
// remove old files from AzureDirectory cache directory
RemoveOldFiles(sourceDirectory.CacheDirectory, sourceFiles, referenceTimestamp);
// remove old files from destination directory
RemoveOldFiles(destinationDirectory, sourceFiles, referenceTimestamp);
}
static void Main(string[] args)
{
// default AzureDirectory stores cache in local temp folder
var azureDirectory = new AzureDirectory(CloudStorageAccount.Parse(ConfigurationManager.AppSettings["blobStorage"]), "TestCatalog6");
var findexExists = IndexReader.IndexExists(azureDirectory);
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), !IndexReader.IndexExists(azureDirectory), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
}
catch (LockObtainFailedException)
{
Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again");
if (Console.ReadLine().ToLower().Trim() == "y" )
azureDirectory.ClearLock("write.lock");
}
};
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
indexWriter.SetRAMBufferSizeMB(10.0);
//indexWriter.SetUseCompoundFile(false);
//indexWriter.SetMaxMergeDocs(10000);
//indexWriter.SetMergeFactor(100);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
if (iDoc % 10 == 0)
Console.WriteLine(iDoc);
var doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.NumDocs());
indexWriter.Dispose();
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(azureDirectory);
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
Console.Read();
}
public IActionResult Get(string searchTerm)
{
if (string.IsNullOrEmpty(searchTerm))
{
return new JsonResult(null);
}
if (searchTerm.StartsWith("*") || searchTerm.StartsWith("?"))
{
return new JsonResult(null);
}
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.DevelopmentStorageAccount;
CloudStorageAccount.TryParse(_options.ConnectionString, out cloudStorageAccount);
var azureDirectory = new AzureDirectory(cloudStorageAccount, "TestCatalog");
var searcher = new IndexSearcher(azureDirectory);
var analyzer = new NuGet.Indexing.IdentifierAnalyzer();
var queryParser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new[] { "Type", "ReturnType" }, analyzer);
Query query = queryParser.Parse(searchTerm);
//execute the query
var hits = searcher.Search(query, 50);
var packages = new List<SearchResult>();
foreach (var hit in hits.ScoreDocs)
{
var doc = searcher.Doc(hit.Doc);
var result = new SearchResult
{
FullTypeName = doc.GetField("Type").StringValue,
PackageName = doc.GetField("Package").StringValue
};
if (doc.GetField("Signature") != null)
{
result.Signature = doc.GetField("Signature").StringValue;
}
if (doc.GetField("ReturnType") != null)
{
result.ReturnType = doc.GetField("ReturnType").StringValue;
}
packages.Add(result);
}
return new JsonResult(packages);
}
public AzureDirectorySynchronizer(AzureDirectory sourceDirectory, Directory destinationDirectory)
{
if (sourceDirectory == null)
{
throw new ArgumentNullException(nameof(sourceDirectory));
}
if (destinationDirectory == null)
{
throw new ArgumentNullException(nameof(destinationDirectory));
}
SourceDirectory = sourceDirectory;
DestinationDirectory = destinationDirectory;
}
public ActionResult Index(string searchText) {
if (string.IsNullOrWhiteSpace(searchText))
{
return View((object)null);
}
var searchResults = new List<SearchResult>();
try
{
var cloudAccount = Azure.GetStorageAccount();
using (var cacheDirectory = new RAMDirectory())
{
using (
var azureDirectory = new AzureDirectory(
cloudAccount,
Azure.StorageContainerName,
cacheDirectory))
{
using (var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30))
{
using (var searcher = new IndexSearcher(azureDirectory))
{
var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "Body", analyzer);
var searchQuery = parser.Parse(searchText);
var hits = searcher.Search(searchQuery, 200);
var results = hits.TotalHits;
for (var i = 0; i < results; i++)
{
var doc = searcher.Doc(hits.ScoreDocs[i].Doc);
searchResults.Add(new SearchResult(doc));
}
}
}
}
}
}
catch (StorageException ex)
{
Trace.TraceError(ex.Message);
return this.View("SearchUnavailable");
}
return this.View(searchResults);
}
public override bool OnStart()
{
var readPath = RoleEnvironment.GetLocalResource("ReadStorage").RootPath;
var writePath = RoleEnvironment.GetLocalResource("WriteStorage").RootPath;
var subscription = "lucene" ;
var account = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("Blob"));
var hub = new Hub(RoleEnvironment.GetConfigurationSettingValue("ServiceBus"));
var criticalWait = Task.Run(() =>
{
string writeFolder = Path.Combine(writePath, "lucene");
if (!System.IO.Directory.Exists(writeFolder))
{
System.IO.Directory.CreateDirectory(writeFolder);
}
string combine = Path.Combine(readPath, subscription);
if (!System.IO.Directory.Exists(combine))
{
System.IO.Directory.CreateDirectory(combine);
}
var localDirectory = new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(combine));
var masterDirectory = new AzureDirectory(account, subscription);
var irs = new IntermediateReaderService(masterDirectory, hub, localDirectory, true);
masterDirectory.Dispose();
return new IndexInfo {IRS = irs as IDisposable, Directory = localDirectory};
})
.ContinueWith(t => _localIndexes = new[] {t.Result});
_luceneWriter = new SearchWriterService(new AzureDirectory(account, "lucene", new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(writePath))), true);
var subscriptionName = string.Format("{0:yyyyMMddHHmmss}_{1}", DateTime.UtcNow, Guid.NewGuid().ToString().Replace("-", string.Empty));
hub.CreateSubscription("lucene", subscriptionName)
.ContinueWith(t =>
{
_luceneTempInstanceSubscription = t.Result;
return hub.SubscribeWithRoutingKey("lucene", subscriptionName, OnSearchMessage)
.ContinueWith(t2 => _luceneSearchSubscription = t.Result);
})
.Unwrap();
return base.OnStart();
}
public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob)
{
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azureDirectory;
_blob = blob;
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
// create the local cache one we will operate against...
_indexOutput = CacheDirectory.createOutput(_name, IOContext.DEFAULT);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureIndexOutput(AzureDirectory azureDirectory, CloudBlob blob)
{
_fileMutex = new Mutex(false, _name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azureDirectory;
_blobContainer = _azureDirectory.BlobContainer;
_blob = blob;
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
// create the local cache one we will operate against...
_indexOutput = CacheDirectory.CreateOutput(_name);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary>
/// Initializes the Storage Provider.
/// </summary>
/// <param name="host">The Host of the Component.</param>
/// <param name="config">The Configuration data, if any.</param>
/// <param name="wiki">The wiki.</param>
/// <exception cref="ArgumentNullException">If <paramref name="host"/> or <paramref name="config"/> are <c>null</c>.</exception>
/// <exception cref="InvalidConfigurationException">If <paramref name="config"/> is not valid or is incorrect.</exception>
public void Init(IHostV40 host, string config, string wiki)
{
if (host == null)
{
throw new ArgumentNullException("host");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
if (config == "")
{
config = Config.GetConnectionString();
}
_host = host;
_wiki = string.IsNullOrEmpty(wiki) ? "root" : wiki;
_directory = new AzureDirectory(TableStorage.StorageAccount(config), wiki + "-searchindex");
}
public void TestReadAndWrite()
{
var connectionString = Environment.GetEnvironmentVariable("DataConnectionString") ?? "UseDevelopmentStorage=true";
var cloudStorageAccount = CloudStorageAccount.Parse(connectionString);
// default AzureDirectory stores cache in local temp folder
var azureDirectory = new AzureDirectory(cloudStorageAccount, "testcatalog");
using (var indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), !IndexReader.IndexExists(azureDirectory), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)))
{
indexWriter.SetRAMBufferSizeMB(10.0);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
var doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString() + "-" + iDoc.ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.NumDocs());
}
using (var searcher = new IndexSearcher(azureDirectory))
{
Assert.AreNotEqual(0, SearchForPhrase(searcher, "dog"));
Assert.AreNotEqual(0, SearchForPhrase(searcher, "cat"));
Assert.AreNotEqual(0, SearchForPhrase(searcher, "car"));
}
// check the container exists, and delete it
var blobClient = cloudStorageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference("testcatalog");
Assert.IsTrue(container.Exists()); // check the container exists
container.Delete();
}
public AzureIndexInput(AzureIndexInput cloneInput)
: base(cloneInput._blob.Name)
{
_fileMutex = BlobMutexManager.GrabMutex(cloneInput._name);
_fileMutex.WaitOne();
try
{
_azureDirectory = cloneInput._azureDirectory;
_blobContainer = cloneInput._blobContainer;
_blob = cloneInput._blob;
_indexInput = cloneInput._indexInput.clone();
}
catch (Exception)
{
// sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet
// but this covers our tail until I do
Debug.WriteLine(String.Format("Dagnabbit, falling back to memory clone for {0}", cloneInput._name));
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private List<SearchResult> GetResults(string searchText)
{
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=diningsearchstorage;AccountKey=xeYMzXThFxrU7SsAMGbSWLdy9psLFRMk5NI8x0bx24xtEg9MPIstf/xwPdjvDm6HpHZaCPxxVFCv/7DDd5wymA==");
AzureDirectory azureDirectory = new AzureDirectory(storageAccount, "diningsearchindex");
IndexReader indexReader = IndexReader.Open(azureDirectory, true);
Searcher indexSearch = new IndexSearcher(indexReader);
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
var queryParser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "Dish", analyzer);
var query = queryParser.Parse(searchText);
Console.WriteLine("Searching for: " + query.ToString());
TopDocs resultDocs = indexSearch.Search(query, 100);
Console.WriteLine("Results Found: " + resultDocs.TotalHits);
List<SearchResult> searchResults = new List<SearchResult>();
var hits = resultDocs.ScoreDocs;
foreach (var hit in hits)
{
Document documentFromSearcher = indexSearch.Doc(hit.Doc);
SearchResult result = new SearchResult
{
CafeName = documentFromSearcher.Get("Cafe/Restaurant"),
CafeUrl = documentFromSearcher.Get("URL"),
DishName = documentFromSearcher.Get("Dish"),
Price = documentFromSearcher.Get("Price"),
Description = documentFromSearcher.Get("Description")
};
searchResults.Add(result);
}
return searchResults;
}
public invTypeIndexRepository()
{
cloudStorageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=eveapi;AccountKey=KrpLpd9DQkbKFC718eVzkkUdt5fafelr6W+Pyzu+kT+CrHSWEFBhlB4OmXMgVQ2iC+vYPTTNRbmbk8guUrKmDw==");
azureDirectory = new AzureDirectory(cloudStorageAccount, "invTypesIndex");
searcher = new IndexSearcher(azureDirectory);
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
string blobLengthMetadata;
bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata);
long blobLength = blob.Properties.Length;
if (hasMetadataValue) long.TryParse(blobLengthMetadata, out blobLength);
string blobLastModifiedMetadata;
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata)) {
if (long.TryParse(blobLastModifiedMetadata, out longLastModified))
blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
if (cachedLength != blobLength)
fFileNeeded = true;
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
var cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName), DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
fFileNeeded = true;
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
if (_azureDirectory.ShouldCompressFile(_name))
{
InflateStream(fileName);
}
else
{
using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName))
{
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
}
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
static void Main(string[] args)
{
// get settings from azure settings or app.config
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
try
{
configSetter(RoleEnvironment.GetConfigurationSettingValue(configName));
}
catch (Exception)
{
// for a console app, reading from App.config
configSetter(System.Configuration.ConfigurationManager.AppSettings[configName]);
}
});
// default AzureDirectory stores cache in local temp folder
AzureDirectory azureDirectory = new AzureDirectory(CloudStorageAccount.FromConfigurationSetting("blobStorage"), "TestCatalog6");
bool findexExists = IndexReader.IndexExists(azureDirectory);
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(), !IndexReader.IndexExists(azureDirectory));
}
catch (LockObtainFailedException)
{
Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again");
if (Console.ReadLine().ToLower().Trim() == "y" )
azureDirectory.ClearLock("write.lock");
}
};
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
indexWriter.SetRAMBufferSizeMB(10.0);
indexWriter.SetUseCompoundFile(false);
indexWriter.SetMaxMergeDocs(10000);
indexWriter.SetMergeFactor(100);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
if (iDoc % 10 == 0)
Console.WriteLine(iDoc);
Document doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.DocCount());
indexWriter.Close();
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(azureDirectory);
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
}
public AzureLockFactory(AzureDirectory directory, Directory cacheDirectory, string rootFolder)
{
_directory = directory;
_cacheDirectory = cacheDirectory;
_rootFolder = rootFolder;
}
public void ProcessRequest(HttpContext context)
{
Int64 expirationSeconds = 1;
HttpCacheability hc = HttpCacheability.NoCache;
try
{
string retVal = string.Empty;
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(new Utility().getConfigXML());
XmlNode xNode = xdoc.SelectSingleNode("//blobData[@name='default']");
string azureAccount = xNode.SelectSingleNode("Setting[@name='account']").Attributes["value"].Value;
string azureEndpoint = xNode.SelectSingleNode("Setting[@name='endpoint']").Attributes["value"].Value;
string azureSharedKey = xNode.SelectSingleNode("Setting[@name='accountSharedKey']").Attributes["value"].Value;
string blobStorage = azureEndpoint;
xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='luceneFragments']"));
string fragmentLocation = xNode.Attributes["value"].Value ;
string queryName = string.Empty;
SetCompression(context);
try
{
AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey");
azureResults ar = new azureResults();
// Get the page name and replace the .q extension with .xml
queryName = context.Request.Path;
queryName = queryName.Substring(queryName.LastIndexOf("/") + 1);
queryName = queryName.Substring(0, queryName.LastIndexOf(".")) + ".xml";
byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, "");
if (!ar.Succeeded)
{
context.Response.StatusCode = (int)ar.StatusCode;
}
else
{
xdoc = new XmlDocument();
System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
xdoc.LoadXml(enc.GetString(xmlFragment));
/*
* http://azure-architect.com/portals/16/MOOPData.xsd
*/
string azureContainerName = string.Empty;
azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneSearch/azureContainer[1]").InnerText;
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
AzureDirectory azDir = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
// IndexSearcher searcher = new IndexSearcher("azure-lucene-search");
try
{
var q = new BooleanQuery();
XmlNode xn = xdoc.SelectSingleNode("//luceneSearch[1]");
XmlNodeList xnl = xn.SelectNodes("//query");
Query[] qArray = new Query[xnl.Count];
bool hasReplaceableSearchValues = false;
bool hasReplacedAtLeastOneValue = false;
bool requirementsMet = true;
for (int i = 0; i < xnl.Count; i++)
{
XmlNode node = xnl[i];
string term = string.Empty;
term = node.Attributes["term"].Value;
string termValue = node.Attributes["termValue"].Value;
string variableValue = node.Attributes["variableValue"].Value;
string parmValue = string.Empty;
string requiredField = node.Attributes["required"].Value;
if (variableValue == "true") // See if there is a replacement attempt
{
if (requiredField == "true" && !context.Request.Params.AllKeys.Contains(termValue))
requirementsMet = false;
hasReplaceableSearchValues = true; // Set the flag to say we are attempting to replace the value
if (context.Request.Params.AllKeys.Contains(termValue)) // The parameter exists
{
hasReplacedAtLeastOneValue = true;
parmValue = context.Request.Params[termValue].Replace("+", " ").Replace("%20", " ");
}
}
if (node.Attributes["useAnalyzer"].Value == "true") // Should we use the analyzer
{
if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value
qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(parmValue);
if (variableValue != "true")
qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(termValue);
}
else // Using a Boolean
{
if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value
qArray[i] = new TermQuery(new Term(term, parmValue));
if (variableValue != "true")
qArray[i] = new TermQuery(new Term(term, termValue));
}
if (qArray[i] != null)
{
switch (node.Attributes["booleanClauseOccurs"].Value.ToLower())
{
case "must": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break;
case "must_not": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST_NOT)); break;
case "should": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.SHOULD)); break;
default: q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break;
}
}
}
IndexSearcher searcher2 = new IndexSearcher(azDir, true);
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true);
// If we have either no replaceable values or have replaceable values and at least one was provided
if (!hasReplaceableSearchValues || (hasReplaceableSearchValues && hasReplacedAtLeastOneValue))
if (requirementsMet)
searcher2.Search(q, collector);
int indexID = 1;
ScoreDoc[] hits = collector.TopDocs().ScoreDocs;
StringBuilder xmlOutput = new StringBuilder();
xmlOutput.AppendFormat("<?xml version=\"1.0\"?><root>");
for (int i = 0; i < hits.Length; ++i)
{
xmlOutput.AppendFormat("<hit>");
int docId = hits[i].Doc;
Document d = searcher2.Doc(docId);
xmlOutput.AppendFormat("<score>{0}</score><docID>{1}</docID>", hits[i].Score, indexID ++);
foreach (Field f in d.GetFields())
{
if (f.StringValue() == null)
xmlOutput.AppendFormat("<{0} />", f.Name());
else
xmlOutput.AppendFormat("<{0}>{1}</{0}>", f.Name(), System.Security.SecurityElement.Escape(f.StringValue()));
}
xmlOutput.AppendFormat("</hit>");
}
xmlOutput.AppendFormat("</root>");
retVal = xmlOutput.ToString();
}
catch (Exception ex)
{
retVal = "<root />";
}
MessageFormatOptions defaultOptions = new MessageFormatOptions();
string preferredContentType = string.Empty;
if (context.Request.HttpMethod == "GET" || context.Request.HttpMethod == "DELETE")
{
preferredContentType = (context.Request.AcceptTypes ?? new string[0])
.Select(m => m.Split(';')[0])
.FirstOrDefault(m => defaultOptions.MimeInputTypes.ContainsKey(m))
?? defaultOptions.DefaultContentType;
if (preferredContentType.Trim() == string.Empty)
preferredContentType = context.Request.Headers["Content-Type"];
}
else preferredContentType = context.Request.Headers["Content-Type"];
if (preferredContentType == "application/json")
{
context.Response.ContentType = preferredContentType;
XmlDocument doc = new XmlDocument();
doc.LoadXml(retVal);
retVal = JsonConvert.SerializeXmlNode(doc);
}
else
{
string luceneTransform = string.Empty;
luceneTransform = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").InnerText;
if (luceneTransform != string.Empty && luceneTransform != null)
{
xmlFragment = abs.GetBlob(fragmentLocation, luceneTransform, "", ref ar, "");
if (ar.Succeeded)
{
XsltUtil xslu = new XsltUtil();
retVal = XsltUtil.TransformXml(retVal, System.Text.ASCIIEncoding.ASCII.GetString(xmlFragment));
string transformContentType = "text/html";
try { transformContentType = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").Attributes["contentType"].Value; }
catch { }
context.Response.ContentType = transformContentType;
}
}
else
{
context.Response.ContentType = "text/xml";
}
}
}
}
catch (Exception ex)
{
FrameworkUtility u = new FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection"));
FrameworkUtility.ParsedURI pUri = new FrameworkUtility.ParsedURI(context.Request.RawUrl);
u.LogData(pUri.domainName, pUri.folderName, pUri.pageName, context.Request.HttpMethod, context.Request.UserHostAddress, "", "QueryError", ex.ToString(),
u.nvc2XML(context.Request.Headers));
//retVal = string.Format("<!-- {0} -->", ex.ToString());
context.Response.StatusDescription = "An error occured but it was logged for later review";
context.Response.StatusCode = 400;
}
finally { if (retVal == string.Empty) retVal = "<root />"; }
context.Response.Cache.SetExpires(DateTime.Now.AddSeconds(expirationSeconds));
context.Response.Cache.SetCacheability(hc);
context.Response.Write(retVal);
}
catch (Exception ex)
{
context.Response.StatusCode = 404;
context.Response.Close();
}
}
public AzureIndexInput(AzureDirectory azuredirectory, BlobClient blob, string resourceDescription) : base(resourceDescription)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
string blobLengthMetadata;
var properties = blob.GetProperties().Value;
bool hasMetadataValue = properties.Metadata.TryGetValue("CachedLength", out blobLengthMetadata);
long blobLength = properties.ContentLength;
if (hasMetadataValue)
{
long.TryParse(blobLengthMetadata, out blobLength);
}
string blobLastModifiedMetadata;
long longLastModified = 0;
DateTime blobLastModifiedUTC = properties.LastModified.UtcDateTime;
if (properties.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata))
{
if (long.TryParse(blobLastModifiedMetadata, out longLastModified))
{
blobLastModifiedUTC = DateTime.FromFileTimeUtc(longLastModified);
}
//blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// cachedLastModifiedUTC was not ouputting with a date (just time) and the time was always off
var filePath = Path.Combine(_azureDirectory.CatalogPath, fileName);
var lastModified = File.GetLastWriteTimeUtc(filePath);
//long unixDate = CacheDirectory.FileModified(fileName);
//DateTime start = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
//var cachedLastModifiedUTC = start.AddMilliseconds(unixDate).ToUniversalTime();
var cachedLastModifiedUTC = lastModified;
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
if (_azureDirectory.ShouldCompressFile(_name))
{
InflateStream(fileName);
}
else
{
using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName))
{
// get the blob
_blob.DownloadTo(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
}
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
protected override void Dispose(bool disposing)
{
_fileMutex.WaitOne();
try
{
#if FULLDEBUG
Debug.WriteLine(String.Format("CLOSED READSTREAM local {0}", _name));
#endif
_indexInput.Dispose();
_indexInput = null;
_azureDirectory = null;
_blobContainer = null;
_blob = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureLock(string lockFile, AzureDirectory directory)
{
_lockFile = lockFile;
_azureDirectory = directory;
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
string fileName = _name;
bool fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
long blobLength = blob.Properties.Length;
long.TryParse(blob.Metadata["CachedLength"], out blobLength);
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified))
{
// normalize RAMDirectory and FSDirectory times
if (longLastModified > ticks1970)
{
longLastModified -= ticks1970;
}
blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
var elapsed = CacheDirectory.FileModified(fileName);
// normalize RAMDirectory and FSDirectory times
if (elapsed > ticks1970)
{
elapsed -= ticks1970;
}
DateTime cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
#if COMPRESSBLOBS
if (_azureDirectory.ShouldCompressFile(_name))
{
// then we will get it fresh into local deflatedName
// StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
MemoryStream deflatedStream = new MemoryStream();
// get the deflated blob
_blob.DownloadToStream(deflatedStream);
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length));
// seek back to begininng
deflatedStream.Seek(0, SeekOrigin.Begin);
// open output file for uncompressed contents
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// create decompressor
DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress);
byte[] bytes = new byte[65535];
int nRead = 0;
do
{
nRead = decompressor.Read(bytes, 0, 65535);
if (nRead > 0)
{
fileStream.Write(bytes, 0, nRead);
}
} while (nRead == 65535);
decompressor.Close(); // this should close the deflatedFileStream too
fileStream.Close();
}
else
#endif
{
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
fileStream.Close();
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureIndexInput(AzureIndexInput cloneInput)
{
_fileMutex = new Mutex(false, cloneInput._name);
_fileMutex.WaitOne();
try
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Creating clone for {0}", cloneInput._name));
#endif
_azureDirectory = cloneInput._azureDirectory;
_blobContainer = cloneInput._blobContainer;
_blob = cloneInput._blob;
_indexInput = cloneInput._indexInput.Clone() as IndexInput;
}
catch (Exception)
{
// sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet
// but this covers our tail until I do
Debug.WriteLine(String.Format("Dagnabbit, falling back to memory clone for {0}", cloneInput._name));
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public DataTable MyFullTextSearcher(String myQuery)
{
DataTable dochits = new DataTable();
dochits.Columns.Add(new DataColumn("MyUserName", typeof(String)));
String result = CreateMyIndexDoc();
// Create the AzureDirectory for blob storage
try
{
dochits.Rows.Add(result);
String storageConnectionString =
"DefaultEndpointsProtocol=https;"
+ "AccountName=classifiedfilestorage;"
+ "AccountKey=P/GSa/P8vmBicBT45Jgv7pmRHdYWYfJeTFJ963Q1aEHlJDZBCzYkTBcjH1JoGgl+k34x9koBW4lsgYhCym0JLQ==";
CloudStorageAccount cloudAccount = CloudStorageAccount.Parse(storageConnectionString);
var cacheDirectory = new RAMDirectory();
AzureDirectory azureDirectory = new AzureDirectory(cloudAccount, "JobCat",cacheDirectory);
// Create the IndexSearcher
//IndexReader indexReader = DirectoryReader.Open(azureDirectory,true);
IndexSearcher indexSearcher = new IndexSearcher(azureDirectory);
// Create the QueryParser
//MoreLikeThisQuery mltq = new MoreLikeThisQuery(myQuery, new String[] { "title", "jobCategory" }, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30));
//MoreLikeThis mlt = new MoreLikeThis(indexReader);
//mlt.SetFieldNames(new String[] { "title", "jobCategory" });
//mlt.MinDocFreq = MoreLikeThis.DEFAULT_MIN_DOC_FREQ;
//mlt.MinTermFreq = MoreLikeThis.DEFAULT_MIN_TERM_FREQ;
//TextReader reader = new StringReader(myQuery);
//Query query = mlt.Like(reader);
MultiFieldQueryParser parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new String[] { "title", "jobCategory" }, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30));
// Create a query from the Parser
Query query = parser.Parse(myQuery+"*");
// Retrieve matching hits
var hits = indexSearcher.Search(query, 100);
// Loop through the matching hits, retrieving the document
for (int i = 0; i < hits.TotalHits; i++)
{
Document doc = indexSearcher.Doc(hits.ScoreDocs[i].Doc);
dochits.Rows.Add(doc.Get("jobID"));
}
// indexReader.Dispose();
}
catch (Exception e) {
result = "" + e;
dochits.Rows.Add(result);
}
return dochits;
}
public void processRequest(string queryName, NameValueCollection htKeys)
{
string retVal = string.Empty;
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(new Utility().getConfigXML());
XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']"));
string azureAccount = xNode.Attributes["account"].Value;
string azureEndpoint = xNode.Attributes["endpoint"].Value;
string azureSharedKey = xNode.Attributes["accountSharedKey"].Value;
string blobStorage = xNode.Attributes["endpoint"].Value;
xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']"));
string fragmentLocation = xNode.Attributes["value"].Value;
try
{
AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey");
azureResults ar = new azureResults();
// Get the page name and replace the .q extension with .xml
if (!queryName.ToLower().EndsWith(".xml"))
queryName += ".xml";
byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, "");
if (!ar.Succeeded)
{
NotifyError(new Exception(ar.StatusCode.ToString()));
}
else
{
xdoc = new XmlDocument();
System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
xdoc.LoadXml(enc.GetString(xmlFragment));
/*
* http://azure-architect.com/portals/16/MOOPData.xsd
*/
XmlNode xn = xdoc.SelectSingleNode("//storedProcedure[1]");
string storedProcedureName = xn.Attributes["procedureName"].Value;
string connectionStringName = xn.Attributes["connectionName"].Value;
SqlCommand cmd = new SqlCommand(storedProcedureName, new SqlConnection(new Utility().ResolveDataConnection(connectionStringName)));
cmd.CommandType = CommandType.StoredProcedure;
XmlNodeList xnl = xdoc.SelectNodes("/MOOPData/luceneData/field");
Field.Store[] fieldStore = new Field.Store[xnl.Count];
Field.Index[] indexType = new Field.Index[xnl.Count];
string[] luceneName = new string[xnl.Count];
string[] dataName = new string[xnl.Count];
bool[] isIncludedInOlioSearchFlag = new bool[xnl.Count];
bool[] isKeyFieldFlag = new bool[xnl.Count];
string olioSearchFieldName = string.Empty;
string azureContainerName = string.Empty;
olioSearchFieldName = xdoc.SelectSingleNode("//MOOPData/luceneData/olioSearchFieldName[1]").InnerText;
azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneData/azureContainer[1]").InnerText;
for (int i = 0; i < xnl.Count; i++)
{
XmlNode node = xnl[i];
switch (node.Attributes["store"].Value.ToLower())
{
case "compress": fieldStore[i] = Field.Store.COMPRESS; break;
case "no": fieldStore[i] = Field.Store.NO; break;
case "yes": fieldStore[i] = Field.Store.YES; break;
default: fieldStore[i] = Field.Store.NO; break;
}
switch (node.Attributes["index"].Value.ToLower())
{
case "analyzed": indexType[i] = Field.Index.ANALYZED; break;
case "analyzed_no_norms": indexType[i] = Field.Index.ANALYZED_NO_NORMS; break;
case "no": indexType[i] = Field.Index.NO; break;
case "no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break;
case "not_analyzed": indexType[i] = Field.Index.NOT_ANALYZED; break;
case "not_analyzed_no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break;
case "tokenized": indexType[i] = Field.Index.ANALYZED; break;
case "un_tokenized": indexType[i] = Field.Index.NOT_ANALYZED; break;
default: indexType[i] = Field.Index.NO; break;
}
dataName[i] = node.Attributes["dataName"].Value;
luceneName[i] = node.Attributes["luceneName"].Value;
isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true";
isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true";
isIncludedInOlioSearchFlag[i] = node.Attributes["isIncludedInOlioSearch"].Value == "true";
}
xnl = xdoc.SelectNodes("//parameter");
foreach (XmlNode node in xnl)
{
string parameterName = node.Attributes["parameterName"].Value;
string urlParameterName = node.Attributes["urlParameterName"].Value;
string dataType = node.Attributes["dataType"].Value;
string dataLength = node.Attributes["dataLength"].Value;
string defaultValue = node.Attributes["defaultValue"].Value;
if (!parameterName.StartsWith("@"))
parameterName = "@" + parameterName;
SqlParameter sp = new SqlParameter();
sp.ParameterName = parameterName;
switch (dataType)
{
case "bigint": sp.SqlDbType = SqlDbType.BigInt; break;
case "binary": sp.SqlDbType = SqlDbType.Binary; break;
case "bit": sp.SqlDbType = SqlDbType.Bit; break;
case "char": sp.SqlDbType = SqlDbType.Char; break;
case "date": sp.SqlDbType = SqlDbType.Date; break;
case "datetime": sp.SqlDbType = SqlDbType.DateTime; break;
case "datetime2": sp.SqlDbType = SqlDbType.DateTime2; break;
case "datetimeoffset": sp.SqlDbType = SqlDbType.DateTimeOffset; break;
case "decimal": sp.SqlDbType = SqlDbType.Decimal; break;
case "float": sp.SqlDbType = SqlDbType.Float; break;
case "geography": sp.SqlDbType = SqlDbType.Structured; break;
case "geometry": sp.SqlDbType = SqlDbType.Structured; break;
case "hierarchyid": sp.SqlDbType = SqlDbType.Structured; break;
case "image": sp.SqlDbType = SqlDbType.Image; break;
case "int": sp.SqlDbType = SqlDbType.Int; break;
case "money": sp.SqlDbType = SqlDbType.Money; break;
case "nchar": sp.SqlDbType = SqlDbType.NChar; break;
case "ntext": sp.SqlDbType = SqlDbType.NText; break;
case "nvarchar": sp.SqlDbType = SqlDbType.NVarChar; break;
case "real": sp.SqlDbType = SqlDbType.Real; break;
case "smalldatetime": sp.SqlDbType = SqlDbType.SmallDateTime; break;
case "smallint": sp.SqlDbType = SqlDbType.SmallInt; break;
case "smallmoney": sp.SqlDbType = SqlDbType.SmallMoney; break;
case "sql_variant": sp.SqlDbType = SqlDbType.Variant; break;
case "text": sp.SqlDbType = SqlDbType.Text; break;
case "time": sp.SqlDbType = SqlDbType.Time; break;
case "timestamp": sp.SqlDbType = SqlDbType.Timestamp; break;
case "tinyint": sp.SqlDbType = SqlDbType.TinyInt; break;
case "uniqueidentifier": sp.SqlDbType = SqlDbType.UniqueIdentifier; break;
case "varbinary": sp.SqlDbType = SqlDbType.VarBinary; break;
case "varchar": sp.SqlDbType = SqlDbType.VarChar; break;
case "xml": sp.SqlDbType = SqlDbType.Xml; break;
default: sp.SqlDbType = SqlDbType.Variant; break;
}
switch (urlParameterName.ToLower())
{
case "ipaddress": sp.Value = "127.0.0.1"; break;
case "domainname": sp.Value = ""; break;
default: if (htKeys[urlParameterName] != null)
sp.Value = htKeys[urlParameterName];
else
sp.Value = (defaultValue.ToLower() == "dbnull" ? DBNull.Value
: (object)defaultValue);
break;
}
cmd.Parameters.Add(sp);
}
cmd.Connection.Open();
SqlDataReader dr = cmd.ExecuteReader();
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
bool findexExists = false;
try
{
findexExists = IndexReader.IndexExists(azureDirectory);
if ((findexExists) && IndexWriter.IsLocked(azureDirectory))
azureDirectory.ClearLock("write.lock");
}
catch (Exception e)
{
Trace.WriteLine(e.ToString());
return;
}
IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024));
idxW.SetRAMBufferSizeMB(10.0);
idxW.SetUseCompoundFile(false);
idxW.SetMaxMergeDocs(10000);
idxW.SetMergeFactor(100);
while (dr.Read())
{
StringBuilder olioSearch = new StringBuilder();
Document doc = new Document();
for (int i = 0; i <= dataName.GetUpperBound(0); i++)
{
if (isKeyFieldFlag[i])
{
NotifyCaller(string.Format("Processing {0}", dr[dataName[i]].ToString().ToLower()));
idxW.DeleteDocuments(new Term(luceneName[i], dr[dataName[i]].ToString().ToLower()));
doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString().ToLower(), Field.Store.YES, Field.Index.NOT_ANALYZED));
}
else
try
{
doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString(), fieldStore[i], indexType[i]));
if (isIncludedInOlioSearchFlag[i])
olioSearch.AppendFormat("\r\n{0}", dr[dataName[i]].ToString());
}
catch (Exception ex)
{
NotifyError(ex);
}
}
if (olioSearch.ToString() != string.Empty && olioSearchFieldName != string.Empty)
doc.Add(new Field(olioSearchFieldName, olioSearch.ToString(), Field.Store.NO, Field.Index.ANALYZED));
idxW.AddDocument(doc);
}
idxW.Commit();
idxW.Close();
}
}
catch (Exception ex)
{
MOOPFramework.FrameworkUtility u = new MOOPFramework.FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection"));
u.LogData("localhost", "quoteSearchLoader", "testing", string.Empty, string.Empty, "", "QueryError", ex.ToString(),
u.nvc2XML(htKeys));
//retVal = string.Format("<!-- {0} -->", ex.ToString());
NotifyError(new Exception("An error occured but it was logged for later review"));
}
finally { if (retVal == string.Empty) retVal = "<root />"; }
}
static void Main(string[] args)
{
// get settings from azure settings or app.config
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
try
{
configSetter(RoleEnvironment.GetConfigurationSettingValue(configName));
}
catch (Exception)
{
// for a console app, reading from App.config
configSetter(System.Configuration.ConfigurationManager.AppSettings[configName]);
}
});
// default AzureDirectory stores cache in local temp folder
AzureDirectory azureDirectory = new AzureDirectory(CloudStorageAccount.FromConfigurationSetting("blobStorage"), "TestCatalog");
bool findexExists = false;
try
{
findexExists = IndexReader.IndexExists(azureDirectory);
if ((findexExists) && IndexReader.IsLocked(azureDirectory))
azureDirectory.ClearLock("write.lock");
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
return;
}
IndexWriter indexWriter = new IndexWriter(azureDirectory, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), !findexExists);
indexWriter.SetRAMBufferSizeMB(10.0);
indexWriter.SetUseCompoundFile(false);
indexWriter.SetMaxMergeDocs(10000);
indexWriter.SetMergeFactor(100);
fExit = true;
for (int iDoc = 0; iDoc < 100; iDoc++)
{
if (fExit)
break;
if (iDoc % 10 == 0)
Console.WriteLine(iDoc);
Document doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.DocCount());
indexWriter.Close();
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(azureDirectory); // IndexReader.Open(
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) : base("Azure" + azuredirectory.ToString())
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
string blobLengthMetadata;
bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata);
long blobLength = blob.Properties.Length;
if (hasMetadataValue)
{
long.TryParse(blobLengthMetadata, out blobLength);
}
string blobLastModifiedMetadata;
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata))
{
if (long.TryParse(blobLastModifiedMetadata, out longLastModified))
{
blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
}
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// cachedLastModifiedUTC was not ouputting with a date (just time) and the time was always off
long unixDate = _azureDirectory.FileModified(fileName);
// DateTime start = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var cachedLastModifiedUTC = new DateTime(unixDate).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
if (_azureDirectory.ShouldCompressFile(_name))
{
InflateStream(fileName);
}
else
{
using (var fileStream = _azureDirectory.CreateOutput(fileName, IOContext.DEFAULT))
{
// get the blob
_blob.FetchAttributes();
long fileByteLength = _blob.Properties.Length;
byte[] fileContent = new byte[fileByteLength];
for (int i = 0; i < fileByteLength; i++)
{
fileContent[i] = 0x20;
}
_blob.DownloadToByteArray(fileContent, 0);
fileStream.WriteBytes(fileContent, (int)fileByteLength);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
}
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
string blobLengthMetadata;
bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata);
long blobLength = blob.Properties.Length;
if (hasMetadataValue)
{
long.TryParse(blobLengthMetadata, out blobLength);
}
string blobLastModifiedMetadata;
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata))
{
if (long.TryParse(blobLastModifiedMetadata, out longLastModified))
{
blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
}
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
var cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName), DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
if (_azureDirectory.ShouldCompressFile(_name))
{
InflateStream(fileName);
}
else
{
using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName))
{
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
}
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public override void close()
{
_fileMutex.WaitOne();
try
{
_indexInput.close();
_indexInput = null;
_azureDirectory = null;
_blobContainer = null;
_blob = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureLock(string lockFile, AzureDirectory directory)
{
_lockFile = lockFile;
_azureDirectory = directory;
}
static void Main()
{
// default AzureDirectory stores cache in local temp folder
CloudStorageAccount cloudStorageAccount;
CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("blobStorage"), out cloudStorageAccount);
//AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", new RAMDirectory());
//AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", FSDirectory.Open(@"c:\test"));
var azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest" /* default is FSDirectory.Open(@"%temp%/AzureDirectory/TestTest"); */ );
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
var config = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_CURRENT, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT));
indexWriter = new IndexWriter(azureDirectory, config);
}
catch (LockObtainFailedException)
{
Console.WriteLine("Lock is taken, waiting for timeout...");
Thread.Sleep(1000);
}
}
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
//indexWriter.setRAMBufferSizeMB(10.0);
//indexWriter.SetUseCompoundFile(false);
//indexWriter.SetMaxMergeDocs(10000);
//indexWriter.SetMergeFactor(100);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
if (iDoc % 10 == 0)
Console.WriteLine(iDoc);
var doc = new Document();
doc.add(new TextField("id", DateTime.Now.ToFileTimeUtc().ToString(CultureInfo.InvariantCulture), Field.Store.YES));
doc.add(new TextField("Title", GeneratePhrase(10), Field.Store.YES));
doc.add(new TextField("Body", GeneratePhrase(40), Field.Store.YES));
indexWriter.addDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.numDocs());
Console.Write("Flushing and disposing writer...");
// Potentially Expensive: this ensures that all writes are commited to blob storage
indexWriter.commit();
indexWriter.close();
Console.WriteLine("done");
Console.WriteLine("Hit Key to search again");
Console.ReadKey();
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(DirectoryReader.open(azureDirectory));
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, Random.Next(32768).ToString(CultureInfo.InvariantCulture));
SearchForPhrase(searcher, Random.Next(32768).ToString(CultureInfo.InvariantCulture));
Console.WriteLine("Hit a key to dispose and exit");
Console.ReadKey();
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
string fileName = _name;
bool fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
long blobLength = blob.Properties.Length;
long.TryParse(blob.Metadata["CachedLength"], out blobLength);
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
//if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified))
// blobLastModifiedUTC = new DateTime(longLastModified, DateTimeKind.Local).ToUniversalTime();
if (cachedLength != blobLength)
fFileNeeded = true;
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
DateTime cachedLastModifiedUTC = DateTime.MinValue;
if (CacheDirectory is RAMDirectory) {
cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Local).ToUniversalTime();
}
else if (CacheDirectory is FSDirectory) {
cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Utc).ToUniversalTime().AddYears(1969);
}
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
fFileNeeded = true;
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
#if COMPRESSBLOBS
if (_azureDirectory.ShouldCompressFile(_name))
{
// then we will get it fresh into local deflatedName
// StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
MemoryStream deflatedStream = new MemoryStream();
// get the deflated blob
_blob.DownloadToStream(deflatedStream);
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length));
// seek back to begininng
deflatedStream.Seek(0, SeekOrigin.Begin);
// open output file for uncompressed contents
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// create decompressor
DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress);
byte[] bytes = new byte[65535];
int nRead = 0;
do
{
nRead = decompressor.Read(bytes, 0, 65535);
if (nRead > 0)
fileStream.Write(bytes, 0, nRead);
} while (nRead == 65535);
decompressor.Close(); // this should close the deflatedFileStream too
fileStream.Close();
}
else
#endif
{
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
fileStream.Close();
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public void optimizeIndex(string azureContainerName)
{
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(new Utility().getConfigXML());
XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']"));
string azureAccount = xNode.Attributes["account"].Value;
string azureEndpoint = xNode.Attributes["endpoint"].Value;
string azureSharedKey = xNode.Attributes["accountSharedKey"].Value;
string blobStorage = xNode.Attributes["endpoint"].Value;
xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']"));
string fragmentLocation = xNode.Attributes["value"].Value;
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey);
Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false);
AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory());
bool findexExists = false;
try
{
findexExists = IndexReader.IndexExists(azureDirectory);
if ((findexExists) && IndexWriter.IsLocked(azureDirectory))
azureDirectory.ClearLock("write.lock");
}
catch (Exception e)
{
Trace.WriteLine(e.ToString());
return;
}
IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024));
idxW.SetRAMBufferSizeMB(10.0);
idxW.SetUseCompoundFile(false);
idxW.SetMaxMergeDocs(10000);
idxW.SetMergeFactor(100);
idxW.Optimize();
}
// Saten
public String CreateMyIndexDoc()
{
String status = "";
try
{
if (dbConnection.State.ToString() == "Closed")
{
dbConnection.Open();
}
String storageConnectionString =
"DefaultEndpointsProtocol=https;"
+ "AccountName=classifiedfilestorage;"
+ "AccountKey=P/GSa/P8vmBicBT45Jgv7pmRHdYWYfJeTFJ963Q1aEHlJDZBCzYkTBcjH1JoGgl+k34x9koBW4lsgYhCym0JLQ==";
CloudStorageAccount cloudAccount = CloudStorageAccount.Parse(storageConnectionString);
var cacheDirectory = new RAMDirectory();
AzureDirectory azureDirectory = new AzureDirectory(cloudAccount, "JobCat",cacheDirectory);
bool findexExists = IndexReader.IndexExists(azureDirectory);
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),findexExists, new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
}
catch (LockObtainFailedException)
{
Thread.Sleep(1000);
}
};
indexWriter.SetRAMBufferSizeMB(10.0);
String query = "SELECT jobID,title,jobCategory from jobs";
SqlCommand command = new SqlCommand(query, dbConnection);
SqlDataReader reader = command.ExecuteReader();
if (reader.HasRows) {
while (reader.Read())
{
Document docx = new Document();
docx.Add(new Field("jobID", reader["jobID"].ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
docx.Add(new Field("title", reader["title"].ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
docx.Add(new Field("jobCategory", reader["jobCategory"].ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
indexWriter.AddDocument(docx);
}
}
indexWriter.Close();
dbConnection.Close();
status = "Success";
}
catch (Exception e) {
status = "" + e;
}
return status;
}
private IndexSearcher CreateIndexSearcher()
{
CloudStorageAccount cloudStorageAccount;
CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("blobStorage"), out cloudStorageAccount);
var cacheDirectory = new RAMDirectory();
var azureDirectory = new AzureDirectory(cloudStorageAccount, "FarmCatalog", cacheDirectory);
return new IndexSearcher(azureDirectory, true);
}
