public void TestReadAndWrite()
{
var connectionString = _connectionString ?? "UseDevelopmentStorage=true";
var cloudStorageAccount = CloudStorageAccount.Parse(connectionString);
const string containerName = "testcatalog";
var blobClient = cloudStorageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(containerName);
container.DeleteIfExists();
var azureDirectory = new AzureDirectory(cloudStorageAccount, containerName);
var indexWriterConfig = new IndexWriterConfig(
Lucene.Net.Util.LuceneVersion.LUCENE_48,
new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48));
int dog = 0, cat = 0, car = 0;
using (var indexWriter = new IndexWriter(azureDirectory, indexWriterConfig))
{
for (var iDoc = 0; iDoc < 1000; iDoc++)
{
var bodyText = GeneratePhrase(40);
var doc = new Document {
new TextField("id", DateTime.Now.ToFileTimeUtc() + "-" + iDoc, Field.Store.YES),
new TextField("Title", GeneratePhrase(10), Field.Store.YES),
new TextField("Body", bodyText, Field.Store.YES)
};
dog += bodyText.Contains(" dog ") ? 1 : 0;
cat += bodyText.Contains(" cat ") ? 1 : 0;
car += bodyText.Contains(" car ") ? 1 : 0;
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}, {1} dog, {2} cat, {3} car", indexWriter.NumDocs, dog, cat, car);
}
try
{
var ireader = DirectoryReader.Open(azureDirectory);
for (var i = 0; i < 100; i++)
{
var searcher = new IndexSearcher(ireader);
var searchForPhrase = SearchForPhrase(searcher, "dog");
Assert.AreEqual(dog, searchForPhrase);
searchForPhrase = SearchForPhrase(searcher, "cat");
Assert.AreEqual(cat, searchForPhrase);
searchForPhrase = SearchForPhrase(searcher, "car");
Assert.AreEqual(car, searchForPhrase);
}
Console.WriteLine("Tests passsed");
}
catch (Exception x)
{
Console.WriteLine("Tests failed:\n{0}", x);
}
finally
{
// check the container exists, and delete it
Assert.IsTrue(container.Exists()); // check the container exists
container.Delete();
}
}
static void Main()
{
// default AzureDirectory stores cache in local temp folder
CloudStorageAccount cloudStorageAccount;
CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("blobStorage"), out cloudStorageAccount);
//AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", new RAMDirectory());
//AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", FSDirectory.Open(@"c:\test"));
var azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest" /* default is FSDirectory.Open(@"%temp%/AzureDirectory/TestTest"); */);
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
var config = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_CURRENT, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT));
indexWriter = new IndexWriter(azureDirectory, config);
}
catch (LockObtainFailedException)
{
Console.WriteLine("Lock is taken, waiting for timeout...");
Thread.Sleep(1000);
}
}
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
//indexWriter.setRAMBufferSizeMB(10.0);
//indexWriter.SetUseCompoundFile(false);
//indexWriter.SetMaxMergeDocs(10000);
//indexWriter.SetMergeFactor(100);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
if (iDoc % 10 == 0)
{
Console.WriteLine(iDoc);
}
var doc = new Document();
doc.add(new TextField("id", DateTime.Now.ToFileTimeUtc().ToString(CultureInfo.InvariantCulture), Field.Store.YES));
doc.add(new TextField("Title", GeneratePhrase(10), Field.Store.YES));
doc.add(new TextField("Body", GeneratePhrase(40), Field.Store.YES));
indexWriter.addDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.numDocs());
Console.Write("Flushing and disposing writer...");
// Potentially Expensive: this ensures that all writes are commited to blob storage
indexWriter.commit();
indexWriter.close();
Console.WriteLine("done");
Console.WriteLine("Hit Key to search again");
Console.ReadKey();
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(DirectoryReader.open(azureDirectory));
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, Random.Next(32768).ToString(CultureInfo.InvariantCulture));
SearchForPhrase(searcher, Random.Next(32768).ToString(CultureInfo.InvariantCulture));
Console.WriteLine("Hit a key to dispose and exit");
Console.ReadKey();
}
static void Main(string[] args)
{
// get settings from azure settings or app.config
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
try
{
configSetter(RoleEnvironment.GetConfigurationSettingValue(configName));
}
catch (Exception)
{
// for a console app, reading from App.config
configSetter(System.Configuration.ConfigurationManager.AppSettings[configName]);
}
});
// default AzureDirectory stores cache in local temp folder
AzureDirectory azureDirectory = new AzureDirectory(CloudStorageAccount.FromConfigurationSetting("blobStorage"), "TestCatalog6");
bool findexExists = IndexReader.IndexExists(azureDirectory);
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(), !IndexReader.IndexExists(azureDirectory));
}
catch (LockObtainFailedException)
{
Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again");
if (Console.ReadLine().ToLower().Trim() == "y")
{
azureDirectory.ClearLock("write.lock");
}
}
}
;
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
indexWriter.SetRAMBufferSizeMB(10.0);
indexWriter.SetUseCompoundFile(false);
indexWriter.SetMaxMergeDocs(10000);
indexWriter.SetMergeFactor(100);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
if (iDoc % 10 == 0)
{
Console.WriteLine(iDoc);
}
Document doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.DocCount());
indexWriter.Close();
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(azureDirectory);
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
}
static void Main(string[] args)
{
// default AzureDirectory stores cache in local temp folder
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.DevelopmentStorageAccount;
CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("blobStorage"), out cloudStorageAccount);
//AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", new RAMDirectory());
//AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", FSDirectory.Open(@"c:\test"));
AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest" /* default is FSDirectory.Open(@"%temp%/AzureDirectory/TestTest"); */);
bool findexExists = IndexReader.IndexExists(azureDirectory);
IndexSearcher searcher;
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(azureDirectory);
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
Console.WriteLine("Hit a key to add 10000 docs");
Console.ReadKey();
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try
{
indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), !IndexReader.IndexExists(azureDirectory), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
}
catch (LockObtainFailedException)
{
Console.WriteLine("Lock is taken, waiting for timeout...");
Thread.Sleep(1000);
}
}
;
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
indexWriter.SetRAMBufferSizeMB(10.0);
//indexWriter.SetUseCompoundFile(false);
//indexWriter.SetMaxMergeDocs(10000);
//indexWriter.SetMergeFactor(100);
for (int iDoc = 0; iDoc < 10000; iDoc++)
{
if (iDoc % 10 == 0)
{
Console.WriteLine(iDoc);
}
Document doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.NumDocs());
Console.WriteLine("done");
Console.WriteLine("Hit Key to search again");
Console.ReadKey();
using (new AutoStopWatch("Creating searcher"))
{
searcher = new IndexSearcher(azureDirectory);
}
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
Console.WriteLine("Hit a key to dispose and exit");
Console.ReadKey();
Console.Write("Flushing and disposing writer...");
// Potentially Expensive: this ensures that all writes are commited to blob storage
indexWriter.Flush(true, true, true);
indexWriter.Dispose();
}
public async Task <ActionResult> Complete(string code, string error, string error_description, string resource, string state)
{
using (var db = new TimesheetContext())
{
if (!String.IsNullOrEmpty(error) || !String.IsNullOrEmpty(error_description))
{
return(View("RegistrationError", new RegistrationErrorModel()
{
Error = error, ErrorDescription = error_description
}));
}
var registrationRequest = await db.RegistrationRequests.FirstOrDefaultAsync(r => r.SignupToken == state);
if (registrationRequest == null)
{
return(View("RegistrationRequestUnknown"));
}
// Get the user's profile from Azure AD.
var credential = new ClientCredential(ConfigurationManager.AppSettings["AzureAD:ClientID"],
ConfigurationManager.AppSettings["AzureAD:Key"]);
var authContext = new AuthenticationContext("https://login.windows.net/common/");
var result = authContext.AcquireTokenByAuthorizationCode(code, new Uri(Request.Url.GetLeftPart(UriPartial.Path)), credential);
// Clean up the registration request.
db.RegistrationRequests.Remove(registrationRequest);
// Prevent duplicate users.
var userExists = await db.Users.AnyAsync(u => u.AzureTenantId == result.TenantId && u.AzureUpn == result.UserInfo.DisplayableId);
if (!userExists)
{
// Create the user.
var user = new User();
user.CreatedOn = DateTime.UtcNow;
user.Id = Guid.NewGuid();
user.Firstname = result.UserInfo.GivenName;
user.Lastname = result.UserInfo.FamilyName;
user.AzureTenantId = result.TenantId;
user.AzureUpn = result.UserInfo.DisplayableId;
db.Users.Add(user);
}
// Consent happend by administrator for the whole organization..
if (registrationRequest.AdminConsented)
{
// Prevent duplicate tenants.
var directoryExists = await db.AzureDirectories.AnyAsync(u => u.TenantId == result.TenantId);
if (!directoryExists)
{
// Create the tenant.
var tenant = new AzureDirectory();
tenant.CreatedOn = DateTime.UtcNow;
tenant.Id = Guid.NewGuid();
tenant.Issuer = String.Format("https://sts.windows.net/{0}/", result.TenantId);
tenant.TenantId = result.TenantId;
tenant.Name = registrationRequest.OrganizationName;
db.AzureDirectories.Add(tenant);
}
// Save.
await db.SaveChangesAsync();
// Show confirmation page.
return(View("TenantRegistrationSuccess"));
}
// Save.
await db.SaveChangesAsync();
// Show user confirmation page.
return(View("UserRegistrationSuccess"));
}
}
private void UpdateSearchIndex(IEnumerable <TagEntity> tags, IEnumerable <RedirectEntity> redirects)
{
var storage = this.TagBlobContainer.StorageAccount();
#if false
var search = "wix";
using (var azureDirectory = new AzureDirectory(storage, StorageName.SearchIndexBlobContainer))
using (var indexReader = IndexReader.Open(azureDirectory, true))
using (var searcher = new IndexSearcher(indexReader))
{
var query = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, _fields, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)).Parse(search);
var results = searcher.Search(query, 50);
foreach (var scoreDoc in results.ScoreDocs)
{
Console.WriteLine("Tag #{0}", scoreDoc.Doc);
var d = searcher.Doc(scoreDoc.Doc);
}
}
#endif
var workingFolder = Path.Combine(Path.GetTempPath(), typeof(IndexTagsCommand).ToString());
using (var workingDirectory = FSDirectory.Open(workingFolder))
using (var azureDirectory = new AzureDirectory(storage, StorageName.SearchIndexBlobContainer, workingDirectory))
using (var indexWriter = CreateIndexWriter(azureDirectory))
{
foreach (var tag in tags)
{
if (tag.Primary)
{
var document = CreateDocumentForPrimaryTag(tag);
indexWriter.AddDocument(document);
}
else
{
var document = CreateDocumentForHistory(tag);
indexWriter.AddDocument(document);
}
}
// Add fake data to bulk up the index.
#if FAKE_DATA
var s = "The Adobe Flash Player is freeware software for viewing multimedia, executing Rich Internet Applications, and streaming video and audio, content created on the Adobe Flash platform." +
"Chrome is a fast, simple, and secure web browser, built for the modern web." +
"Git (for Windows) Git is a powerful distributed Source Code Management tool. If you just want to use Git to do your version control in Windows, you will need to download Git for Windows, run the installer, and you are ready to start.\r\n\r\nNote: Git for Windows is a project run by volunteers, so if you want it to improve, volunteer!" +
"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum." +
"It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)." +
"Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections de Finibus Bonorum et Malorum (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum Lorem ipsum dolor sit amet comes from a line in section" +
"The standard chunk of Lorem Ipsum used since the is reproduced below for those interested. Sections and from de Finibus Bonorum et Malorum by Cicero are also reproduced in their exact original form, accompanied by English versions from the 1914 translation by H. Rackham.";
var words = s.Split(' ');
var rnd = new Random();
for (int i = 1; i < 3001; ++i)
{
var name = Generate(rnd, words, 2, 5);
var alias = AzureUris.AzureSafeId(name);
var tagAzid = alias.ToLowerInvariant();
var description = Generate(rnd, words, 60, 120);
var keywords = Generate(rnd, words, 1, 6);
string version = String.Format("{0}.{1}.{2}", 10000 % i, 4000 % i, 3005 % i);
var downloads = rnd.Next(0, 2000000);
var imageUrl = String.Format("http://www.example.com/imgs/{0}.jpg", words[rnd.Next(words.Length)]);
var updated = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
//updated = DateTime.UtcNow.ToString("o");
var url = (i % 2) == 1 ? "https://appsyndication.blob.core.windows.net/tags/sources/https-github-com-appsyndication-test-tree-master/http-wixtoolset-org-releases-wix4/v4.0.2220.0.json.swidtag" :
"https://appsyndication.blob.core.windows.net/tags/sources/https-github-com-appsyndication-test-tree-master/https-github-com-robmen-gitsetup/v1.9.4.40929.json.swidtag";
var document = new Document();
document.Add(new Field("id", tagAzid, Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("alias", alias, Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("name", name, Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("description", description, Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("keywords", keywords, Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("tagSource", String.Empty, Field.Store.YES, Field.Index.NO));
document.Add(new Field("version", version, Field.Store.YES, Field.Index.NO));
document.Add(new Field("imageUrl", imageUrl, Field.Store.YES, Field.Index.NO));
document.Add(new Field("downloads", downloads.ToString(), Field.Store.YES, Field.Index.NO));
document.Add(new Field("tagUrl", url, Field.Store.YES, Field.Index.NO));
document.Add(new Field("updated", updated, Field.Store.YES, Field.Index.NO));
indexWriter.AddDocument(document);
}
Console.WriteLine("Total docs is {0}", indexWriter.NumDocs());
#endif
foreach (var redirect in redirects)
{
var document = CreateDocumentForRedirect(redirect);
indexWriter.AddDocument(document);
}
indexWriter.Optimize(true);
}
}
