public override void ProcessMessage(BasicDeliverEventArgs message)
{
try
{
iCount++;
var pt =
Newtonsoft.Json.JsonConvert.DeserializeObject <HistoryProductSorl>(
Encoding.UTF8.GetString(message.Body));
if (pt != null)
{
_lstCacheList.Add(pt);
if (_lstCacheList.Count > 2000)
{
_solr.AddRange(_lstCacheList);
_lstCacheList.Clear();
_solr.Commit();
_log.Info("Added 1000 item");
}
}
}
catch (Exception ex)
{
_log.Error(ex + Encoding.UTF8.GetString(message.Body));
}
this.GetChannel().BasicAck(message.DeliveryTag, true);
}
public void Write(IEnumerable <IRow> rows)
{
var fullCount = 0;
var batchCount = (uint)0;
foreach (var part in rows.Partition(_context.Entity.InsertSize))
{
var docs = new List <Dictionary <string, object> >();
foreach (var row in part)
{
batchCount++;
fullCount++;
docs.Add(_fields.ToDictionary(field => field.Alias.ToLower(), field => row[field]));
}
var response = _solr.AddRange(docs);
_context.Increment(@by: batchCount);
if (response.Status == 0)
{
_context.Debug(() => $"{batchCount} to output");
}
else
{
_context.Error("ah!");
}
batchCount = 0;
}
_solr.Commit();
if (fullCount > 0)
{
_context.Info($"{fullCount} to output");
}
}
public void InsertHashDataForTrack(IEnumerable <HashedFingerprint> hashes, IModelReference trackReference)
{
var dtos = hashes.Select(hash => GetSubFingerprintDto(trackReference, hash))
.ToList();
solr.AddRange(dtos);
solr.Commit();
}
public async Task AddManyAsync(IEnumerable <CandidatoDocumento> documentos)
{
try
{
_solr.AddRange(documentos);
}
catch (Exception ex)
{
Debug.Write(ex.Message);
}
await _solr.CommitAsync();
}
public void Write(IEnumerable <IRow> rows)
{
_originalConnectionLimit = ServicePointManager.DefaultConnectionLimit;
ServicePointManager.DefaultConnectionLimit = _context.Connection.MaxDegreeOfParallelism * 2;
try {
Parallel.ForEach(rows.Partition(_context.Entity.InsertSize), _options, part => {
var docs = new List <Dictionary <string, object> >();
foreach (var row in part)
{
Interlocked.Increment(ref _count);
docs.Add(_fields.ToDictionary(field => field.Alias.ToLower(), field => row[field]));
}
var response = _solr.AddRange(docs);
if (response.Status != 0)
{
_context.Error($"Couldn't add range of {docs.Count} document{docs.Count.Plural()} to SOLR.");
}
});
} catch (AggregateException ex) {
foreach (var exception in ex.InnerExceptions)
{
_context.Error(exception.Message);
_context.Error(exception.StackTrace);
}
return;
}
ServicePointManager.DefaultConnectionLimit = _originalConnectionLimit;
if (_count > 0)
{
try {
var commit = _solr.Commit();
if (commit.Status == 0)
{
_context.Entity.Inserts += Convert.ToUInt32(_count);
_context.Info($"Committed {_count} documents in {TimeSpan.FromMilliseconds(commit.QTime)}");
}
else
{
_context.Error($"Failed to commit {_count} documents. SOLR returned status {commit.Status}.");
}
} catch (SolrNetException ex) {
_context.Error($"Failed to commit {_count} documents. {ex.Message}");
}
}
}
public void Write(IEnumerable <IRow> rows)
{
var fullCount = 0;
foreach (var part in rows.Partition(_context.Entity.InsertSize))
{
var batchCount = 0;
foreach (var row in part)
{
var doc = _documents[batchCount];
foreach (var field in _fields)
{
doc[field.Alias] = row[field];
}
batchCount++;
fullCount++;
}
var response = _solr.AddRange(_documents.Take(batchCount));
if (response.Status == 0)
{
var count = batchCount;
_context.Debug(() => $"{count} to output");
}
else
{
_context.Error($"Couldn't add range of documents to SOLR.");
}
}
if (fullCount > 0)
{
try {
var commit = _solr.Commit();
if (commit.Status == 0)
{
_context.Entity.Inserts += System.Convert.ToUInt32(fullCount);
_context.Info($"Committed {fullCount} documents in {TimeSpan.FromMilliseconds(commit.QTime)}");
}
else
{
_context.Error($"Failed to commit {fullCount} documents. SOLR returned status {commit.Status}.");
}
} catch (SolrNetException ex) {
_context.Error($"Failed to commit {fullCount} documents. {ex.Message}");
}
}
}
public static bool add(Guid applicationId, List <SearchDoc> documents)
{
try
{
ISolrOperations <SolrDoc> solr = get_solr_operator();
solr.AddRange(documents.Select(d => d.toSolrDoc(applicationId)));
ResponseHeader response = solr.Commit();
return(response.Status == 0);
}
catch (Exception ex)
{
LogController.save_error_log(null, null, "SolrAddDocuments", ex, ModuleIdentifier.SRCH);
return(false);
}
}
public int IndexItems(List <CmsSearchResultItem> items)
{
var itemsPublished = Mapper.Map <List <CmsSearchResultItem>, List <CmsSearchResultItemPublished> >(items);
ISolrOperations <CmsSearchResultItem> instance = ServiceLocator.Current.GetInstance <ISolrOperations <CmsSearchResultItem> >();
ISolrOperations <CmsSearchResultItemPublished> instancePublished = ServiceLocator.Current.GetInstance <ISolrOperations <CmsSearchResultItemPublished> >();
int int32 = Convert.ToInt32(Math.Ceiling(Convert.ToDecimal(items.Count) / new Decimal(100)));
for (int index = 0; index < int32; ++index)
{
this.log.AddLogentry(SolisSearch.Log.Enum.LogLevel.Debug, string.Format("Indexing items {0} to {1}", (object)(100 * index), (object)(100 * (index + 1))), (Exception)null);
List <CmsSearchResultItem> list = items.Skip <CmsSearchResultItem>(100 * index).Take <CmsSearchResultItem>(100 * (index + 1)).ToList <CmsSearchResultItem>();
List <CmsSearchResultItemPublished> listPublished = itemsPublished.Skip <CmsSearchResultItemPublished>(100 * index).Take <CmsSearchResultItemPublished>(100 * (index + 1)).ToList <CmsSearchResultItemPublished>();
instance.AddRange((IEnumerable <CmsSearchResultItem>)list);
instancePublished.AddRange((IEnumerable <CmsSearchResultItemPublished>)listPublished);
this.log.AddLogentry(SolisSearch.Log.Enum.LogLevel.Debug, "Indexing documents in collection", (Exception)null);
foreach (CmsSearchResultItem searchResultItem in list.Where <CmsSearchResultItem>((Func <CmsSearchResultItem, bool>)(item => item.Documents.Any <string>())))
{
foreach (string document in searchResultItem.Documents)
{
this.log.AddLogentry(SolisSearch.Log.Enum.LogLevel.Debug, "Finding media for document " + document, (Exception)null);
ICmsMedia cmsMedia = this.cmsIndexer.ResolveMedia(document);
if (cmsMedia != null)
{
this.log.AddLogentry(SolisSearch.Log.Enum.LogLevel.Debug, "Indexing media item " + cmsMedia.Name, (Exception)null);
this.IndexRichText(document, cmsMedia.Id, searchResultItem.Acl, searchResultItem.Languages, searchResultItem.StartPublish, searchResultItem.EndPublish, searchResultItem.Id);
}
}
}
}
instance.Commit();
instance.Optimize();
instancePublished.Commit();
instancePublished.Optimize();
return(items.Count);
}
private void Initialize()
{
//See if there are movies in the solr dataset
bool hasRecords = _solr.Query(new SolrQuery("*:*"), new QueryOptions()
{
Rows = 10
}).Count > 0;
if (hasRecords)
{
return;
}
//Populate the dataset with the csv data
List <Movie> movies;
string filepath = Path.Combine(_appBasePath, "data", "movies_metadata.csv");
//Load csv, add movies
using (StreamReader reader = new StreamReader(filepath))
using (CsvReader csv = new CsvReader(reader))
{
csv.Configuration.RegisterClassMap <MovieCsvMapper>();
csv.Configuration.MissingFieldFound = null;
movies = csv.GetRecords <Movie>().ToList();
}
//Send to solr
ResponseHeader addResult = _solr.AddRange(movies);
ResponseHeader commitResult = _solr.Commit();
ResponseHeader spellCheckResult = _solr.BuildSpellCheckDictionary();
//Something went wrong
if (addResult.Status != 0)
{
HandleError(addResult, "add");
}
if (commitResult.Status != 0)
{
HandleError(commitResult, "commit");
}
if (spellCheckResult.Status != 0)
{
HandleError(spellCheckResult, "build spellcheck library");
}
}
/**
* Inserts data in bulk.
**/
public string BulkInsert(List <Product> products)
{
string res = "Didnt run..";
try
{
res = solr.AddRange(products).Status.ToString();
Commit();
}
catch (Exception e)
{
solr.Rollback();
res = e.ToString();
}
return(res);
}
public void Write(IEnumerable <IRow> rows)
{
var fullCount = 0;
foreach (var part in rows.Partition(_context.Entity.InsertSize))
{
var batchCount = (uint)0;
var docs = new List <Dictionary <string, object> >();
foreach (var row in part)
{
batchCount++;
fullCount++;
docs.Add(_fields.ToDictionary(field => field.Alias.ToLower(), field => row[field]));
}
var response = _solr.AddRange(docs);
if (response.Status == 0)
{
var count = batchCount;
_context.Debug(() => $"{count} to output");
}
else
{
_context.Error($"Couldn't add range of {docs.Count} document{docs.Count.Plural()} to SOLR.");
}
}
if (fullCount > 0)
{
try {
var commit = _solr.Commit();
if (commit.Status == 0)
{
_context.Entity.Inserts += System.Convert.ToUInt32(fullCount);
_context.Info($"Committed {fullCount} documents in {TimeSpan.FromMilliseconds(commit.QTime)}");
}
else
{
_context.Error($"Failed to commit {fullCount} documents. SOLR returned status {commit.Status}.");
}
} catch (SolrNetException ex) {
_context.Error($"Failed to commit {fullCount} documents. {ex.Message}");
}
}
}
/// <summary>
/// Tracks a provided repository for a channel and adds its documents to Solr.
/// </summary>
public void TrackRepository(GitHubUser user, List <Octokit.Repository> repositories)
{
ISolrOperations <CodeDoc> solr = ServiceLocator.Current.GetInstance <ISolrOperations <CodeDoc> >();
try
{
var clone_options = new CloneOptions();
clone_options.CredentialsProvider = (_url, _user, _cred) => new UsernamePasswordCredentials {
Username = user.GitHubAccessToken, Password = String.Empty
};
foreach (var repo in repositories)
{
string repo_path = GetTemporaryDirectory();
string git_url = repo.CloneUrl;
LibGit2Sharp.Repository.Clone(git_url, repo_path, clone_options);
List <CodeDoc> commit_files = ParseRepository(user, repo_path);
solr.AddRange(commit_files);
solr.Commit();
Directory.Delete(repo_path, true);
Console.WriteLine(String.Format("Finished tracking {0} for {1} to Solr", repo.Name, user.UUID));
}
}
catch (RateLimitExceededException) {
Console.WriteLine(String.Format("Rate limit exceeded for {0}. Stopping...", user.UUID));
}
catch (AbuseException) {
Console.WriteLine(String.Format("Abuse detection triggered for {0}. Stopping...", user.UUID));
}
catch (Exception ex) {
Console.WriteLine(String.Format("Error trying to track repository for {0}. Stopping...", user.UUID));
Console.WriteLine(ex.ToString());
}
finally {
solr.Commit();
}
}
public override void ProcessMessage(BasicDeliverEventArgs message)
{
try
{
var pt =
Newtonsoft.Json.JsonConvert.DeserializeObject <VisitedLinkFindNew>(
Encoding.UTF8.GetString(message.Body));
if (pt != null)
{
_lstCacheList.Add(new HistoryProductSorl()
{
Id = pt.Session + ":" + pt.ProductId,
CompanyId = pt.CompanyId,
ProductId = pt.ProductId,
DetailUrl = pt.Url,
TypeRun = 0,
LastUpdate = DateTime.Now.Ticks,
Session = pt.Session,
DateUpdate = Convert.ToInt32(DateTime.Now.ToString("yyyyMMdd"))
});
if (_lstCacheList.Count > 2000)
{
_solr.AddRange(_lstCacheList);
_lstCacheList.Clear();
_solr.Commit();
_log.Info("Added 1000 item");
}
}
}
catch (Exception ex)
{
_log.Error(ex + Encoding.UTF8.GetString(message.Body));
}
this.GetChannel().BasicAck(message.DeliveryTag, true);
}
private void UpdateSolrIndexForProject(IndexSettings settings, ISolrOperations<CodeDocument> solr, Project proj)
{
List<string> alldocs = null;
//find out if directory exists before doing anything to the index
if (!Directory.Exists(proj.Path))
{
Console.WriteLine(DateTime.Now.ToString() + ": Directory for project " + proj.ProjectName + " did not exist, skipping");
return;
}
//find all of the files
using (var csw = new ConsoleStopWatch(""))
{
alldocs = GetDocsForProject(proj, settings.DefaultIncludedPath, settings.DefaultExcludedPath);
csw.Name = "Finding " + alldocs.Count.ToString() + " files for project " + proj.ProjectName;
}
using (var csw = new ConsoleStopWatch("Deleting all solr docs for project " + proj.ProjectName))
{
solr.Delete(new SolrQuery("project:\"" + proj.ProjectName + "\""));
solr.Commit();
}
//breakout the file list into chunks of DOCS_PER_POST for speed. One at a time is too slow, too many can cause solr memory and thread issues
var fileChunks = Chunkify(alldocs.ToArray(), DOCS_PER_POST);
using (var csw = new ConsoleStopWatch("Adding the documents to solr for project " + proj.ProjectName))
{
//convert each to a solr document
for (int i = 0; i < fileChunks.Length; i++)
{
var file = fileChunks[i];
var codedocs = MakeDocument(file, proj);
//submit each to solr
//Tweak to leverage new CommitIWithin option of SolrNet so that we do not need to pay the cost of a commit for each group.
solr.AddRange(codedocs, new AddParameters { CommitWithin = 10000 });
}
solr.Optimize();
}
}
public void Index(IEnumerable <ServiceIndexDoc> docs)
{
Log.Info(string.Format("Indexing {0} docs", docs.Count()));
_solr.AddRange(docs);
_solr.Commit();
}
static void Main(string[] args)
{
//Change alpharetta to either localhost or your machine name
Startup.Init <Course>("http://localhost:8983/solr/courses");
ISolrOperations <Course> solr = ServiceLocator.Current.GetInstance <ISolrOperations <Course> >();
//Show Solr Admin UI and then delete all
solr.Delete(SolrQuery.All); //Equivalent to "*:*"
//Don't forget to commit
solr.Commit();
//Add one document
//Note: create and udpate are the same
Course courseSolr = new Course();
courseSolr.CourseId = "enterprise-search-using-apache-solr";
courseSolr.CourseTitle = "Getting Started with JSON in C# Using Json.NET";
courseSolr.Author = new List <string>();
courseSolr.Author.Add("Xavier Morera");
courseSolr.Tags = new List <string>();
courseSolr.Tags.Add("solr");
courseSolr.Tags.Add("big-data");
courseSolr.Description = "Search is one of the most misunderstood functionalities in the IT industry. Even further, Enterprise Search used to be neither for the faint of heart, nor for those with a thin wallet. However, since the introduction of Apache Solr, the name of the game has changed. Apache Solr brings high quality Enterprise Search to the masses. Don?t leave home without it!";
solr.Add(courseSolr);
solr.Commit();
//Add a couple of documents in a single command
List <Course> courses = new List <Course>();
Course courseGit = new Course();
courseGit.CourseId = "using-git-with-gui";
courseGit.CourseTitle = "Getting Started with JSON in C# Using Json.NET";
courseGit.Author = new List <string>();
courseGit.Author.Add("Xavier Morera");
courseGit.Tags = new List <string>();
courseGit.Tags.Add("json");
courseGit.Tags.Add("microsoft");
courseGit.Tags.Add("javascript");
courseGit.Description = "There is no doubt Git is taking over the world of source control management, mainly in open source but growing rapidly in enterprise as well. The problem usually lies in that Git is not for the faint of heart as it comes with a steep learning curve - it is hard to get started. In this course, I will help you smooth out the learning curve, understand Git?s mechanics and branching model, and in general be productive much quicker with the aid of a great free tool: Atlassian SourceTree.";
courses.Add(courseGit);
Course courseJson = new Course();
courseJson.CourseId = "json-csharp-jsondotnet-getting-started";
courseJson.CourseTitle = "Using Git with a GUI";
courseJson.Author = new List <string>();
courseJson.Author.Add("Xavier Morera");
courseJson.Tags = new List <string>();
courseJson.Tags.Add("git");
courseJson.Tags.Add("source-control");
courseJson.Description = "The rising popularity of the web, mainly around JavaScript related technologies, has given JSON a great deal of importance over other data interchange formats, like XML. JSON is a lightweight, human-readable, efficient, and easy to understand data interchange format. JSON stands for JavaScript Object Notation. If you are a C# or .NET developer who needs to work with JSON, or even understand what JSON is, then this course is for you. In this course, we will learn how to serialize, deserialize, use LINQ To JSON, improve serialization performance, BSON, create schemas, validate JSON, and, in general, work with Json.NET, a popular, high performance, free, and open source JSON framework for .NET.";
courses.Add(courseJson);
solr.AddRange(courses);
solr.Commit();
}
private static void PopulatePivotalProject(MapperFactory mapperFactory, ISolrOperations<RequirementsDocument> solr)
{
Configuration.Project pivotproject = new Configuration.Project() { ProjectName = "Program Mgmt", SystemType = SystemType.Pivotal,Url = @"https://www.pivotaltracker.com/services/v3/" };
var mapper = mapperFactory.GetMapperForProject(pivotproject);
RequirementsDocument[] docs = mapper.FindAllWorkItemsForProject();
//submit them to solr
solr.AddRange(docs, new AddParameters { CommitWithin = 10000 });
}
public void PopulateData()
{
_solr.Delete(SolrQuery.All);
_solr.AddRange(_todoItemService.GetAll());
_solr.Commit();
}
