private Task _cmd_OnMessageNoTrigger(IUserMessage arg)
{
if (!(arg.Author is SocketGuildUser user) || user.IsBot)
return Task.CompletedTask;
var _ = Task.Run(() =>
{
if (_excludedChannels.TryGetValue(user.Guild.Id, out var chans) &&
chans.Contains(arg.Channel.Id))
return;
if (_excludedServers.Contains(user.Guild.Id))
return;
if (_excludedRoles.TryGetValue(user.Guild.Id, out var roles) &&
user.Roles.Any(x => roles.Contains(x.Id)))
return;
if (!arg.Content.Contains(' ') && arg.Content.Length < 5)
return;
if (!SetUserRewarded(user.Id))
return;
_addMessageXp.Enqueue(new UserCacheItem { Guild = user.Guild, Channel = arg.Channel, User = user });
});
return Task.CompletedTask;
}
/* XXX: KeyCommand processing on keyboard input */
/// <summary>
/// Returns true if the given <paramref name="command"/> was able to be registered to <paramref name="combo"/>
/// </summary>
/// <param name="combo">The key combination to register</param>
/// <returns></returns>
public bool Register_Key_Command(KeyCombination combo, Action command)
{
if (ProtectedKeyCommands.Contains(combo))
{
return(false);
}
return(KeyCommands.TryAdd(combo, command));
}
public void Remove_RemovesTheItem()
{
var hashSet = new ConcurrentHashSet <int>();
hashSet.Add(5);
Assert.IsTrue(hashSet.Contains(5));
hashSet.TryRemove(5);
Assert.AreEqual(0, hashSet.Count);
Assert.IsFalse(hashSet.Contains(5));
}
public void ContainsTest()
{
var people = new ConcurrentHashSet <PersonProper>(RandomData.GeneratePersonRefCollection <PersonProper>(100));
var person = people.Shuffle(1).First();
var result = people.Contains(person);
Assert.IsTrue(result);
Assert.IsFalse(people.Contains(RandomData.GenerateRefPerson <PersonProper>()));
}
public void ContainsTest()
{
var people = new ConcurrentHashSet <PersonProper>(2, 10);
people.AddRange(RandomData.GeneratePersonCollection <PersonProper>(10));
var person = people.Last();
var result = people.Contains(person);
Assert.IsTrue(result);
Assert.IsFalse(people.Contains(RandomData.GeneratePerson <PersonProper>()));
}
public void Add_MultipleTimes_AddsCorrectly()
{
var hashSet = new ConcurrentHashSet <int>();
hashSet.Add(5);
hashSet.Add(8);
Assert.IsTrue(hashSet.Contains(5));
Assert.IsTrue(hashSet.Contains(8));
Assert.AreEqual(2, hashSet.Count);
hashSet.Clear();
Assert.AreEqual(0, hashSet.Count);
}
public void CreateFromEnumerable_AddsCorrectly()
{
var list = new List <int>();
list.Add(5);
list.Add(5);
list.Add(8);
var hashSet = new ConcurrentHashSet <int>(list);
Assert.IsTrue(hashSet.Contains(5));
Assert.IsTrue(hashSet.Contains(8));
Assert.AreEqual(2, hashSet.Count);
}
private void ConditionalMute(ushort cl)
{
if (cl != LocalClientId && !_UnmutedClients.Contains(cl))
{
_MutedClients.Add(cl);
}
}
public async Task StartCommunication(string connectionId, string publicKey)
{
if (!ExistingConnection.Contains(connectionId))
{
await Clients.Caller.SendAsync("ErrorCommunication", $"Unknown connection id {connectionId}");
}
else
{
Communication[Context.ConnectionId] = new UserAgrs(connectionId, publicKey);
if (Communication.ContainsKey(connectionId))
{
if (Communication[connectionId].ConnectionId == Context.ConnectionId)
{
await Clients.Caller.SendAsync("SuccessCommunication",
Communication[connectionId].Key);
await Clients.Client(connectionId).SendAsync("SuccessCommunication",
publicKey);
}
}
else
{
await Clients.Client(connectionId).SendAsync("RequestStart", Context.ConnectionId);
}
}
}
private static void ExerciseFullApi(ConcurrentHashSet <int> hashSet, int[] numbersToAdd)
{
dynamic _;
foreach (var number in numbersToAdd)
{
hashSet.Add(number);
}
var index = 0;
var genericEnumerator = hashSet.GetEnumerator();
while (index < numbersToAdd.Length && genericEnumerator.MoveNext())
{
_ = genericEnumerator.Current;
}
index = 0;
var nongenericEnumerator = ((IEnumerable)hashSet).GetEnumerator();
while (index < numbersToAdd.Length && nongenericEnumerator.MoveNext())
{
_ = nongenericEnumerator.Current;
}
_ = hashSet.Count;
var destinationArray = new int[500];
hashSet.CopyTo(destinationArray, 0);
_ = hashSet.Contains(numbersToAdd.First());
hashSet.Remove(numbersToAdd.First());
hashSet.Clear();
}
public void EnsureApplicationMainUrl(Uri currentApplicationUrl)
{
// Fixme: This causes problems with site swap on azure because azure pre-warms a site by calling into `localhost` and when it does that
// it changes the URL to `localhost:80` which actually doesn't work for pinging itself, it only works internally in Azure. The ironic part
// about this is that this is here specifically for the slot swap scenario https://issues.umbraco.org/issue/U4-10626
// see U4-10626 - in some cases we want to reset the application url
// (this is a simplified version of what was in 7.x)
// note: should this be optional? is it expensive?
if (currentApplicationUrl is null)
{
return;
}
if (_webRoutingSettings.CurrentValue.UmbracoApplicationUrl is not null)
{
return;
}
var change = !_applicationUrls.Contains(currentApplicationUrl);
if (change)
{
if (_applicationUrls.TryAdd(currentApplicationUrl))
{
ApplicationMainUrl = currentApplicationUrl;
}
}
}
private async Task ExecuteItem(string itemId, EventWaitHandle waitHandle)
{
try
{
await ProcessItem(itemId, _cancellationTokenSource.Token);
while (EnableSecondPasses && _secondPasses.Contains(itemId))
{
_secondPasses.TryRemove(itemId);
await ProcessItem(itemId, _cancellationTokenSource.Token);
}
}
catch (OperationCanceledException)
{
Logger.LogInformation($"Operation cancelled while processing {itemId}");
}
catch (Exception ex)
{
Logger.LogError(default(EventId), ex, $"Error executing item {itemId} - {ex.Message}");
}
finally
{
waitHandle.Set();
lock (_activeTasks)
{
_activeTasks.Remove(itemId);
}
}
}
public void ConcurrentHashSetContainsReturnsTrueIfSetContainsItem()
{
for (int i = 0; i < _validWords.Length; i++)
{
Assert.IsTrue(_hashSet.Contains(_validWords[i]));
}
}
private Task DelMsgOnCmd_Handler(IUserMessage msg, CommandInfo cmd)
{
var _ = Task.Run(async() =>
{
try
{
var channel = msg.Channel as SocketTextChannel;
if (channel == null)
{
return;
}
if (DeleteMessagesOnCommand.Contains(channel.Guild.Id) && cmd.Name != "prune" && cmd.Name != "pick")
{
await msg.DeleteAsync().ConfigureAwait(false);
}
}
catch (Exception ex)
{
_log.Warn("Delmsgoncmd errored...");
_log.Warn(ex);
}
});
return(Task.CompletedTask);
}
/// <inheritdoc />
public override bool IsSupportedType(Type type)
{
if (type == null)
{
throw new ArgumentNullException(nameof(type));
}
if (s_unsupportedTypeInfoSet.Contains(type))
{
return(false);
}
#if NET40
if (type.IsGenericType && type.IsConstructedGenericType() == false)
{
s_unsupportedTypeInfoSet.TryAdd(type); return(false);
}
#else
if (type.IsGenericType && type.IsConstructedGenericType == false)
{
s_unsupportedTypeInfoSet.TryAdd(type); return(false);
}
#endif
if (!s_model.CanSerialize(type))
{
s_unsupportedTypeInfoSet.TryAdd(type); return(false);
}
return(true);
}
private void Recevie_Data(IAsyncResult asyncResult)
{
var client = (UdpClient)asyncResult.AsyncState;
try
{
var remotePoint = _endPoint;
var data = client.EndReceive(asyncResult, ref remotePoint);
if (filterPeers.Count > 0 && filterPeers.Contains(remotePoint.ToInt64()))
{
_logger.InfoFormat("Filter Bad Address {0}", remotePoint);
}
else
{
while (!_receveRateLimit.Require(data.Length, out var limitTime))
{
Thread.Sleep(limitTime);
}
_recvMessageQueue.TryAdd(new DhtData()
{
Data = data, RemoteEndPoint = remotePoint
}, EnqueueWaitTime);
}
if (!running)
{
return;
}
}
catch (Exception ex)
{
_logger.Error(ex);
}
while (true)
{
try
{
client.BeginReceive(Recevie_Data, client);
break;
}
catch (SocketException ex)
{
_logger.Error("begin receive error", ex);
}
}
}
public void Add_SingleItem_AddsCorrectly()
{
var hashSet = new ConcurrentHashSet <int>();
hashSet.Add(5);
Assert.IsTrue(hashSet.Contains(5));
Assert.AreEqual(1, hashSet.Count);
}
public void Contains(bool expected, int input)
{
// Arrange
var list = new ConcurrentHashSet <int>();
// Act
var actual = list.Contains(input);
// Assert
Assert.Equal(expected, actual);
list.Dispose();
}
private bool ShouldTrackXp(SocketGuildUser user, ulong channelId)
{
if (_excludedChannels.TryGetValue(user.Guild.Id, out var chans) &&
chans.Contains(channelId))
{
return(false);
}
if (_excludedServers.Contains(user.Guild.Id))
{
return(false);
}
if (_excludedRoles.TryGetValue(user.Guild.Id, out var roles) &&
user.Roles.Any(x => roles.Contains(x.Id)))
{
return(false);
}
return(true);
}
public void ConcurrentHashSetContainsReturnsTrueIfItemIsPresent()
{
var items = new List <string> {
"Frodo", "Sam", "Merry", "Pippin"
};
var set = new ConcurrentHashSet <string>(items);
bool result = set.Contains("Frodo");
Assert.IsTrue(result);
}
public void Add_SameItemMultipleTimes_AddsCorrectly()
{
var hashSet = new ConcurrentHashSet <int>();
for (var i = 0; i < 500; i++)
{
hashSet.Add(5);
}
Assert.IsTrue(hashSet.Contains(5));
Assert.AreEqual(1, hashSet.Count);
}
private Task PotentialFlowerGeneration(IMessage imsg)
{
var msg = imsg as IUserMessage;
if (msg == null || msg.IsAuthor() || msg.Author.IsBot)
{
return(Task.CompletedTask);
}
var channel = imsg.Channel as ITextChannel;
if (channel == null)
{
return(Task.CompletedTask);
}
if (!generationChannels.Contains(channel.Id))
{
return(Task.CompletedTask);
}
var t = Task.Run(async() =>
{
var lastGeneration = lastGenerations.GetOrAdd(channel.Id, DateTime.MinValue);
if (DateTime.Now - TimeSpan.FromSeconds(cooldown) < lastGeneration) //recently generated in this channel, don't generate again
{
return;
}
var num = rng.Next(1, 101) + chance * 100;
if (num > 100)
{
lastGenerations.AddOrUpdate(channel.Id, DateTime.Now, (id, old) => DateTime.Now);
try
{
var sent = await channel.SendFileAsync(
GetRandomCurrencyImagePath(),
$"❗ A random { Gambling.Gambling.CurrencyName } appeared! Pick it up by typing `{FaultyBot.ModulePrefixes[typeof(Games).Name]}pick`")
.ConfigureAwait(false);
plantedFlowers.AddOrUpdate(channel.Id, new List <IUserMessage>()
{
sent
}, (id, old) => { old.Add(sent); return(old); });
}
catch { }
}
});
return(Task.CompletedTask);
}
public void ConcurrentHashSet_FunctionsAsNormalHashSet_ForSingleThreadedAccess(params int[] numbersToAdd)
{
// Because we're not doing anything interesting with the hashset itself, it seems reasonable to just wrap all of the basic hashset API tests into one test
var distinctNumbers = numbersToAdd.Distinct().ToList();
// Add
foreach (var number in numbersToAdd)
{
_concurrentHashSet.Add(number);
}
// GetEnumerator<T>
var index = 0;
var genericEnumerator = _concurrentHashSet.GetEnumerator();
while (index < numbersToAdd.Length && genericEnumerator.MoveNext())
{
Assert.AreEqual(distinctNumbers[index++], genericEnumerator.Current);
}
Assert.AreEqual(distinctNumbers.Count, index);
// GetEnumerator
index = 0;
var nongenericEnumerator = ((IEnumerable)_concurrentHashSet).GetEnumerator();
while (index < numbersToAdd.Length && nongenericEnumerator.MoveNext())
{
Assert.AreEqual(distinctNumbers[index++], nongenericEnumerator.Current);
}
Assert.AreEqual(distinctNumbers.Count, index);
// Count
Assert.AreEqual(_concurrentHashSet.Count, distinctNumbers.Count);
// CopyTo
var destinationArray = new int[distinctNumbers.Count];
_concurrentHashSet.CopyTo(destinationArray, 0);
Assert.True(distinctNumbers.SequenceEqual(destinationArray));
// Contains
Assert.True(distinctNumbers.All(_concurrentHashSet.Contains));
// Remove
_concurrentHashSet.Remove(distinctNumbers.First());
Assert.False(_concurrentHashSet.Contains(distinctNumbers.First()));
// Clear
_concurrentHashSet.Clear();
Assert.AreEqual(0, _concurrentHashSet.Count);
Assert.False(distinctNumbers.Any(_concurrentHashSet.Contains));
}
public void Contains_Success()
{
var hashSet = new ConcurrentHashSet <string>();
Parallel.For(0, 10, i =>
{
hashSet.Add(i.ToString());
});
bool result = hashSet.Contains(5.ToString());
Assert.IsTrue(result);
}
private async Task ValidateAuthorityAsync(Authority authority)
{
// race conditions could occur here, where multiple requests validate the authority at the same time
// but this is acceptable and once the cache is filled, no more HTTP requests will be made
if (!s_validatedEnvironments.Contains(authority.AuthorityInfo.Host))
{
// validate the original authority, as the resolved authority might be regionalized and we cannot validate regionalized authorities.
var validator = AuthorityValidatorFactory.Create(authority.AuthorityInfo, _requestContext);
await validator.ValidateAuthorityAsync(authority.AuthorityInfo).ConfigureAwait(false);
s_validatedEnvironments.Add(authority.AuthorityInfo.Host);
}
}
public void ConcurrentHashSetTryRemoveRemovesIfSetContainsItem()
{
var items = new List <string> {
"Frodo", "Sam", "Merry", "Pippin"
};
var set = new ConcurrentHashSet <string>(items);
set.TryRemove("Frodo");
Assert.IsFalse(set.Contains("Frodo"));
Assert.AreEqual(3, set.Count);
}
public void ConcurrentHashSetTryAddDoesNotAddIfSetContainsItem()
{
var items = new List <string> {
"Frodo", "Sam", "Merry", "Pippin"
};
var set = new ConcurrentHashSet <string>(items);
set.TryAdd("Sam");
Assert.IsTrue(set.Contains("Sam"));
Assert.AreEqual(4, set.Count);
}
public ReturnT <String> pushTriggerQueue(TriggerParam triggerParam)
{
// avoid repeat
if (triggerLogIdSet.Contains(triggerParam.logId))
{
logger.Debug(string.Format("repeate trigger job, logId:{0}", triggerParam.logId));
return(new ReturnT <String>(ReturnT <string> .FAIL_CODE, "repeate trigger job, logId:" + triggerParam.logId));
}
triggerLogIdSet.Add(triggerParam.logId);
triggerQueue.Add(triggerParam);
return(ReturnT <string> .SUCCESS);
}
public async Task Add_ViaMultipleTaskYields_AddsCorrectly()
{
async Task AddToHashSet(ConcurrentHashSet <int> setToAddTo)
{
await Task.Yield();
for (var i = 0; i < 10000; i++)
{
setToAddTo.Add(i);
}
}
async Task RemoveFromHashSet(ConcurrentHashSet <int> setToAddTo)
{
await Task.Yield();
for (var i = 0; i < 10000; i++)
{
setToAddTo.TryRemove(i);
}
}
var hashSet = new ConcurrentHashSet <int>();
var tasks = new List <Task>();
for (var i = 0; i < 100; i++)
{
var task = AddToHashSet(hashSet);
tasks.Add(task);
}
await Task.WhenAll(tasks);
Assert.AreEqual(10000, hashSet.Count);
for (var i = 0; i < 10000; i++)
{
Assert.IsTrue(hashSet.Contains(i));
}
tasks.Clear();
for (var i = 0; i < 100; i++)
{
var task = RemoveFromHashSet(hashSet);
tasks.Add(task);
}
await Task.WhenAll(tasks);
Assert.AreEqual(0, hashSet.Count);
}
public async Task <bool> TryBlockLate(DiscordSocketClient client, IUserMessage msg, IGuild guild, IMessageChannel channel, IUser user, string moduleName, string commandName)
{
await Task.Yield();
commandName = commandName.ToLowerInvariant();
if (commandName != "resetglobalperms" &&
(BlockedCommands.Contains(commandName) ||
BlockedModules.Contains(moduleName.ToLowerInvariant())))
{
return(true);
//return new ExecuteCommandResult(cmd, null, SearchResult.FromError(CommandError.Exception, $"Command or module is blocked globally by the bot owner."));
}
return(false);
}
static void Main(string[] args)
{
int startRaw = Environment.TickCount;
HashSet<int> testRaw = new HashSet<int>();
for (int i = 0; i < 4000000; i++)
testRaw.Add(i);
int timeRaw = Environment.TickCount - startRaw;
for (int i = 0; i < 4000000; i++)
testRaw.Contains(i);
int timeRaw2 = Environment.TickCount - startRaw;
Console.WriteLine(timeRaw.ToString());
Console.WriteLine(timeRaw2.ToString());
//Action<int, int> fnOutput = (total, start) =>
// Enumerable.Range(start, total).ForEach(i => Console.WriteLine(i));
//for (int i = 0; i < 40; i++)
// Console.WriteLine(i);
////fnOutput(40, 0);
//Action<int> fnOutput40 = fnOutput.Curry()(40);
//fnOutput40(0);
//fnOutput40(30);
//ICollection<int> coll = new HashSet<int>();
//int[] data = { 42, 46, 77, 345, -12 };
//foreach (var local in data)
// coll.Add(local);
//int test5 = TestFunctions.fnThunk(new HashSet<int>(), 1);
//Console.Write("None\t");
//Console.WriteLine(test5);
//var test6 = TestFunctions.CalcP<LockFreeList<int>>();
//Console.Write("LFLS\t");
//Console.WriteLine(test6);
//var test1 = TestFunctions.CalcP<CourseGrainedICollectionWrapper<int>, HashSet<int>>();
//Console.Write("Lock\t");
//Console.WriteLine(test1);
//var test2 = TestFunctions.CalcP<ReadWriteICollectionWrapper<int>, HashSet<int>>();
//Console.Write("RWLS\t");
//Console.WriteLine(test2);
//var test3 = TestFunctions.CalcP<InterlockedICollectionWrapper<int>, HashSet<int>>();
//Console.Write("Intr\t");
//Console.WriteLine(test3);
CollectionWrapperCourseGrained<int> wrp = new CollectionWrapperCourseGrained<int>();
wrp.Collection = new HashSet<int>();
var resultn = TestFunctions.fnTest4MAdd(wrp, 4);
Console.WriteLine(resultn.ToString());
var resultnc = TestFunctions.fnTest4MContains(wrp, 4);
Console.WriteLine(resultnc.ToString());
var resultnr = TestFunctions.fnTest4MRemove(wrp, 4);
Console.WriteLine(resultnr.ToString());
int start = Environment.TickCount;
//LockFreeList<int> list = new LockFreeList<int>();
//ConcurrentHashSet<int> list = new ConcurrentHashSet<int>(4100000,true);
ConcurrentHashSet<int> list = new ConcurrentHashSet<int>(4100000, true);
var result2 = TestFunctions.fnTest4MAdd(list, 4);
Console.Write("INSERTS = ");
Console.WriteLine(result2.ToString());
Console.Write("TICKS = ");
Console.WriteLine(Environment.TickCount - start);
#if (DEBUG)
Console.WriteLine(list.DebugDump);
list.DebugReset();
//list.DebugDataValidate();
bool success = list.Contains(1000000);
#endif
start = Environment.TickCount;
var resultnc2 = TestFunctions.fnTest4MContains(list, 4);
Console.Write("CONTAIN = ");
Console.WriteLine(resultnc2.ToString());
Console.Write("TICKS = ");
Console.WriteLine(Environment.TickCount - start);
#if (DEBUG)
Console.WriteLine(list.DebugDump);
#endif
//start = Environment.TickCount;
//var resultnr2 = TestFunctions.fnTest4MRemove(list, 4);
//Console.Write("REMOVE = ");
//Console.WriteLine(resultnr2.ToString());
//Console.Write("TICKS = ");
//Console.WriteLine(Environment.TickCount - start);
//start = Environment.TickCount;
list.Clear();
Console.Write("CLEAR = ");
Console.WriteLine(Environment.TickCount - start);
#if (DEBUG)
//Console.WriteLine(list.DebugEmpty);
#endif
start = Environment.TickCount;
var result2b = TestFunctions.fnTest4MAdd(list, 4);
Console.Write("INSERTS = ");
Console.WriteLine(result2b.ToString());
Console.Write("TICKS = ");
Console.WriteLine(Environment.TickCount - start);
#if (DEBUG)
Console.WriteLine(list.DebugDump);
#endif
Console.ReadLine();
}
public void TestLockFreeHashSet_Long()
{
ConcurrentHashSet<long> listTest = new ConcurrentHashSet<long>();
listTest.Add(42);
listTest.Add(22);
listTest.Add(22);
listTest.Add(64);
listTest.Add(55);
Assert.IsTrue(listTest.Count == 4);
Assert.IsTrue(listTest.Contains(42));
Assert.IsFalse(listTest.Contains(142));
Assert.IsFalse(listTest.Contains(2));
Assert.IsTrue(listTest.Contains(64));
listTest.Remove(42);
Assert.IsFalse(listTest.Contains(42));
Assert.IsFalse(listTest.Remove(42));
Assert.IsTrue(listTest.Count == 3);
listTest.Add(42);
listTest.Add(41);
listTest.Add(0);
Assert.IsTrue(listTest.Count == 6);
listTest.Add(0);
Assert.IsTrue(listTest.Count == 6);
Assert.IsTrue(listTest.Contains(41));
Assert.IsTrue(listTest.Contains(0));
Assert.IsTrue(listTest.Remove(0));
Assert.IsFalse(listTest.Contains(0));
Assert.IsTrue(listTest.Remove(22));
Assert.IsTrue(listTest.Remove(55));
listTest.Add(1212);
listTest.Add(323);
listTest.Add(7567);
listTest.Add(567);
Assert.IsTrue(listTest.Count == 7);
}
public void TestLockFreeHashSet_Stream()
{
ICollection<Stream> listTest = new ConcurrentHashSet<Stream>();
MemoryStream ms1 = new MemoryStream();
MemoryStream ms2 = new MemoryStream();
try
{
Enumerable.Range(0, 5000)
.Convert(i => new MemoryStream())
.InsertAtPosition(ms1, 455)
.ForEach(s => listTest.Add(s));
}
catch (Exception ex)
{
throw ex;
}
Assert.IsTrue(listTest.Count == 5001);
Assert.IsTrue(listTest.Contains(ms1));
Assert.IsFalse(listTest.Contains(ms2));
}
// Collections.synchronizedMap(new WeakHashMap<SegmentCoreReaders, bool?>());
public virtual void RunTest(string testName)
{
Failed.Set(false);
AddCount.Set(0);
DelCount.Set(0);
PackCount.Set(0);
DateTime t0 = DateTime.UtcNow;
Random random = new Random(Random().Next());
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues());
DirectoryInfo tempDir = CreateTempDir(testName);
Dir = GetDirectory(NewMockFSDirectory(tempDir)); // some subclasses rely on this being MDW
if (Dir is BaseDirectoryWrapper)
{
((BaseDirectoryWrapper)Dir).CheckIndexOnClose = false; // don't double-checkIndex, we do it ourselves.
}
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetInfoStream(new FailOnNonBulkMergesInfoStream());
if (LuceneTestCase.TEST_NIGHTLY)
{
// newIWConfig makes smallish max seg size, which
// results in tons and tons of segments for this test
// when run nightly:
MergePolicy mp = conf.MergePolicy;
if (mp is TieredMergePolicy)
{
((TieredMergePolicy)mp).MaxMergedSegmentMB = 5000.0;
}
else if (mp is LogByteSizeMergePolicy)
{
((LogByteSizeMergePolicy)mp).MaxMergeMB = 1000.0;
}
else if (mp is LogMergePolicy)
{
((LogMergePolicy)mp).MaxMergeDocs = 100000;
}
}
conf.SetMergedSegmentWarmer(new IndexReaderWarmerAnonymousInnerClassHelper(this));
if (VERBOSE)
{
conf.InfoStream = new PrintStreamInfoStreamAnonymousInnerClassHelper(this, Console.Out);
}
Writer = new IndexWriter(Dir, conf);
TestUtil.ReduceOpenFiles(Writer);
TaskScheduler es = Random().NextBoolean() ? null : TaskScheduler.Default;
DoAfterWriter(es);
int NUM_INDEX_THREADS = TestUtil.NextInt(Random(), 2, 4);
int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : RANDOM_MULTIPLIER;
ISet<string> delIDs = new ConcurrentHashSet<string>(new HashSet<string>());
ISet<string> delPackIDs = new ConcurrentHashSet<string>(new HashSet<string>());
IList<SubDocs> allSubDocs = new SynchronizedCollection<SubDocs>();
DateTime stopTime = DateTime.UtcNow.AddSeconds(RUN_TIME_SEC);
ThreadClass[] indexThreads = LaunchIndexingThreads(docs, NUM_INDEX_THREADS, stopTime, delIDs, delPackIDs, allSubDocs);
if (VERBOSE)
{
Console.WriteLine("TEST: DONE start " + NUM_INDEX_THREADS + " indexing threads [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]");
}
// Let index build up a bit
Thread.Sleep(100);
DoSearching(es, stopTime);
if (VERBOSE)
{
Console.WriteLine("TEST: all searching done [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]");
}
for (int thread = 0; thread < indexThreads.Length; thread++)
{
indexThreads[thread].Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: done join indexing threads [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]; addCount=" + AddCount + " delCount=" + DelCount);
}
IndexSearcher s = FinalSearcher;
if (VERBOSE)
{
Console.WriteLine("TEST: finalSearcher=" + s);
}
Assert.IsFalse(Failed.Get());
bool doFail = false;
// Verify: make sure delIDs are in fact deleted:
foreach (string id in delIDs)
{
TopDocs hits = s.Search(new TermQuery(new Term("docid", id)), 1);
if (hits.TotalHits != 0)
{
Console.WriteLine("doc id=" + id + " is supposed to be deleted, but got " + hits.TotalHits + " hits; first docID=" + hits.ScoreDocs[0].Doc);
doFail = true;
}
}
// Verify: make sure delPackIDs are in fact deleted:
foreach (string id in delPackIDs)
{
TopDocs hits = s.Search(new TermQuery(new Term("packID", id)), 1);
if (hits.TotalHits != 0)
{
Console.WriteLine("packID=" + id + " is supposed to be deleted, but got " + hits.TotalHits + " matches");
doFail = true;
}
}
// Verify: make sure each group of sub-docs are still in docID order:
foreach (SubDocs subDocs in allSubDocs.ToList())
{
TopDocs hits = s.Search(new TermQuery(new Term("packID", subDocs.PackID)), 20);
if (!subDocs.Deleted)
{
// We sort by relevance but the scores should be identical so sort falls back to by docID:
if (hits.TotalHits != subDocs.SubIDs.Count)
{
Console.WriteLine("packID=" + subDocs.PackID + ": expected " + subDocs.SubIDs.Count + " hits but got " + hits.TotalHits);
doFail = true;
}
else
{
int lastDocID = -1;
int startDocID = -1;
foreach (ScoreDoc scoreDoc in hits.ScoreDocs)
{
int docID = scoreDoc.Doc;
if (lastDocID != -1)
{
Assert.AreEqual(1 + lastDocID, docID);
}
else
{
startDocID = docID;
}
lastDocID = docID;
Document doc = s.Doc(docID);
Assert.AreEqual(subDocs.PackID, doc.Get("packID"));
}
lastDocID = startDocID - 1;
foreach (string subID in subDocs.SubIDs)
{
hits = s.Search(new TermQuery(new Term("docid", subID)), 1);
Assert.AreEqual(1, hits.TotalHits);
int docID = hits.ScoreDocs[0].Doc;
if (lastDocID != -1)
{
Assert.AreEqual(1 + lastDocID, docID);
}
lastDocID = docID;
}
}
}
else
{
// Pack was deleted -- make sure its docs are
// deleted. We can't verify packID is deleted
// because we can re-use packID for update:
foreach (string subID in subDocs.SubIDs)
{
Assert.AreEqual(0, s.Search(new TermQuery(new Term("docid", subID)), 1).TotalHits);
}
}
}
// Verify: make sure all not-deleted docs are in fact
// not deleted:
int endID = Convert.ToInt32(docs.NextDoc().Get("docid"));
docs.Dispose();
for (int id = 0; id < endID; id++)
{
string stringID = "" + id;
if (!delIDs.Contains(stringID))
{
TopDocs hits = s.Search(new TermQuery(new Term("docid", stringID)), 1);
if (hits.TotalHits != 1)
{
Console.WriteLine("doc id=" + stringID + " is not supposed to be deleted, but got hitCount=" + hits.TotalHits + "; delIDs=" + string.Join(",", delIDs.ToArray()));
doFail = true;
}
}
}
Assert.IsFalse(doFail);
Assert.AreEqual(AddCount.Get() - DelCount.Get(), s.IndexReader.NumDocs, "index=" + Writer.SegString() + " addCount=" + AddCount + " delCount=" + DelCount);
ReleaseSearcher(s);
Writer.Commit();
Assert.AreEqual(AddCount.Get() - DelCount.Get(), Writer.NumDocs(), "index=" + Writer.SegString() + " addCount=" + AddCount + " delCount=" + DelCount);
DoClose();
Writer.Dispose(false);
// Cannot shutdown until after writer is closed because
// writer has merged segment warmer that uses IS to run
// searches, and that IS may be using this es!
/*if (es != null)
{
es.shutdown();
es.awaitTermination(1, TimeUnit.SECONDS);
}*/
TestUtil.CheckIndex(Dir);
Dir.Dispose();
System.IO.Directory.Delete(tempDir.FullName, true);
if (VERBOSE)
{
Console.WriteLine("TEST: done [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]");
}
}
