public SafeReaderWriterLock()
{
this.locker = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
this.upgradableReaderLock = new SafeUpgradeReaderLock(this.locker);
this.readerLock = new SafeReaderLock(this.locker);
this.writerLock = new SafeWriterLock(this.locker);
}
internal HMDManager(IFactory factory, IDeviceManager manager)
{
if (factory == null)
throw new ArgumentNullException();
if (manager == null)
throw new ArgumentNullException();
_factory = factory;
_lock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
_manager = manager;
_handler = new InternalMessageHandler(this);
_manager.MessageHandler = _handler;
_nativeResources = new Dictionary<DeviceKey, DeviceResources>();
_devices = new Dictionary<DeviceKey, HMD>();
_defaultprofile = manager.DeviceDefaultProfile;
// Initially, we must to enumerate all devices, which are currently attached
// to the computer.
using (var handles = _manager.HMDDevices)
{
foreach (var handle in handles)
{
AddDevice(handle);
}
}
}
public BaseSocketConnectionHost(HostType hostType, CallbackThreadType callbackThreadtype, ISocketService socketService, DelimiterType delimiterType, byte[] delimiter, int socketBufferSize, int messageBufferSize, int idleCheckInterval, int idleTimeOutValue)
{
context = new SocketProviderContext
{
Active = false,
SyncActive = new object(),
SocketCreators = new List<BaseSocketConnectionCreator>(),
SocketConnections = new Dictionary<long, BaseSocketConnection>(),
BufferManager = BufferManager.CreateBufferManager(0, messageBufferSize),
SocketService = socketService,
IdleCheckInterval = idleCheckInterval,
IdleTimeOutValue = idleTimeOutValue,
CallbackThreadType = callbackThreadtype,
DelimiterType = delimiterType,
Delimiter = delimiter,
DelimiterEncrypt = new byte[] { 0xFE, 0xDC, 0xBA, 0x98, 0xBA, 0xDC, 0xFE },
MessageBufferSize = messageBufferSize,
SocketBufferSize = socketBufferSize,
HostType = hostType
};
fSocketConnectionsSync = new ReaderWriterLockSlim();
fWaitCreatorsDisposing = new ManualResetEvent(false);
fWaitConnectionsDisposing = new ManualResetEvent(false);
fWaitThreadsDisposing = new ManualResetEvent(false);
}
protected virtual void Dispose(bool disposing)
{
if (!_disposed)
{
if (disposing)
{
//free managed resources here
}
//free unmanaged resrources here
if (_lock.IsReadLockHeld)
{
_lock.ExitReadLock();
}
else if (_lock.IsWriteLockHeld)
{
_lock.ExitWriteLock();
}
else if (_lock.IsUpgradeableReadLockHeld)
{
_lock.ExitUpgradeableReadLock();
}
_lock = null;
_disposed = true;
}
}
public Region(int id)
{
_creaturesRWLS = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
_propsRWLS = new ReaderWriterLockSlim();
_itemsRWLS = new ReaderWriterLockSlim();
this.Id = id;
_creatures = new Dictionary<long, Creature>();
_props = new Dictionary<long, Prop>();
_items = new Dictionary<long, Item>();
_clients = new HashSet<ChannelClient>();
_regionData = AuraData.RegionInfoDb.Find(this.Id);
if (_regionData == null)
{
Log.Warning("Region: No data found for '{0}'.", this.Id);
return;
}
this.Collissions = new RegionCollision(_regionData.X1, _regionData.Y1, _regionData.X2, _regionData.Y2);
this.Collissions.Init(_regionData);
this.LoadClientProps();
}
internal PacketReceiver(string LocalAddress) : base (LocalAddress)
{
packetReceiver = new Queue();
tokenSource = new CancellationTokenSource();
readerWriterLock = new ReaderWriterLockSlim();
tokenSource.Token.Register(() => {
// Clear on cancel
packetReceiver.Clear();
});
var thReceiveQueue = Task.Factory.StartNew(() => {
while (tokenSource.Token.IsCancellationRequested == false)
{
readerWriterLock.EnterUpgradeableReadLock();
if (packetReceiver.Count > 0)
{
readerWriterLock.EnterWriteLock();
byte[] data = (byte[])packetReceiver.Dequeue();
readerWriterLock.ExitWriteLock();
if (OnNewPacketReceived != null)
OnNewPacketReceived(this, new NewPacketEventArgs(data));
}
readerWriterLock.ExitUpgradeableReadLock();
}
});
}
protected TypePort t_port = TypePort.Primary; // тип порта
#endregion Fields
#region Constructors
/// <summary>
/// инициализирует новый экземпляр класса
/// </summary>
public Serial(Repository reposit)
{
t_mutex = new Mutex();
answerWaiter = new ManualResetEvent(true);
timer = new Timer(TimerElapsed, null, Timeout.Infinite, timerPerion);
//static_list = new List<Packet>();
s_mutex = new Mutex();
port = new SerialPort();
passive_mutex = new Mutex();
calculator = new CalculaterCRC();
translater = new Translater(TranslaterFunction);
input = new List<byte>();
output = new List<byte>();
opt_slim = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
attemptsToRead = 1;
attemptsCycled = 128;
waitTimeout = 20;
repository = reposit;
is_primary_done = false;
is_secondary_done = false;
is_slim = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
public Topic(uint storeSize, TimeSpan lifespan)
{
_lifespan = lifespan;
Subscriptions = new List<ISubscription>();
Store = new MessageStore<Message>(storeSize);
SubscriptionLock = new ReaderWriterLockSlim();
}
public RideProcessor(string userId)
{
_outputLock = new ReaderWriterLockSlim();
UserId = userId;
// load the turbo
_spotData = new Spot();
_lastSpotData = null;
// Set-up the lease
LeaseId = GenerateLease(12);
// Set the initial timeout
LastUpdateTimestamp = DateTime.UtcNow;
_messageQueue = new Queue<InterfaceDatum>(QUEUE_CAPACITY);
_queueLock = new ReaderWriterLockSlim();
_zeroingTimer = new Timer(CheckZeroing, null, 1000, 4000);
_Rider = Rider.LoadAny(userId);
// will load either a real or virtual rider
if (_Rider != null)
{
_Turbo = TurboTrainer.Load(UserId, _Rider.CurrentTurbo, _Rider.TurboIsCalibrated);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="T:Stumps.Server.StumpsServerInstance"/> class.
/// </summary>
/// <param name="serverFactory">The factory used to initialize new server instances.</param>
/// <param name="serverId">The unique identifier of the Stumps server.</param>
/// <param name="dataAccess">The data access provider used by the instance.</param>
public StumpsServerInstance(IServerFactory serverFactory, string serverId, IDataAccess dataAccess)
{
if (serverFactory == null)
{
throw new ArgumentNullException("serverFactory");
}
_serverFactory = serverFactory;
this.ServerId = serverId;
_lock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
_dataAccess = dataAccess;
// Setup the objects needed to keep track of Stumps.
_stumpList = new List<StumpContract>();
_stumpReference = new Dictionary<string, StumpContract>(StringComparer.OrdinalIgnoreCase);
// Setup the recordings maintained by the server instance.
this.Recordings = new Recordings();
// Initialize the server
InitializeServer();
// Initialize the Stumps
InitializeStumps();
}
public DefaultNodeLocator()
{
this.servers = new Dictionary<uint, IMemcachedNode>(new UIntEqualityComparer());
this.deadServers = new Dictionary<IMemcachedNode, bool>();
this.allServers = new List<IMemcachedNode>();
this.serverAccessLock = new ReaderWriterLockSlim();
}
/// <summary>
/// Acquires a read lock on both the block and the block collection.
/// </summary>
/// <param name="collectionLock">The lock on the block collection.</param>
/// <param name="blockLock">The lock object used to acquire the lock.</param>
/// <param name="requestLock"></param>
public BlockLock(
IDisposable collectionLock,
ReaderWriterLockSlim accessLock,
RequestLock requestLock)
{
// Keep track of the collection lock so we can release it.
this.collectionLock = collectionLock;
// Acquire the lock based on the requested type.
switch (requestLock)
{
case RequestLock.Read:
blockLock = new NestableReadLock(accessLock);
break;
case RequestLock.UpgradableRead:
blockLock = new NestableUpgradableReadLock(accessLock);
break;
case RequestLock.Write:
blockLock = new NestableWriteLock(accessLock);
break;
default:
throw new InvalidOperationException(
"Could not acquire lock with unknown type: " + requestLock);
}
}
public SimpleSubscriber()
{
_handlersStore = new Dictionary<Type, ISet<object>>();
_emptySet = new object[0];
_latch = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
_typesCache = new Dictionary<Type, Type[]>();
}
public void Dispose_Errors()
{
var v = new ReaderWriterLockSlim();
v.Dispose();
try
{
v.EnterUpgradeableReadLock();
Assert.Fail("1");
}
catch (ObjectDisposedException)
{
}
try
{
v.EnterReadLock();
Assert.Fail("2");
}
catch (ObjectDisposedException)
{
}
try
{
v.EnterWriteLock();
Assert.Fail("3");
}
catch (ObjectDisposedException)
{
}
}
/// <summary>
/// Executed ASYNCHRONOUSLY,
/// Allows to do a given action (in a new task) every XXX ms for XXX times
/// by default it does the action when creating this instance, set doActionOnCreate = false to not do it immediatly
/// </summary>
public ReccurentAction(Action actionToDo, long timeLapse, int nbRepeat = 0, bool doActionOnCreate = true)
{
_nbRepeat = nbRepeat;
_actionToDo = actionToDo;
_lock = new ReaderWriterLockSlim();
if (_actionToDo == null)
throw new Exception("ReccurentAction > the action can't be null");
// initiate the timer if needed
if (_timer == null) {
_timer = new Timer(timeLapse) {
AutoReset = true
};
_timer.Elapsed += OnTick;
_timer.Start();
}
// do the recurrent action immediatly?
if (doActionOnCreate)
OnTick(null, null);
// keep a reference to this so we can clean them all if needed
_savedReccurentActionStarted.Add(this);
}
public void Dispose()
{
if (IsDisposed)
throw new ObjectDisposedException(ToString());
_readerWriterLock.ExitWriteLock();
_readerWriterLock = null;
}
public void Dispose()
{
if (_sync != null) {
_sync.ExitWriteLock();
_sync = null;
}
}
// Private constructor to prevent explicit instantiation
private Hub()
{
flows = new List<Flow>();
subscriptions = new Dictionary<string, List<Flow>>();
rwlock = new ReaderWriterLockSlim();
}
public ServiceHost(int servicePort)
{
pools = new List<SocketPool>();
poolsLock = new ReaderWriterLockSlim();
container = new ServiceContainer();
this.servicePort = servicePort;
}
private ReaderWriterLockSlim thisLock; //读写锁
#endregion Fields
#region Constructors
public AqiManage()
{
thisLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
allDLLs = new List<Assembly>();
InitDll();
allAqiWebs = new Dictionary<string, IAqiWeb>();
InitAqiWeb();
allSrcUrls = new Dictionary<string, ISrcUrl>();
InitSrcUrl();
initSaver();
//aqiSaver = new AqiFileSaver(this);
aqiNoter = new AqiNoter(this);
//目前默认为一个Retryer
aqiRetryer = new AqiRetryer(this);
initRunner();
//目前默认为一个Runner
//aqiRunner = new AqiRunner(this, allSrcUrls);
//aqiRunner.RunEvent += new AqiRunner.RunEventHandler(aqiRunner_RunEvent);
}
protected int timerPerion; // частота отрисовки графиков в активном режиме
#endregion Fields
#region Constructors
/// <summary>
/// Инициализирует новый экземпляр класса
/// </summary>
/// <param name="GPanel">Панель которую будет обслуживать манеджер</param>
public GraphicManager(Panel GPanel)
{
mutex = new Mutex();
slim = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
mode = DrawMode.Default;
if (GPanel != null)
{
panel = GPanel;
//panel.StartTime = DateTime.Now;
panel.OnResize += new EventHandler(Sheet_Resize);
panel.Sheet.onOrientationChange += new EventHandler(Sheet_onOrientationChange);
panel.Sheet.onIntervalInCellChange += new EventHandler(Sheet_onIntervalInCellChange);
timerPerion = 500;
timer = new Timer(TimerCallback, null, Timeout.Infinite, timerPerion);
}
else
{
throw new ArgumentNullException();
}
}
public ObjectVersionStore(Config config, ObjectMetadataStore metadata)
{
_metadata = metadata;
_store = new KeyValueStore(Path.Combine(config.BaseDataPath, "ObjectVersionStore"));
_versions = new Dictionary<string, uint>(StringComparer.OrdinalIgnoreCase);
_lock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
protected List<Place> places = null; // место в репозитарии
#endregion Fields
#region Constructors
/// <summary>
/// Инициализирует новый экземпляр класса
/// </summary>
public Repository()
{
mutex = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
places = new List<Place>();
packets = new List<Packet>();
}
/// <summary>
/// 默认构造
/// 没有更新间隔
/// </summary>
/// <param name="name"></param>
public SrcUrlGroupTimer(string name)
{
this.name = name;
this.intervalseconds = -1;
this.listISU = new Dictionary<string, ISrcUrl>();
this.thisLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
public MorphologicalFilteringOptionsViewModel()
{
StackSync = new ReaderWriterLockSlim();
if (Execute.InDesignMode)
LoadDesignTimeData();
}
public OpticalFlow()
{
//_faces = new HaarCascade("haarcascade_frontalface_alt.xml");
try
{
_faces = new HaarCascade("haarcascade_frontalface_alt.xml");
}
catch
{
DialogResult Result;
Result = MessageBox.Show("HAAR FILE NOT FOUND", "ERROR", MessageBoxButtons.OK, MessageBoxIcon.Error);
Application.Exit();
CrashError = true;
}
rand = new Random();
detected = false;
ActualFeature = new PointF[1][];
currentCentroid = new PointF(200, 150);
sema = new Semaphore(0, 1);
kalman = new KalmanFilter();
kalmanLock = new System.Threading.ReaderWriterLockSlim();
//_capture = new Capture("C:\\Users\\Zenith\\SkyDrive\\2012 FALL\\CSCE 483 Computer System Design\\ARDroneOut.avi");
Task.Factory.StartNew(() => initializeThread());
mut = new Mutex();
// Initialize boolean variables
isVectorsVisible = false;
isHaarDetectionVisible = false;
isAllPointsVisible = false;
isWithin100PixelsVisible = false;
isKalmanPredictVisible = false;
isKalmanEstimatedVisible = false;
}
public NntpParser(NzbConnection connection)
{
ArgumentChecker.ThrowIfNull("connection", connection);
_connection = connection;
_synchronizationObject = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
public Rider()
{
_outputDatum = new InterfaceDatum();
_datumLock = new ReaderWriterLockSlim();
_IsSent = true; // This will prevent any send until new data are available
_rnd = new Random(); // For test data
}
public DbDiskCache(IFileWrapper fileWrapper, IRRConfiguration configuration, IUriBuilder uriBuilder) : base(fileWrapper, configuration, uriBuilder, null)
{
RRTracer.Trace("Creating db disk cache");
const int interval = 1000*60*5;
timer = new Timer(PurgeOldFiles, null, 0, interval);
_cacheLock = new ReaderWriterLockSlim();
}
public MysqlDatabase(DatabaseConfig config)
{
Connected = false;
Connection = null;
DatabaseLock = new ReaderWriterLockSlim();
Connect(config);
}
private SlimThreadLockHandler(ReaderWriterLockSlim lockInstance, bool isWriteLock, bool blocking)
{
this.lockInstance = lockInstance;
this.isWriteLock = isWriteLock;
this.Enter(blocking);
}
private void Dispose()
{
if (m_ReadWriterLock != null)
{
m_ReadWriterLock.Dispose();
m_ReadWriterLock = null;
}
}
protected void Dispose(bool disposing)
{
// Check to see if Dispose has already been called.
if (!disposed)
{
m_itemLock.Dispose();
m_itemLock = null;
disposed = true;
}
}
/// <summary>
/// Lock our inventory list for writing (many can read, one can write)
/// </summary>
public void LockItemsForWrite(bool locked)
{
if (locked)
{
//Enter a write lock, wait indefinately for one to open.
if (m_itemLock.RecursiveReadCount > 0)
{
m_log.Error("[TaskInventoryDictionary] Recursive read lock requested. This should not happen and means something needs to be fixed. For now though, it's safe to continue.");
m_itemLock.ExitReadLock();
}
if (m_itemLock.RecursiveWriteCount > 0)
{
m_log.Error("[TaskInventoryDictionary] Recursive write lock requested. This should not happen and means something needs to be fixed.");
m_itemLock.ExitWriteLock();
}
while (!m_itemLock.TryEnterWriteLock(60000))
{
if (m_itemLock.IsWriteLockHeld)
{
m_log.Error("Thread lock detected while trying to aquire WRITE lock in TaskInventoryDictionary. Locked by thread " + LockedByThread.Name + ". I'm going to try to solve the thread lock automatically to preserve region stability, but this needs to be fixed.");
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("My call stack:\n" + Environment.StackTrace);
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("Locker's call stack:\n" + WriterStack);
// System.Console.WriteLine("------------------------------------------");
}
else
{
m_log.Error("Thread lock detected while trying to aquire WRITE lock in TaskInventoryDictionary. Locked by a reader. I'm going to try to solve the thread lock automatically to preserve region stability, but this needs to be fixed.");
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("My call stack:\n" + Environment.StackTrace);
// System.Console.WriteLine("------------------------------------------");
// foreach (KeyValuePair<Thread, string> kvp in ReadLockers)
// {
// System.Console.WriteLine("Locker name {0} call stack:\n" + kvp.Value, kvp.Key.Name);
// System.Console.WriteLine("------------------------------------------");
// }
}
m_itemLock = new System.Threading.ReaderWriterLockSlim();
// ReadLockers.Clear();
}
LockedByThread = Thread.CurrentThread;
// WriterStack = Environment.StackTrace;
}
else
{
if (m_itemLock.RecursiveWriteCount > 0)
{
m_itemLock.ExitWriteLock();
}
}
}
public static LockHandle Write(this ReaderWriterLockSlim self)
{
Check.Self(self);
if (self.IsWriteLockHeld)
{
return(new LockHandle());
}
self.EnterWriteLock();
return(new LockHandle(self, LockHandle.LockType.Write));
}
public static LockHandle UpgradeableRead(this ReaderWriterLockSlim self)
{
Check.Self(self);
if (self.IsUpgradeableReadLockHeld || self.IsWriteLockHeld)
{
return(new LockHandle());
}
self.EnterUpgradeableReadLock();
return(new LockHandle(self, LockHandle.LockType.UpgradeableRead));
}
internal static Thread.ReaderWriterLockSlim GetLock(object instance)
{
Thread.ReaderWriterLockSlim @lock;
instance = instance ?? StaticInstance;
if (!Locks.TryGetValue(instance, out @lock))
{
@lock = Locks[instance] = new Thread.ReaderWriterLockSlim();
}
return(@lock);
}
public DictionaryLocker()
: base()
{
if (dictRWLock != null)
{
throw new Exception();
}
dictRWLock = new System.Threading.ReaderWriterLockSlim();
count++;
if (count > 1)
{
throw new Exception();
}
}
public RegionInfoCache()
{
lock (m_creationLock)
{
if (m_rwLock == null)
{
starttimeS = Util.GetTimeStamp();
m_rwLock = new ReaderWriterLockSlim();
if (m_timer == null)
{
m_timer = new Timer(PurgeCache, null, CACHE_PURGE_TIME, Timeout.Infinite);
}
}
}
}
public void Dispose()
{
ReaderWriterLockSlim copy = Interlocked.Exchange(ref rwlock, null);
if (copy == null)
{
return;
}
switch (type)
{
case LockType.Read:
copy.ExitReadLock();
break;
case LockType.UpgradeableRead:
copy.ExitUpgradeableReadLock();
break;
case LockType.Write:
copy.ExitWriteLock();
break;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ReaderWriterLockSlim" /> class.
/// </summary>
public ReaderWriterLockSlim()
{
locker = new GlobalThreading.ReaderWriterLockSlim();
lockerLocal = true;
}
/// <summary>
/// Initializes a new instance of the <see cref="ReaderWriterLockSlim" /> class.
/// </summary>
/// <param name="lockRecursionPolicy">The lock recursion policy.</param>
public ReaderWriterLockSlim(LockRecursionPolicy lockRecursionPolicy)
{
locker = new GlobalThreading.ReaderWriterLockSlim(lockRecursionPolicy);
lockerLocal = true;
}
/// <summary> /// Initializes a new instance of the <see cref="ReaderWriterLockSlim" /> class. /// </summary> /// <param name="locker">The existing locker.</param> public ReaderWriterLockSlim(GlobalThreading.ReaderWriterLockSlim locker) => this.locker = locker;
/// <summary>
/// Threadeds the atomic method.
/// </summary>
/// <typeparam name="TValue">The type of the value.</typeparam>
/// <typeparam name="TKey">The type of the key.</typeparam>
/// <typeparam name="TList">The type of the list.</typeparam>
/// <param name="rwLock">The reader writer lock.</param>
/// <param name="list">The list.</param>
/// <param name="key">The key.</param>
/// <param name="method">The method.</param>
/// <returns></returns>
public static TValue ThreadedAtomicMethod <TValue, TKey, TList>(this ReaderWriterLockSlim rwLock, TList list, TKey key, Func <TValue> method)
where TList : IList <TKey>
{
// blocking calls
throw new NotImplementedException();
}
static void Main(string[] args)
{
rwls = new System.Threading.ReaderWriterLockSlim();
DELG Read1 = (state) =>
{
string thread = (string)state;
while (true)
{
rwls.EnterReadLock();
try
{
Console.WriteLine("Thread -> {0} -- Message -> {1}", thread, val.ToString());
}
finally
{
rwls.ExitReadLock();
}
System.Threading.Thread.Sleep(2000);
}
};
DELG Write1 = (state) =>
{
string thread = (string)state;
int[] tb = { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19 };
for (int i = 0; i < 10; i++)
{
rwls.EnterWriteLock();
try
{
val = tb[i];
Console.WriteLine("Changement de val par Thread -> {0}", thread);
}
finally
{
rwls.ExitWriteLock();
}
System.Threading.Thread.Sleep(3000);
}
};
DELG Write2 = (state) =>
{
string thread = (string)state;
int[] tb = { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 };
for (int i = 0; i < 10; i++)
{
rwls.EnterWriteLock();
try
{
val = tb[i];
Console.WriteLine("Changement de val par Thread -> {0}", thread);
}
finally
{
rwls.ExitWriteLock();
}
System.Threading.Thread.Sleep(3000);
}
};
System.Threading.Thread t1 = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(Read1.Invoke));
System.Threading.Thread t2 = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(Read1.Invoke));
System.Threading.Thread t3 = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(Write1.Invoke));
System.Threading.Thread t4 = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(Write1.Invoke));
t1.Start((object)("T1"));
t2.Start((object)("T2"));
t3.Start((object)("T3"));
t4.Start((object)("T4"));
Console.Read();
}
public async Task ProcessCrawlingQueueAsync(CrawlingQueue crawlingQueue)
{
_crawlingParameters.CancellationTokenSource.Token.Register(() =>
crawlingQueue.QueueCancellationTokenSource.Cancel()
);
var tasksLock = new System.Threading.ReaderWriterLockSlim();
var tasks = new HashSet <Task>();
var queueItemsProcessingSemaphore = new SemaphoreSlim(crawlingQueue.CrawlingConfiguration.MaxSimmultaneousQueueItemsProcessed / 2, crawlingQueue.CrawlingConfiguration.MaxSimmultaneousQueueItemsProcessed);
while (await queueItemsProcessingSemaphore.WaitAsync(crawlingQueue.CrawlingConfiguration.MaxTimeToProcessOneQueueItem))
{
if (crawlingQueue.QueueCancellationTokenSource.IsCancellationRequested)
{
await Task.WhenAll(tasks.ToArray());
// TODO: Move remaining items from local queue to the distributed queue
// TODO: Figure out how to filter out duplicates from the queue? Or should we?
// We will probably have to resort to known urls-based duplicates check
// Because otherwise we will drown in failing sql queries on multiplie machines
Trace.TraceWarning("ProcessCrawlingQueueAsync: Queue cancellation requested. Preventing dequeing of new elements. Processing will be shut down after currently executing items are complete.");
break;
}
var queueItem = await crawlingQueue.DequeueAsync();
if (queueItem == null) // Both Local and Proxy queues are depleted
{
// NOTE: If Queue is depleted, we must wait until all running tasks are executed, because they might add new items to queue
await Task.WhenAll(tasks.ToArray());
// wait for all queue proxies to complete fetching items
// TODO: consider locking (multithreading scenario)
var queueProxiesPending = crawlingQueue.QueueProxies.Where(queueProxy => queueProxy.IsPending()).ToArray();
if (queueProxiesPending.Length > 0)
{
continue;
}
if (crawlingQueue.LocalQueue.Count > 0)
{
continue;
}
break;
}
if (!await _crawlingEventInterceptorManager.OnAfterDequeueAsync(queueItem))
{
// If interceptor returns false, means it's an instruction to ignore this item;
continue;
}
tasksLock.EnterWriteLock();
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.Downloading);
tasks.Add(System.Threading.Tasks.TaskExtensions.Unwrap(
CrawlAsync(queueItem.ResourceLink)
.ContinueWith(async task =>
{
tasksLock.EnterWriteLock();
tasks.Remove(task); // to avoid infinite bloating of the collection
tasksLock.ExitWriteLock();
try
{
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.Downloaded);
if (task.Status == TaskStatus.RanToCompletion)
{
var resourceContentUnits = task.Result;
var httpResultUnit = resourceContentUnits.OfType <HttpResultUnit>().Single();
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.Processing);
var resourceContentUnitsProcessingCountdown = new AsyncCountdownEvent(resourceContentUnits.Count);
// Process resource content units extracted from Response
foreach (var resourceContentUnit in resourceContentUnits)
{
switch (resourceContentUnit)
{
case ExtractedLinksUnit extractedLinksUnit:
if (extractedLinksUnit.ExtractedLinks.Count > 0)
{
var linksProcessingCountdown = new AsyncCountdownEvent(extractedLinksUnit.ExtractedLinks.Count);
foreach (var extractedLink in extractedLinksUnit.ExtractedLinks)
{
var crawlingQueueItem = new CrawlingQueueItem(extractedLink);
// Do not enqueue item if prevented by any interceptor
if (!await _crawlingEventInterceptorManager.OnBeforeEnqueueAsync(crawlingQueueItem))
{
continue;
}
crawlingQueueItem.ProcessingCompleted += () =>
linksProcessingCountdown.AddCount(1)
;
crawlingQueue.Enqueue(crawlingQueueItem);
}
// Wait while all links are processed before releasing the content units semaphore and set Status = Processed for parent
linksProcessingCountdown.WaitAsync()
.ContinueWith(linksProcessingTask =>
resourceContentUnitsProcessingCountdown.AddCount(1)
);
}
else
{
resourceContentUnitsProcessingCountdown.AddCount(1);
}
// Set Processed status when all extracted links are processed
break;
case ExtractedDataUnit extractedDataUnit:
if (!await _crawlingEventInterceptorManager.OnDataDocumentDownloadedAsync(
queueItem.ResourceLink, // May be a DocumentLink, or a FrameLink. Not quite intuitive and probably requires redesign.
extractedDataUnit,
httpResultUnit
))
{
// If any of interceptors failed to process the download result,
// AND failed to store download result for later processing
// we must re-enqueue the item, in order to ensure the results are not lost for good
// We ignore the item and log the error. Chances are we couldn't process the item for a reason. And repeating would just make it stuck infinitely (re-downloading and re-processing)
// (WAS) we must re-enqueue the item, in order to ensure the results are not lost for good
//crawlingQueue.EnqueueAsync(queueItem);
}
resourceContentUnitsProcessingCountdown.Signal();
break;
case DownloadedFilesUnit downloadedFileUnit:
// If download file is a result of redirection,
// we must either explicitly declare that we're expecting a file, or throw a processing exception
var fileLink = queueItem.ResourceLink as FileLink;
if (fileLink == null)
{
Trace.TraceError($"ProcessCrawlingQueueAsync: Downloaded file unit. Resource link is of type {queueItem.ResourceLink.GetType().Name}, expecting FileLink. Preventing processing.");
break;
}
if (!await _crawlingEventInterceptorManager.OnFileDownloadedAsync(
fileLink,
downloadedFileUnit,
httpResultUnit
))
{
// If any of interceptors failed to process the download result,
// AND failed to store download result for later processing....
// We ignore the item and log the error. Chances are we couldn't process the item for a reason. And repeating would just make it stuck infinitely (re-downloading and re-processing)
// (WAS) we must re-enqueue the item, in order to ensure the results are not lost for good
//crawlingQueue.EnqueueAsync(queueItem);
}
resourceContentUnitsProcessingCountdown.Signal();
break;
case HttpResultUnit httpResultUnitStub:
// TODO: Determine what we should do if HTTP download failed. Either re-enqueue or ignore, or alert/do something else
switch (httpResultUnitStub.HttpStatus)
{
//case HttpStatusCode.InternalServerError: // it's likely to repeat within the same run
case HttpStatusCode.GatewayTimeout:
case HttpStatusCode.RequestTimeout:
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.NotLinked);
crawlingQueue.Enqueue(queueItem); // Trying to recrawl item if it failed for some intermitent reason
break;
default:
// We need to invoke ProcessingCompleted only after Data and Links extracted are really processed.
//queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.ProcessingCompleted);
break;
}
resourceContentUnitsProcessingCountdown.Signal();
break;
default:
throw new NotSupportedException();
}
}
// Do not actually wait for related resources processing completion.
// Those might be extracted links or files. No need to hold queue resources while linked units are downloaded
// Set Processed status after all content units were registered and interceptors triggered
await resourceContentUnitsProcessingCountdown.WaitAsync()
.ContinueWith(resourceContentUnitsProcessingTask =>
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.Processed)
);
}
else
{
Trace.TraceError("CrawlAsync: Failed for queue item {0} with exception [{1}]", queueItem.ResourceLink, task.Exception);
}
}
finally
{
queueItemsProcessingSemaphore.Release();
}
})
)
);
tasksLock.ExitWriteLock();
}
await Task.WhenAll(tasks.ToArray());
}
internal LockHandle(ReaderWriterLockSlim rwlock, LockType type)
{
this.rwlock = rwlock;
this.type = type;
}
/// <summary>
/// Lock our inventory list for reading (many can read, one can write)
/// </summary>
public void LockItemsForRead(bool locked)
{
if (locked)
{
if (m_itemLock.IsWriteLockHeld && LockedByThread != null)
{
if (!LockedByThread.IsAlive)
{
//Locked by dead thread, reset.
m_itemLock = new System.Threading.ReaderWriterLockSlim();
}
}
if (m_itemLock.RecursiveReadCount > 0)
{
m_log.Error("[TaskInventoryDictionary] Recursive read lock requested. This should not happen and means something needs to be fixed. For now though, it's safe to continue.");
try
{
// That call stack is useful for end users only. RealProgrammers need a full dump. Commented.
// StackTrace stackTrace = new StackTrace(); // get call stack
// StackFrame[] stackFrames = stackTrace.GetFrames(); // get method calls (frames)
//
// // write call stack method names
// foreach (StackFrame stackFrame in stackFrames)
// {
// m_log.Error("[SceneObjectGroup.m_parts] "+(stackFrame.GetMethod().Name)); // write method name
// }
// The below is far more useful
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("My call stack:\n" + Environment.StackTrace);
// System.Console.WriteLine("------------------------------------------");
// foreach (KeyValuePair<Thread, string> kvp in ReadLockers)
// {
// System.Console.WriteLine("Locker name {0} call stack:\n" + kvp.Value, kvp.Key.Name);
// System.Console.WriteLine("------------------------------------------");
// }
}
catch
{ }
m_itemLock.ExitReadLock();
}
if (m_itemLock.RecursiveWriteCount > 0)
{
m_log.Error("[TaskInventoryDictionary] Recursive write lock requested. This should not happen and means something needs to be fixed.");
// try
// {
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("My call stack:\n" + Environment.StackTrace);
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("Locker's call stack:\n" + WriterStack);
// System.Console.WriteLine("------------------------------------------");
// }
// catch
// {}
m_itemLock.ExitWriteLock();
}
while (!m_itemLock.TryEnterReadLock(60000))
{
m_log.Error("Thread lock detected while trying to aquire READ lock in TaskInventoryDictionary. Locked by thread " + LockedByThread.Name + ". I'm going to try to solve the thread lock automatically to preserve region stability, but this needs to be fixed.");
//if (m_itemLock.IsWriteLockHeld)
//{
m_itemLock = new System.Threading.ReaderWriterLockSlim();
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("My call stack:\n" + Environment.StackTrace);
// System.Console.WriteLine("------------------------------------------");
// System.Console.WriteLine("Locker's call stack:\n" + WriterStack);
// System.Console.WriteLine("------------------------------------------");
// LockedByThread = null;
// ReadLockers.Clear();
//}
}
// ReadLockers[Thread.CurrentThread] = Environment.StackTrace;
}
else
{
if (m_itemLock.RecursiveReadCount > 0)
{
m_itemLock.ExitReadLock();
}
// if (m_itemLock.RecursiveReadCount == 0)
// ReadLockers.Remove(Thread.CurrentThread);
}
}
/// <summary>
/// Converts the source <see cref="GlobalThreading.ReaderWriterLockSlim" /> to a <see cref="IReaderWriterLock" />
/// abstraction.
/// </summary>
/// <param name="source">The source locker.</param>
/// <returns>The abstracted version of the same event.</returns>
/// <exception cref="ArgumentNullException">
/// <paramref name="source" /> is <see langword="null" /> (
/// <see langword="Nothing" /> in Visual Basic).
/// </exception>
public static IReaderWriterLock AsAbstraction(this GlobalThreading.ReaderWriterLockSlim source) =>
new ReaderWriterLockSlim(
Requires.NotNull(
source,
nameof(source)));