public void Test5(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
DateTime now = DateTime.Now;
crawlQueue.Push(new CrawlerQueueEntry
{
CrawlStep = new CrawlStep(new Uri("http://www.biz.org/"), 0),
Properties = new Dictionary <string, object>
{
{ "one", "string" },
{ "two", 123 },
{ "three", now },
},
Referrer = new CrawlStep(new Uri("http://www.biz3.org/"), 1)
});
Assert.AreEqual(1, crawlQueue.Count);
CrawlerQueueEntry entry = crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
Assert.NotNull(entry);
Assert.NotNull(entry.CrawlStep);
Assert.NotNull(entry.Properties);
Assert.NotNull(entry.Referrer);
Assert.AreEqual(0, entry.CrawlStep.Depth);
Assert.AreEqual("http://www.biz.org/", entry.CrawlStep.Uri.ToString());
Assert.AreEqual("one", entry.Properties.Keys.First());
Assert.AreEqual("two", entry.Properties.Keys.Skip(1).First());
Assert.AreEqual("three", entry.Properties.Keys.Skip(2).First());
Assert.AreEqual("string", entry.Properties["one"]);
Assert.AreEqual(123, entry.Properties["two"]);
Assert.AreEqual(now, entry.Properties["three"]);
Assert.AreEqual(0, crawlQueue.Count);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
var data = crawlerQueueEntry.ToJson();
string fileName = Path.Combine(m_StoragePath, Guid.NewGuid().ToString());
File.WriteAllText(fileName, data);
Interlocked.Increment(ref m_Count);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
var data = crawlerQueueEntry.ToJson();
var fileName = Path.Combine(this.m_StoragePath, Guid.NewGuid().ToString());
File.WriteAllText(fileName, data);
Interlocked.Increment(ref this.m_Count);
}
public CrawlerQueueEntry Pop()
{
var qt = this._queue.Pop();
CrawlerQueueEntry obj = null;
if (qt != null)
{
obj = qt.ToCrawlerQueueEntry();
}
return(obj);
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_StackLock).
Do(() =>
{
if (!m_Stack.Contains(crawlerQueueEntry))
{
m_Stack.Push(crawlerQueueEntry);
}
});
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_StackLock).
Do(() =>
{
if (!m_Stack.Contains(crawlerQueueEntry))
{
m_Stack.Push(crawlerQueueEntry);
}
});
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_CrawlQueueLock).
Do <NCrawlerEntitiesSQLite>(e =>
{
CrawlQueue crawlQueueEntry = CrawlQueue.CreateCrawlQueue(0, m_GroupId);
crawlQueueEntry.SerializedData = crawlerQueueEntry.ToBinary();
e.AddToCrawlQueues(crawlQueueEntry);
e.SaveChanges();
});
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_CrawlQueueLock).
Do<NCrawlerEntitiesSQLite>(e =>
{
CrawlQueue crawlQueueEntry = CrawlQueue.CreateCrawlQueue(0, m_GroupId);
crawlQueueEntry.SerializedData = crawlerQueueEntry.ToBinary();
e.AddToCrawlQueues(crawlQueueEntry);
e.SaveChanges();
});
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
Do<NCrawlerEntitiesDbServices>(e =>
{
e.AddToCrawlQueue(new CrawlQueue
{
GroupId = m_GroupId,
SerializedData = crawlerQueueEntry.ToBinary(),
});
e.SaveChanges();
});
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
NotNull(crawlerQueueEntry, "crawlerQueueEntry").
WriteLock(m_QueueLock).
Do(() =>
{
byte[] data = crawlerQueueEntry.ToBinary();
string fileName = Path.Combine(m_StoragePath, Guid.NewGuid().ToString());
File.WriteAllBytes(fileName, data);
});
Interlocked.Increment(ref m_Count);
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
NotNull(crawlerQueueEntry, "crawlerQueueEntry").
WriteLock(m_QueueLock).
Do(() =>
{
byte[] data = crawlerQueueEntry.ToBinary();
string fileName = Path.Combine(m_StoragePath, Guid.NewGuid().ToString());
File.WriteAllBytes(fileName, data);
});
Interlocked.Increment(ref m_Count);
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_CrawlHistoryLock).
Do <NCrawlerEntitiesDbServices>(e =>
{
e.AddToCrawlQueue(new CrawlQueue
{
GroupId = m_GroupId,
SerializedData = crawlerQueueEntry.ToBinary(),
});
e.SaveChanges();
});
}
/// <summary>
/// Push item in the queue.
/// </summary>
/// <param name="crawlerQueueEntry">Queue entry.</param>
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
using (var model = new NCrawlerModel())
{
var entry = new CrawlQueue
{
GroupId = this.groupId,
SerializedData = crawlerQueueEntry.ToJson(),
};
model.CrawlQueues.Add(entry);
model.SaveChanges();
}
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_CrawlHistoryLock).
Do<NCrawlerEntitiesDbServices>(e =>
{
e.AddToCrawlQueue(new CrawlQueue
{
GroupId = m_GroupId,
SerializedData = crawlerQueueEntry.ToBinary(),
});
e.SaveChanges();
});
}
/// <summary>
/// Push item in the queue.
/// </summary>
/// <param name="crawlerQueueEntry">Queue entry.</param>
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
using (var model = new NCrawlerModel())
{
var entry = new CrawlQueue
{
GroupId = this.groupId,
SerializedData = crawlerQueueEntry.ToJson(),
};
model.CrawlQueues.Add(entry);
model.SaveChanges();
}
}
public CrawlerQueueEntry ToCrawlerQueueEntry()
{
CrawlerQueueEntry ce = new CrawlerQueueEntry
{
Properties = Properties,
CrawlStep = new CrawlStep(new Uri(StepUri), StepDepth)
};
if (ReferrerUri != null)
{
ce.Referrer = new CrawlStep(new Uri(ReferrerUri), ReferrerDepth);
}
return(ce);
}
public CrawlerQueueEntry ToCrawlerQueueEntry()
{
var ce = new CrawlerQueueEntry
{
Properties = this.Properties,
CrawlStep = new CrawlStep(new Uri(this.StepUri), this.StepDepth)
};
if (this.ReferrerUri != null)
{
ce.Referrer = new CrawlStep(new Uri(this.ReferrerUri), this.ReferrerDepth);
}
return(ce);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
Do<NCrawlerEntitiesDbServices>(e =>
{
string storeKey = Uri.UnescapeDataString(crawlerQueueEntry.CrawlStep.Uri.ToString());
e.AddToCrawlQueue(new CrawlQueue
{
GroupId = m_GroupId,
SerializedData = crawlerQueueEntry.ToBinary(),
Key = storeKey
});
e.SaveChanges();
});
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
var data = crawlerQueueEntry.ToJson();
string path = Path.Combine(WorkFolderPath, Guid.NewGuid().ToString());
using (var isoFile = new IsolatedStorageFileStream(path, FileMode.Create, m_Store))
{
using (var writer = new StreamWriter(isoFile))
{
writer.WriteLine(data);
}
}
Interlocked.Increment(ref m_Count);
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
NotNull(crawlerQueueEntry, "crawlerQueueEntry").
WriteLock(m_QueueLock).
Do(() =>
{
byte[] data = crawlerQueueEntry.ToBinary();
string path = Path.Combine(WorkFolderPath, Guid.NewGuid().ToString());
using (IsolatedStorageFileStream isoFile = new IsolatedStorageFileStream(path, FileMode.Create, m_Store))
{
isoFile.Write(data, 0, data.Length);
}
});
Interlocked.Increment(ref m_Count);
}
public static Entry FromCrawlerQueueEntry(CrawlerQueueEntry ce)
{
var e = new Entry
{
StepUri = ce.CrawlStep.Uri.ToString(),
StepDepth = ce.CrawlStep.Depth,
Properties = ce.Properties
};
if (ce.Referrer != null)
{
e.ReferrerUri = ce.Referrer.Uri.ToString();
e.ReferrerDepth = ce.Referrer.Depth;
}
return(e);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
this.m_EsentInstance.Cursor((session, dbid) =>
{
using (var transaction = new Transaction(session))
{
using (var table = new Table(session, dbid, EsentTableDefinitions.QueueTableName, OpenTableGrbit.None))
{
using (var update = new Update(session, table, JET_prep.Insert))
{
Api.SetColumn(session, table, this.dataColumn.columnid, crawlerQueueEntry.ToJson(), Encoding.Unicode);
update.Save();
}
}
using (var table = new Table(session, dbid, EsentTableDefinitions.GlobalsTableName, OpenTableGrbit.None))
{
Api.EscrowUpdate(session, table, this.queueCountColumn.columnid, 1);
}
transaction.Commit(CommitTransactionGrbit.None);
}
});
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
var data = crawlerQueueEntry.ToJson();
string path = Path.Combine(WorkFolderPath, Guid.NewGuid().ToString());
using (var isoFile = new IsolatedStorageFileStream(path, FileMode.Create, m_Store))
{
using (var writer = new StreamWriter(isoFile))
{
writer.WriteLine(data);
}
}
Interlocked.Increment(ref m_Count);
}
public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry)
{
m_ThreadSafeCounter = threadSafeCounter;
CrawlerQueueEntry = crawlerQueueEntry;
}
public ThreadSafeCounterCookie EnterCounterScope(CrawlerQueueEntry crawlerQueueEntry)
{
Increment();
return(new ThreadSafeCounterCookie(this, crawlerQueueEntry));
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
byte[] data = crawlerQueueEntry.ToBinary();
string path = Path.Combine(WorkFolderPath, Guid.NewGuid().ToString());
using (IsolatedStorageFileStream isoFile = new IsolatedStorageFileStream(path, FileMode.Create, m_Store))
{
isoFile.Write(data, 0, data.Length);
}
Interlocked.Increment(ref m_Count);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
m_EsentInstance.Cursor((session, dbid) =>
{
using (Transaction transaction = new Transaction(session))
{
using (Table table = new Table(session, dbid, EsentTableDefinitions.QueueTableName, OpenTableGrbit.None))
{
using (Update update = new Update(session, table, JET_prep.Insert))
{
Api.SetColumn(session, table, dataColumn.columnid, crawlerQueueEntry.ToBinary());
update.Save();
}
}
using (Table table = new Table(session, dbid, EsentTableDefinitions.GlobalsTableName, OpenTableGrbit.None))
{
Api.EscrowUpdate(session, table, queueCountColumn.columnid, 1);
}
transaction.Commit(CommitTransactionGrbit.None);
}
});
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(this.m_QueueLock).
Do(() => PushImpl(crawlerQueueEntry));
}
public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry)
{
m_ThreadSafeCounter = threadSafeCounter;
CrawlerQueueEntry = crawlerQueueEntry;
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
WriteLock(m_QueueLock).
Do(() => PushImpl(crawlerQueueEntry));
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
m_Stack.Push(crawlerQueueEntry);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
m_Stack.Push(crawlerQueueEntry);
}
protected override void PushImpl(CrawlerQueueEntry crawlerQueueEntry)
{
m_Db.Store(crawlerQueueEntry);
}
public ThreadSafeCounterCookie EnterCounterScope(CrawlerQueueEntry crawlerQueueEntry)
{
Increment();
return new ThreadSafeCounterCookie(this, crawlerQueueEntry);
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
this._queue.Add(Entry.FromCrawlerQueueEntry(crawlerQueueEntry));
}
public static Entry FromCrawlerQueueEntry(CrawlerQueueEntry ce)
{
Entry e = new Entry
{
StepUri = ce.CrawlStep.Uri.ToString(),
StepDepth = ce.CrawlStep.Depth,
Properties = ce.Properties
};
if (ce.Referrer != null)
{
e.ReferrerUri = ce.Referrer.Uri.ToString();
e.ReferrerDepth = ce.Referrer.Depth;
}
return e;
}
protected abstract void PushImpl(CrawlerQueueEntry crawlerQueueEntry);
protected abstract void PushImpl(CrawlerQueueEntry crawlerQueueEntry);
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
_queue.Add(Entry.FromCrawlerQueueEntry(crawlerQueueEntry));
}
public void Push(CrawlerQueueEntry crawlerQueueEntry)
{
AspectF.Define.
NotNull(crawlerQueueEntry, "crawlerQueueEntry").
WriteLock(m_QueueLock).
Do(() =>
{
byte[] data = crawlerQueueEntry.ToBinary();
string path = Path.Combine(WorkFolderPath, Guid.NewGuid().ToString());
using (IsolatedStorageFileStream isoFile = new IsolatedStorageFileStream(path, FileMode.Create, m_Store))
{
isoFile.Write(data, 0, data.Length);
}
});
Interlocked.Increment(ref m_Count);
}
public CrawlerQueueEntry ToCrawlerQueueEntry()
{
CrawlerQueueEntry ce = new CrawlerQueueEntry
{
Properties = Properties,
CrawlStep = new CrawlStep(new Uri(StepUri), StepDepth)
};
if (ReferrerUri != null)
{
ce.Referrer = new CrawlStep(new Uri(ReferrerUri), ReferrerDepth);
}
return ce;
}
