public BitBytBitThreeFishPermutationTest(ConcurrentQueue <TestTask> tasks)
{
task = new TestTask("BitBytBitThreeFishPermutationTest", StartTests);
tasks.Enqueue(task);
sources = SourceTask.GetIterator();
}
public BitBytBitShuffleTest(ConcurrentQueue <TestTask> tasks, string TableName, int permutationCount)
{
task = new TestTask("BitBytBitShuffleTest " + TableName + " * " + permutationCount, StartTests);
tasks.Enqueue(task);
sources = SourceTask.GetIterator(task);
this.TableName = TableName;
this.permutationCount = permutationCount;
}
public void SourceTaskTest()
{
var runner = new BackgroundTaskRunner <IBackgroundTask>(new BackgroundTaskRunnerOptions {
KeepAlive = true, LongRunning = true
}, _logger);
var task = new SourceTask();
runner.Add(task);
Assert.IsTrue(runner.IsRunning);
Console.WriteLine("completing");
task.Complete(); // in Deploy this does not return ffs - no point until I cannot repro
Console.WriteLine("completed");
Console.WriteLine("done");
}
private void testN3(SourceTask task)
{
for (int cst = 1; cst < 11 && cst < task.Size; cst++)
{
using (var bbs = new BytesBuilderStatic(task.Size))
{
var bbp = new BytesBuilderForPointers();
var bbb = new BytesBuilder();
var allocator = new BytesBuilderForPointers.AllocHGlobal_AllocatorForUnsafeMemory();
var fix = new BytesBuilderForPointers.Fixed_AllocatorForUnsafeMemory();
int size = task.Size;
{
BytesBuilderForPointers.Record res_bbs = null, res_bbp = null, res_bbb = null;
try
{
try
{
byte V = 1;
var r = allocator.AllocMemory(size - 1);
// var r2 = allocator.AllocMemory(size >> 1);
while (bbb.Count < size - 1)
{
for (int i = 0; i < r.len; i++)
{
r.array[i] = V++;
}
bbs.add(r);
bbp.addWithCopy(r, r.len, allocator);
for (int i = 0; i < r.len; i++)
{
bbb.addByte(r.array[i]);
}
}
bbs.add(r.array, 1);
bbp.addWithCopy(r.array, 1, allocator);
bbb.addByte(r.array[0]);
r.Dispose();
r = null;
int remainder = size;
do
{
remainder -= cst;
try
{
res_bbs = allocator.AllocMemory(cst);
bbs.getBytesAndRemoveIt(res_bbs);
if (bbs.Count != remainder)
{
throw new Exception("Test_BytesBuilder.testN: size = " + task.Size + ", bbs.Count > 1: " + bbs.Count);
}
res_bbp = allocator.AllocMemory(cst);
bbp.getBytesAndRemoveIt(res_bbp);
if (bbp.Count != remainder)
{
throw new Exception("Test_BytesBuilder.testN: size = " + task.Size + ", bbp.Count > 1: " + bbp.Count);
}
res_bbb = fix.FixMemory(bbb.getBytesAndRemoveIt(resultCount: cst));
if (bbb.Count != remainder)
{
throw new Exception("Test_BytesBuilder.testN: size = " + task.Size + ", bbb.Count > 1: " + bbb.Count);
}
if (!BytesBuilderForPointers.isArrayEqual_Secure(res_bbs, res_bbp) || !BytesBuilderForPointers.isArrayEqual_Secure(res_bbs, res_bbb))
{
throw new Exception("Test_BytesBuilder.testN: size = " + task.Size + " unequal");
}
}
finally
{
res_bbs?.Dispose();
res_bbp?.Dispose();
res_bbb?.Dispose();
res_bbs = null;
res_bbp = null;
res_bbb = null;
}
try
{
res_bbs = bbs.getBytes();
res_bbp = bbp.getBytes();
res_bbb = fix.FixMemory(bbb.getBytes());
if (!BytesBuilderForPointers.isArrayEqual_Secure(res_bbs, res_bbp) || !BytesBuilderForPointers.isArrayEqual_Secure(res_bbs, res_bbb))
{
throw new Exception("Test_BytesBuilder.testN: size = " + task.Size + " unequal [2]");
}
}
finally
{
res_bbs?.Dispose();
res_bbp?.Dispose();
res_bbb?.Dispose();
res_bbs = null;
res_bbp = null;
res_bbb = null;
}
}while (bbs.Count > cst);
}
finally
{
bbs.Clear();
bbp.Clear();
bbb.Clear();
}
}
finally
{
res_bbs?.Dispose();
res_bbp?.Dispose();
res_bbb?.Dispose();
res_bbs = null;
res_bbp = null;
res_bbb = null;
}
}
}
}
}
private void testN1(SourceTask task)
{
using (var bbs = new BytesBuilderStatic(task.Size))
{
var bbp = new BytesBuilderForPointers();
var bbb = new BytesBuilder();
var allocator = new BytesBuilderForPointers.AllocHGlobal_AllocatorForUnsafeMemory();
var fix = new BytesBuilderForPointers.Fixed_AllocatorForUnsafeMemory();
for (int size = 1; size <= task.Size; size++)
{
BytesBuilderForPointers.Record res_bbs = null, res_bbb = null, res_bbp = null, res_bbs2 = null, r = null;
try
{
try
{
byte V = 1;
r = allocator.AllocMemory(size);
while (bbb.Count + size <= size)
{
for (int i = 0; i < r.len; i++)
{
r.array[i] = V++;
}
bbs.add(r);
bbp.addWithCopy(r, r.len, allocator);
for (int i = 0; i < r.len; i++)
{
bbb.addByte(r.array[i]);
}
}
r.Dispose();
res_bbs = bbs.getBytes();
res_bbp = bbp.getBytes();
res_bbb = fix.FixMemory(bbb.getBytes());
bbs.Resize(bbs.size + 1);
res_bbs2 = bbs.getBytes();
}
finally
{
bbs.Clear();
bbp.Clear();
bbb.Clear();
}
if (!BytesBuilderForPointers.isArrayEqual_Secure(res_bbs, res_bbp) || !BytesBuilderForPointers.isArrayEqual_Secure(res_bbs, res_bbb) || !BytesBuilderForPointers.isArrayEqual_Secure(res_bbs2, res_bbb))
{
throw new Exception("Test_BytesBuilder.testN: size = " + task.Size + " unequal");
}
}
finally
{
res_bbs?.Dispose();
res_bbp?.Dispose();
res_bbb?.Dispose();
res_bbs2?.Dispose();
}
}
}
}
private void test0(SourceTask task)
{
try
{
var t8 = new byte[8];
ulong data = 0xA5A5_B5A5_A5A5_B5B5;
BytesBuilder.ULongToBytes(data, ref t8);
BytesBuilder.BytesToULong(out ulong dt, t8, 0);
if (dt != data)
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: ULongToBytes != BytesToULong"
};
this.task.error.Add(e);
}
BytesBuilder.BytesToUInt(out uint dt4, t8, 0);
if (dt4 != (uint)data)
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: ULongToBytes/4 != BytesToUInt"
};
this.task.error.Add(e);
}
BytesBuilder.BytesToUInt(out dt4, t8, 4);
if (dt4 != (uint)(data >> 32))
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: ULongToBytes>>/4 != BytesToUInt"
};
this.task.error.Add(e);
}
var fa = new BytesBuilderForPointers.Fixed_AllocatorForUnsafeMemory();
var b1 = new byte[8];
var r1 = fa.FixMemory(b1);
var r2 = fa.FixMemory(t8);
if (BytesBuilderForPointers.isArrayEqual_Secure(r1, r2))
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: isArrayEqual_Secure(r1, r2)"
};
this.task.error.Add(e);
}
BytesBuilder.CopyTo(t8, b1, 0, 7);
if (BytesBuilderForPointers.isArrayEqual_Secure(r1, r2))
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: isArrayEqual_Secure(r1, r2) [2]"
};
this.task.error.Add(e);
}
b1[7] = t8[7];
if (!BytesBuilderForPointers.isArrayEqual_Secure(r1, r2))
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: isArrayEqual_Secure(r1, r2) [3]"
};
this.task.error.Add(e);
}
r1.Clear();
BytesBuilderForPointers.BytesToULong(out dt, r2, 0, 8);
if (dt != data)
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: ULongToBytes != BytesToULong (BytesBuilderForPointers)"
};
this.task.error.Add(e);
}
r1.Dispose();
r2.Dispose();
using (new BytesBuilderStatic(task.Size))
{
var e = new Error()
{
Message = "Test_BytesBuilder.test0: нет исключения"
};
this.task.error.Add(e);
}
}
catch (ArgumentOutOfRangeException)
{}
catch (Exception ex)
{
var e = new Error()
{
ex = ex, Message = "Test_BytesBuilder.test0: exception " + ex.Message
};
this.task.error.Add(e);
}
}
/// <summary>
/// 抓取详情页
/// </summary>
/// <param name="token"></param>
async void GrabDetailPagesTaskThreadEntry(CancellationToken token)
{
//AppendLogWarning(txtLog, "[图片来源检测] 正在加载来源数据...");
var client = new HttpClient(); //_context.Session.NetClient;
var data = Service.ImageDownload.TaskContext.Instance.Data;
////这里创建了一个 CancellationTokenSource 的局部变量，主要是为了在循环中对请求也能进行中断
//CancellationTokenSource tcs = new CancellationTokenSource();
//token.Register(() => tcs?.Cancel());
SourceTask task = data.CurrentSourceTask;
while (!token.IsCancellationRequested)
{
#region 站点数据
if (task == null)
{
//检查下载队列，看是否有没有检测的站点……
lock (data.WaitForProcessSourceTasks)
{
if (data.WaitForProcessSourceTasks.Count > 0)
{
data.CurrentSourceTask = task = data.WaitForProcessSourceTasks.Dequeue();
}
}
}
//没有或已经下载过的话，则休息后重新检查
if (task == null)
{
AppendLogWarning(txtLog, "[来源站点] 暂无待处理站点,5秒后重新检测...");
//等待5秒再去检测
Thread.Sleep(5000);
//tcs = null;
continue;
}
AppendLog(txtLog, "[来源站点] 开始处理站点：{0}...", task.Domain);
#endregion
#region 处理页面
var hasMore = true;
var page = 1;
string urlFormat = task.PageFormat;
while (hasMore)
{
if (page % 100 == 0)
{
AppendLog(txtLog, "[调试] 当前域{0} 页面{1}", task.Domain, urlFormat.FormatWith(page));
}
AppendLog(txtLog, "[页面列表] 正在加载站点：{0} 的第 {1} 页...", task.Domain, page);
var ctx = client.Create <string>(HttpMethod.Get, urlFormat.FormatWith(page), allowAutoRedirect: true);
await ctx.SendAsync();
if (!ctx.IsValid())
{
//失败 10秒后重试
AppendLog(txtLog, "[页面列表] 站点：{0} 的第 {1} 页下载失败，稍后重试", task.Domain, page);
Thread.Sleep(new TimeSpan(0, 0, 10));
}
else
{
//下载成功，获得列表
var matches = Regex.Matches(ctx.Result, task.PageListPattern, RegexOptions.Singleline | RegexOptions.IgnoreCase);
//新的任务
var newTasks = matches.Cast <Match>()
.Select(s => new PageTask(s.Groups[2].Value, s.Groups[1].Value, task.ImagePattern))
.Where(s => !data.PageDownloaded.ContainsKey(s.Url) && !data.WaitForDownloadPageTasks.Any(_ => _.Url == s.Url))
.ToArray();
if (newTasks.Length > 0)
{
lock (data.WaitForDownloadPageTasks)
{
newTasks.ForEach(s =>
{
data.WaitForDownloadPageTasks.Enqueue(s);
});
}
AppendLog(txtLog, "[页面列表] 站点：{0} 的第 {1} 页已建立 {2} 个新任务到队列中...", task.Domain, page, newTasks.Length);
}
else
{
AppendLog(txtLog, "[页面列表] 站点：{0} 的第 {1} 页没有检测到新任务...", task.Domain, page);
}
UpdatePageDetailGrabStatus();
//如果没有下一页，则中止
if (!Regex.IsMatch(ctx.Result, task.NextPagePattern, RegexOptions.IgnoreCase))
{
AppendLog(txtLog, "[页面列表] 站点：{0} 没有更多新纪录，尝试检查其他站点...", task.Url);
hasMore = false;
break;
}
//等待2秒继续
Thread.Sleep(2000);
page++;
}
}
#endregion
//存放来源到已处理列表
lock (data.SourceProcessed)
{
data.SourceProcessed.Add(task.Domain, task);
data.CurrentSourceTask = task = null;
}
}
}