public static void ParallelAsync_ForEach_Func_Delay(int?maxDop)
{
var options = maxDop.HasValue ? new ParallelOptions {
MaxDegreeOfParallelism = maxDop.Value
} : null;
var sw = new Stopwatch();
sw.Start();
var actual = 0;
Func <int, Task <KeyValuePair <int, int> > > func = async n =>
{
Interlocked.Increment(ref actual);
await Task.Delay(delay);
return(new KeyValuePair <int, int>(n, n * 2));
};
var result = ParallelAsync.ForEachAsync(Enumerable.Range(0, loops), options, func).Result;
sw.Stop();
Assert.Equal(loops, actual);
Assert.True(sw.ElapsedMilliseconds < delay * loops); // Environmental factors mean we can't assert a lower boundary
}
public virtual async ValueTask <IReadOnlyDictionary <Sha1, ReadOnlyMemory <byte> > > ReadObjectBatchAsync(IEnumerable <Sha1> objectIds, CancellationToken cancellationToken)
{
if (objectIds == null)
{
return(ReadOnlyDictionary.Empty <Sha1, ReadOnlyMemory <byte> >());
}
var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = MaxDop,
CancellationToken = cancellationToken
};
// Enumerate batches
var dict = new ConcurrentDictionary <Sha1, ReadOnlyMemory <byte> >(Sha1Comparer.Default);
await ParallelAsync.ForEachAsync(objectIds, parallelOptions, async sha1 =>
{
// Execute batch
var buffer = await ReadObjectAsync(sha1, cancellationToken).ConfigureAwait(false);
if (buffer.HasValue)
{
dict[sha1] = buffer.Value;
}
}).ConfigureAwait(false);
return(dict);
}
public static void ParallelAsync_ForEach_Action_Default_Arguments(int?maxDop)
{
var data = new int[] { 0, 1, 2 };
var options = maxDop.HasValue ? new ParallelOptions {
MaxDegreeOfParallelism = maxDop.Value
} : null;
// Null body
Func <int, Task> action = null;
Assert.ThrowsAsync <ArgumentNullException>(() => ParallelAsync.ForEachAsync(data, options, action));
var actual = 0;
action = n =>
{
Interlocked.Increment(ref actual);
return(Task.CompletedTask);
};
// Null source
actual = 0;
ParallelAsync.ForEachAsync(null, options, action).Wait();
Assert.Equal(0, actual);
// Empty source
actual = 0;
ParallelAsync.ForEachAsync(Array.Empty <int>(), options, action).Wait();
Assert.Equal(0, actual);
// Null options
actual = 0;
ParallelAsync.ForEachAsync(data, options, action).Wait();
Assert.Equal(data.Length, actual);
}
public static void ParallelAsync_ForEach_Func_Default_Arguments(int?maxDop)
{
var data = new int[] { 0, 1, 2 };
var options = maxDop.HasValue ? new ParallelOptions {
MaxDegreeOfParallelism = maxDop.Value
} : null;
// Null body
Func <int, Task <KeyValuePair <int, int> > > func = null;
Assert.ThrowsAsync <ArgumentNullException>(async() => await ParallelAsync.ForEachAsync(data, options, func));
var actual = 0;
func = n =>
{
Interlocked.Increment(ref actual);
return(Task.FromResult(new KeyValuePair <int, int>(n, n)));
};
// Null source
actual = 0;
var result = ParallelAsync.ForEachAsync(null, options, func).Result;
Assert.Equal(0, actual);
Assert.Empty(result);
// Empty source
actual = 0;
result = ParallelAsync.ForEachAsync(Array.Empty <int>(), options, func).Result;
Assert.Equal(0, actual);
Assert.Empty(result);
}
public async Task SetResult_ReturnsResult()
{
var result = await ParallelAsync.ForEachAsync <int, int>(new int[] { 0 }, async (item, controller) =>
{
controller.ProvideResult(item);
}).ConfigureAwait(false);
result.Should().Be(0);
}
private bool Download(HttpClientEx hc)
{
using (var stopSlim = new ManualResetEventSlim(false))
{
this.m_downloaded = 0;
var updateTask = Task.Factory.StartNew(() =>
{
var startTime = DateTime.Now;
double befSpeed = 0;
while (!stopSlim.IsSet)
{
befSpeed = (befSpeed + Interlocked.Exchange(ref this.m_downloaded, 0) / (DateTime.Now - startTime).TotalSeconds) / 2;
if (double.IsNaN(befSpeed))
{
befSpeed = 0;
}
this.SpeedOrFileSize = Utility.ToEICFormat(befSpeed, "/s");
Thread.Sleep(500);
}
});
var parallelOption = new ParallelOptions
{
MaxDegreeOfParallelism = 8,
};
using (var cts = new CancellationTokenSource())
{
ParallelAsync.ForEachAsync(
this.m_images,
async e => await this.DownloadImage(e, hc, cts.Token, cts.Cancel),
8).GetAwaiter().GetResult();
stopSlim.Set();
updateTask.Wait();
if (!this.IgnoreErrorMissingPage && cts.IsCancellationRequested)
{
return(false);
}
}
}
this.SpeedOrFileSize = null;
// 모든 이미지가 다운로드가 완료되어야 함
return(this.IgnoreErrorMissingPage || this.m_images.All(e => e.Extension != null));
}
public static void ParallelAsync_ForEach_Func(int?maxDop)
{
var data = new int[] { 0, 1, 2 };
var options = maxDop.HasValue ? new ParallelOptions {
MaxDegreeOfParallelism = maxDop.Value
} : null;
Func <int, Task <KeyValuePair <int, int> > > func = n =>
{
return(Task.FromResult(new KeyValuePair <int, int>(n, n * 2)));
};
var actual = ParallelAsync.ForEachAsync(data, options, func);
Assert.Collection(actual.Result, n => Assert.Equal(0, n.Value), n => Assert.Equal(2, n.Value), n => Assert.Equal(4, n.Value));
}
public static void ParallelAsync_ForEach_Action(int?maxDop)
{
var data = new int[] { 0, 1, 2 };
var options = maxDop.HasValue ? new ParallelOptions {
MaxDegreeOfParallelism = maxDop.Value
} : null;
Func <int, Task> action = n =>
{
data[n] = n * 2;
return(Task.CompletedTask);
};
ParallelAsync.ForEachAsync(data, options, action).Wait();
Assert.Collection(data, n => Assert.Equal(0, n), n => Assert.Equal(2, n), n => Assert.Equal(4, n));
}
public static void Train(string udSource)
{
var trainFiles = Directory.GetFiles(udSource, "*-train.conllu", SearchOption.AllDirectories);
var testFiles = Directory.GetFiles(udSource, "*-dev.conllu", SearchOption.AllDirectories);
var trainFilesPerLanguage = trainFiles.Select(f => new { lang = Path.GetFileNameWithoutExtension(f).Replace("_", "-").Split(new char[] { '-' }).First(), file = f }).GroupBy(f => f.lang).ToDictionary(g => g.Key, g => g.Select(f => f.file).ToList());
var testFilesPerLanguage = testFiles.Select(f => new { lang = Path.GetFileNameWithoutExtension(f).Replace("_", "-").Split(new char[] { '-' }).First(), file = f }).GroupBy(f => f.lang).ToDictionary(g => g.Key, g => g.Select(f => f.file).ToList());
var languages = trainFilesPerLanguage.Keys.ToList();
Console.WriteLine($"Found these languages for training: {string.Join(", ", languages)}");
foreach (var forceCase in new EnumCase[] { EnumCase.Original, EnumCase.ForceUpper, EnumCase.ForceLower }) //need tom fix the storage model first - maybe join all in one model
{
ParallelAsync.ForEachAsync(languages, new ParallelOptions(), async lang =>
{
Language language;
try
{
language = Languages.CodeToEnum(lang);
}
catch
{
Console.WriteLine($"Unknown language {lang}");
return;
}
var modelTag = (forceCase == EnumCase.ForceUpper ? "upper" : (forceCase == EnumCase.ForceLower ? "lower" : ""));
var sentenceDetector = new SentenceDetector(language, 0, modelTag);
var trainDocuments = ReadCorpus(trainFilesPerLanguage[lang], ConvertCase: forceCase, sentenceDetector: sentenceDetector);
//TODO: Implement test
//if(testFilesPerLanguage.TryGetValue(lang, out var testFile))
//{
// var testDocuments = ReadUniversalDependencyCorpus(testFile, ConvertCase: forceCase, sentenceDetector: sentenceDetector);
//}
Console.WriteLine($"Now training {lang} in mode {forceCase} using files {string.Join(", ", trainFilesPerLanguage[lang])}");
sentenceDetector.Train(trainDocuments);
await sentenceDetector.StoreAsync();
});
}
}
public virtual async Task WriteObjectBatchAsync(IEnumerable <KeyValuePair <Sha1, ArraySegment <byte> > > items, bool forceOverwrite, CancellationToken cancellationToken)
{
if (items == null || !items.Any())
{
return;
}
var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = MaxDop,
CancellationToken = cancellationToken
};
// Enumerate batches
await ParallelAsync.ForEachAsync(items, parallelOptions, async item =>
{
// Execute batch
await WriteObjectAsync(item.Key, item.Value, forceOverwrite, cancellationToken).ConfigureAwait(false);
}).ConfigureAwait(false);
}
public override ValueTask <IReadOnlyDictionary <Sha1, ReadOnlyMemory <byte> > > ReadObjectBatchAsync(IEnumerable <Sha1> objectIds, ParallelOptions parallelOptions)
{
if (objectIds == null)
{
return(new ValueTask <IReadOnlyDictionary <Sha1, ReadOnlyMemory <byte> > >(ReadOnlyDictionary.Empty <Sha1, ReadOnlyMemory <byte> >()));
}
// Execute batches
var task = ParallelAsync.ForEachAsync(objectIds, parallelOptions, async n =>
{
// Execute batch
var buffer = await ReadObjectAsync(n, parallelOptions.CancellationToken).ConfigureAwait(false);
// Transform batch result
var kvp = new KeyValuePair <Sha1, ReadOnlyMemory <byte> >(n, buffer);
return(kvp);
});
return(task);
}
public async Task Stop_Stops()
{
bool eval1 = false;
bool eval2 = false;
var result = await ParallelAsync.ForEachAsync <Action, int>(new Action[]
{
() => eval1 = true,
() => eval2 = true
}, async (item, controller) =>
{
item();
controller.ProvideResult(0);
controller.Stop();
}, 1).ConfigureAwait(false);
result.Should().Be(0);
eval1.Should().BeTrue();
eval2.Should().BeFalse();
}
