/// <summary>
/// <see cref="IObjectService{T, Pk}.Create(IEnumerable{T})"/>
/// <exception cref="NotSupportedException">Underlying repository layer does not support bulk operations.</exception>
/// </summary>
public virtual IEnumerable <TEntity> Create(IEnumerable <TEntity> items)
{
if (BulkRepository == null)
{
throw new NotSupportedException("Underlying repository layer does not support bulk operations.");
}
try
{
return(BulkRepository.CreateBulk(items));
}
catch (RepositoryException e)
{
throw new ServiceException($"Error creating an object of type {typeof(TEntity).Name}.", e);
}
}
/// <summary>
/// <see cref="IObjectService{T, Pk}.CreateAsync(IEnumerable{T}, CancellationToken)"/>
/// <exception cref="NotSupportedException">Underlying repository layer does not support bulk operations.</exception>
/// </summary>
public virtual async Task <IEnumerable <TEntity> > CreateAsync(
IEnumerable <TEntity> items,
CancellationToken cancellationToken = default)
{
if (BulkRepository == null)
{
throw new NotSupportedException("Underlying repository layer does not support bulk operations.");
}
try
{
return(await BulkRepository.CreateBulkAsync(items, cancellationToken));
}
catch (RepositoryException e)
{
throw new ServiceException($"Error creating an object of type {typeof(TEntity).Name}.", e);
}
}
public void IndexBulkRequest()
{
BulkRequest bulkRequest = new BulkRequest(new List<BulkActionBase>()
{
new IndexBulkAction<TestModel>("test", "testtype", new TestModel(){ Content = "first doc" })
});
Uri bulkUri = new Uri(_mockedUriProvider.Object.Uri, "_bulk");
BulkResponse mockedBulkResponse = new BulkResponse()
{
HadErrors = false
};
_mockedHttpLayer.Setup(x => x.Post(It.Is<HttpRequest>(y => y.Uri.ToString() == bulkUri.ToString() && y.Content.ToString() == bulkRequest.ToString())))
.Returns(new HttpResponse(System.Net.HttpStatusCode.OK, JsonConvert.SerializeObject(mockedBulkResponse)));
BulkRepository bulkRepo = new BulkRepository(_mockedUriProvider.Object, _mockedHttpLayer.Object);
BulkResponse bulkResponse = bulkRepo.DoBulkRequest(bulkRequest);
Assert.IsNotNull(bulkResponse);
Assert.IsFalse(bulkResponse.HadErrors);
}
static int MaxFileSize; // don't download files bigger than 10 MB
static async Task Main(string[] _)
{
//string fs1 = @"C:\Ligonier\webcache\state - theology - does - sin - deserve - damnation.html",
// fs2 = @"C:\Ligonier\webcache\assets\bible - plan.pdf";
//var rel = Utils.GetRelativePath(fs1, fs2);
//Console.WriteLine(rel);
dbctx = new WebModel(); // EF context defaults to config: "name=DefaultConnection"
IAsyncPolicy AdoRetryPolicy = // TODO: probably should configure based on App.config
Policy.Handle <Exception>(ex => true) // retry every exception! TODO: improve
.WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) / 4)); // i.e. 0.5, 1, 2, 4, 8 second retries
//IRepository repo = new Repository(dbctx);
IRepository repo = new BulkRepository(dbctx, AdoRetryPolicy);
MimeCollection.Load(await repo.GetContentTypeToExtnsAsync());
//var ct = new CancellationToken();
htmldir = ConfigurationManager.AppSettings["htmldir"] ?? @"C:\Ligonier\webcache";
if (!Directory.Exists(htmldir))
{
Directory.CreateDirectory(htmldir);
}
var otherdir = ConfigurationManager.AppSettings["otherdir"] ?? (htmldir + Path.DirectorySeparatorChar + OTHFOLDER);
if (!Directory.Exists(otherdir))
{
Directory.CreateDirectory(otherdir);
}
backupdir = ConfigurationManager.AppSettings["backupdir"] ?? (htmldir + Path.DirectorySeparatorChar + BACKUPFOLDER);
if (!Directory.Exists(backupdir))
{
Directory.CreateDirectory(backupdir);
}
if (!int.TryParse(ConfigurationManager.AppSettings["batchsize"], out var batchSize))
{
batchSize = 4;
}
if (!int.TryParse(ConfigurationManager.AppSettings["maxlinks"], out MaxLinks))
{
MaxLinks = 1500;
}
if (!int.TryParse(ConfigurationManager.AppSettings["maxfilesize"], out MaxFileSize))
{
MaxFileSize = 10_000_000; // 10 MB
}
var ValidRetry = new HttpStatusCode[] {
HttpStatusCode.Ambiguous, // 300
HttpStatusCode.Conflict, // 409
HttpStatusCode.InternalServerError, // 500
HttpStatusCode.NotImplemented, // 501
HttpStatusCode.BadGateway, // 502
HttpStatusCode.ServiceUnavailable, // 503
HttpStatusCode.GatewayTimeout
}; // 504
IAsyncPolicy <HttpResponseMessage> HttpRetryPolicy = // TODO: probably should configure based on App.config
Policy.HandleResult <HttpResponseMessage>(rsp => ValidRetry.Contains(rsp.StatusCode))
.WaitAndRetryAsync(0, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) / 2)); // i.e. 1, 2, 4 seconds
#pragma warning disable GCop302 // Since '{0}' implements IDisposable, wrap it in a using() statement
//TODO: plug-in Polly as MessageProcessingHandler / whatever !
var Client = new HttpClient(
new HttpClientHandler {
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, AllowAutoRedirect = true
})
{
Timeout = new TimeSpan(0, 0, 20)
};
#pragma warning restore GCop302 // Since '{0}' implements IDisposable, wrap it in a using() statement
var p = new Program();
var retrycount = 2;
Downloader download;
do
{
HParser = new HapParser(MaxLinks);
download = new Downloader(repo, Client, HttpRetryPolicy, HParser, htmldir, otherdir, backupdir, MaxFileSize);
var dlresult = await p.DownloadAndParse(repo, batchSize, download);
if (!dlresult) // failure may be due to tainted EF context so have to reset all these
{
dbctx = new WebModel(); // EF context defaults to config: "name=DefaultConnection"
repo = new BulkRepository(dbctx, AdoRetryPolicy);
retrycount--;
}
else
{
break;
}
} while (retrycount >= 0);
Console.WriteLine("*** DownloadAndParse FINISHED ***");
var localise = new Localiser(HParser, htmldir, backupdir, download);
await p.HtmlLocalise(repo, batchSize, localise, getMissing : true);
Console.WriteLine("*** HtmlLocalise FINISHED ***");
#if DEBUG
foreach (var extn in MimeCollection.MissingExtns.OrderBy(e => e))
{
Console.WriteLine($"missing extn\t{extn}");
}
#endif
Console.ReadLine();
}
