public async Task <IActionResult> Get()
{
BlobStorageConfig config = _storageConfig.Copy();
config.ContainerName = config.FacetsFilteringContainerName;
List <object> result = new List <object>();
foreach (string name in _searchClient.Model.Facets.Select(f => f.Name))
{
var fileName = string.Format("{0}.txt", name.Replace(" ", "").ToLower());
if (await BlobStorageClient.BlobExistsAsync(config, fileName))
{
string text = await BlobStorageClient.ReadBlobAsync(config, fileName);
result.Add(new { name = name, restrictionList = text });
}
else
{
await BlobStorageClient.UploadBlobAsync(config, fileName, "");
result.Add(new { name = name, restrictionList = "" });
}
}
return(await Task.FromResult(new JsonResult(result)));
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
var restApiConfigSection = Configuration.GetSection("RestApiConfig");
RestApiConfig restApiConfig = restApiConfigSection.Get <RestApiConfig>();
services.AddSingleton(restApiConfig);
var blobStorageConfigSection = Configuration.GetSection("BlobStorageConfig");
BlobStorageConfig blobStorageConfig = blobStorageConfigSection.Get <BlobStorageConfig>();
services.AddSingleton(blobStorageConfig);
var jwtConfigSection = Configuration.GetSection("JwtSettings");
JWTConfig jWTConfig = jwtConfigSection.Get <JWTConfig>();
jWTConfig.CreateSecurityKey();
services.AddSingleton(jWTConfig);
// https://code-maze.com/create-pdf-dotnetcore/
services.AddSingleton(typeof(IConverter), new SynchronizedConverter(new PdfTools()));
// https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1#basic-usage
services.AddHttpClient();
services.AddScoped <ServiceRepository>();
services.AddDbContext <FrontEndContext>(options =>
options.UseSqlServer(Configuration.GetConnectionString("DbConnectionString")));
services.AddDefaultIdentity <FrontEndUser>(options => options.SignIn.RequireConfirmedAccount = false)
.AddRoles <IdentityRole>()
.AddEntityFrameworkStores <FrontEndContext>();
services.AddRazorPages();
services.AddControllersWithViews();
}
public async Task <ActionResult> CreatePersonal(PersonalStorageCreateModel model)
{
if (!ModelState.IsValid)
{
return(View(model));
}
var newStorage = Storage.NewStorage(model.Name, User.Identity.Name, StorageType.Personal);
Mapper.Map(model, newStorage);
try
{
BlobStorageConfig.Initialize(newStorage);
}
catch (SqlException exception)
{
ModelState.AddModelError("", exception.Message);
return(View(model));
}
await _storageDbCommand.CreateAsync(newStorage);
return(RedirectToAction("Index"));
}
public async Task <IActionResult> Upload()
{
foreach (var formFile in Request.Form.Files)
{
if (formFile.Length > 0)
{
BlobStorageConfig config = _storageConfig.Copy();
config.ContainerName = config.UploadContainerName;
await BlobStorageClient.UploadBlobAsync(config,
Path.GetFileNameWithoutExtension(formFile.FileName) + "/" + formFile.FileName,
formFile.OpenReadStream());
var telemetryDict = new Dictionary <string, string>
{
{ "FileName", formFile.FileName },
{ "StorageAccountName", _searchConfig.ServiceName },
{ "ContainerName", _storageConfig.ContainerName }
};
_telemetryClient.TrackEvent("FileUpload", telemetryDict);
}
}
//await _searchClient.RunIndexer();
return(new JsonResult("ok"));
}
/// <summary></summary>
/// <param name="connectionString"></param>
/// <param name="migrationEnable"></param>
public static void Start(string connectionString, bool migrationEnable = true)
{
SqlMapper.AddTypeMap(typeof(DateTime), DbType.DateTime2);
SqlMapper.AddTypeMap(typeof(DateTime?), DbType.DateTime2);
if (!migrationEnable)
{
return;
}
DatabaseHelper.CreateIfNotExists(connectionString);
MigrationRunnerHelper.MigrateToLatest(connectionString);
var storages = new StorageDbCommand(connectionString).GetAllAsync().Result;
foreach (var storage in storages)
{
try
{
BlobStorageConfig.Initialize(storage);
}
catch (Exception) {}
}
}
public MediaServiceConfiguration()
{
CoreConfig = BlobStorageConfig.GetConfig("./Resource/blobstorageconfig.json");
WebConfig = WebConfig.GetWebConfig("./webconfig.json");
MongoDbConfig mongoDbConfig = MongoDbConfig.GetMongoDbConfig("./Resource/mongodbconfig.json");
ImageService = ImageServiceFactory.GetImageService(CoreConfig, mongoDbConfig);
}
public async Task <IActionResult> Update(FacetFilterUpdateRequest request)
{
BlobStorageConfig config = _storageConfig.Copy();
config.ContainerName = config.FacetsFilteringContainerName;
await BlobStorageClient.WriteBlobAsync(config, string.Format("{0}.txt", request.FacetName.Replace(" ", "").ToLower()), request.Text);
return(await Task.FromResult(new JsonResult("success")));
}
public BlobStorage(BlobStorageConfig config)
{
this.config = config;
if (!CloudStorageAccount.TryParse(config.StorageConnectionString, out storageAccount))
{
throw new System.FormatException("storage connection string is invalid. ");
}
cloudBlobClient = storageAccount.CreateCloudBlobClient();
}
public BlobStorageService(IOptions <BlobStorageConfig> storageConfig)
{
this._storageConfig = storageConfig.Value;
StorageCredentials storageCredentials = new StorageCredentials(_storageConfig.AccountName, _storageConfig.AccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(storageCredentials, true);
_blobClient = storageAccount.CreateCloudBlobClient();
}
public IndexerController(AppInsightsConfig appInsightsConfig, SearchConfig searchConfig, BlobStorageConfig storageConfig, TelemetryClient telemetryClient)
{
_appInsightsConfig = appInsightsConfig;
_searchConfig = searchConfig;
_storageConfig = storageConfig;
_telemetryClient = telemetryClient;
_telemetryClient.InstrumentationKey = _appInsightsConfig.InstrumentationKey;
_searchClient = new SearchClient(_searchConfig, _telemetryClient);
}
public void InitializeDatabases()
{
var storageDbCommand = new StorageDbCommand(Constants.ConnectionString);
foreach (var storage in this)
{
BlobStorageConfig.Initialize(storage);
TestUtilities.TruncateAllTables(storage.ConnectionProperties.ToConnectionString());
storageDbCommand.CreateAsync(storage).Wait();
}
}
private static string InsertKnowledgeStoreJson(string skillsetJson, BlobStorageConfig storageConfig)
{
var knowledgeStoreJson = GetJsonFromFile("knowledge-store").GetAwaiter().GetResult();
knowledgeStoreJson = knowledgeStoreJson.Replace("[storage-connection-string]", storageConfig.ConnectionString);
var skillset = JObject.Parse(skillsetJson);
var jtoken = JToken.Parse(knowledgeStoreJson);
skillset.Add("knowledgeStore", jtoken);
return(skillset.ToString());
}
public static ImageService GetImageService(BlobStorageConfig blobStorageConfig, MongoDbConfig mongoDbConfig)
{
BlobStorage blobStorage = new BlobStorage(blobStorageConfig);
ImageProcessor imageProcessor = new ImageProcessor();
//ImageDBManager imageDBManager = new ImageDBManager
// (new MediaRecordDatabaseContext());
MediaRecordMongoDatabaseContext mongoDbContext = new MediaRecordMongoDatabaseContext(mongoDbConfig);
ImageMongoDbManager imageDBManager = new ImageMongoDbManager(mongoDbContext);
Core.ImageService imageService = new Core.ImageService(
blobStorageConfig, imageProcessor, blobStorage, imageDBManager);
return(imageService);
}
public void SuccessfullyOpenBlobReader()
{
BlobStorageConfig config = new BlobStorageConfig()
{
BaseDir = _temporaryPath,
UriPattern = string.Empty
};
IBlobStorageReader reader = IndyDotNet.BlobStorage.Factory.OpenReader(BlobStorageTypes.Default, config);
Assert.IsNotNull(reader);
Assert.IsTrue(0 < reader.Handle);
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddCors(options =>
{
options.AddPolicy(AllowCorsPolicy, builder =>
{
builder.AllowAnyOrigin()
.AllowAnyMethod()
.AllowAnyHeader();
});
});
var appInsightsConfig = new AppInsightsConfig
{
InstrumentationKey = Configuration["APPINSIGHTS_INSTRUMENTATIONKEY"]
};
services.AddSingleton(appInsightsConfig);
services.AddApplicationInsightsTelemetry(appInsightsConfig.InstrumentationKey);
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2);
var searchConfig = new SearchConfig
{
ServiceName = Configuration["SearchServiceName"],
Key = Configuration["SearchServiceKey"],
ApiVersion = Configuration["SearchServiceApiVersion"],
IndexName = Configuration["SearchIndexName"]
};
services.AddSingleton(searchConfig);
var storageConfig = new BlobStorageConfig
{
AccountName = Configuration["StorageAccountName"],
Key = Configuration["StorageAccountKey"],
ContainerName = Configuration["StorageAccountContainerName"],
//FacetsFilteringContainerName = Configuration["FacetsFilteringContainerName"]
UploadContainerName = Configuration["UploadStorageContainerName"]
};
services.AddSingleton(storageConfig);
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddOptions();
// services.Configure<MediaServiceConfiguration>(Configuration);
services.AddLogging(loggingBuilder =>
loggingBuilder.AddSerilog(dispose: true));
BlobStorageConfig blobStorageConfig = BlobStorageConfig.GetConfig("./Resource/blobstorageconfig.json");
FileStorageConfig fileStorageConfig = FileStorageConfig.GetConfig("./Resource/filestorageconfig.json");
MongoDbConfig mongoDbConfig = MongoDbConfig.GetMongoDbConfig("./Resource/mongodbconfig.json");
WebConfig webConfig = WebConfig.GetWebConfig("./webconfig.json");
services.AddSingleton <IImageService, ImageService>(
s => ImageServiceFactory.GetImageServiceCached(blobStorageConfig, fileStorageConfig, mongoDbConfig)
);
services.AddSingleton <WebConfig, WebConfig>(c => webConfig);
services.AddMvc();
}
static void Main(string[] args)
{
try
{
var blobConfig = new BlobStorageConfig
{
ConnectionString = ConfigurationManager.AppSettings["BlobStorageConnectionString"],
Container = ConfigurationManager.AppSettings["BlobContainerName"]
};
var blobStorage = new BlobStorage(blobConfig);
var processor = new BlobStorageProcessor(blobStorage);
processor.Process();
}
catch (Exception ex)
{
Console.WriteLine("An error occurred: " + ex.Message);
}
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public void SuccessfullyOpenBlobWriterInThread()
{
System.Threading.Tasks.Parallel.Invoke(() =>
{
try
{
BlobStorageConfig config = new BlobStorageConfig()
{
BaseDir = _temporaryPath,
UriPattern = string.Empty
};
IBlobStorageWriter writer = IndyDotNet.BlobStorage.Factory.OpenWriter(BlobStorageTypes.Default, config);
Assert.IsNotNull(writer);
Assert.IsTrue(0 < writer.Handle);
} catch (Exception ex)
{
Assert.Fail(ex.Message);
}
});
}
public async Task <ActionResult> EditPersonal(string id, PersonalStorageEditModel model)
{
if (!ModelState.IsValid)
{
return(View(model));
}
var storage = await _storageDbCommand.FindAsync(id);
if (storage == null)
{
return(HttpNotFound());
}
if (storage.Owner != User.Identity.Name)
{
return(new HttpUnauthorizedResult());
}
Mapper.Map(model, storage);
storage.LastModifiedUtcDateTime = DateTime.UtcNow;
try
{
BlobStorageConfig.Initialize(storage);
}
catch (SqlException exception)
{
ModelState.AddModelError("", exception.Message);
return(View(model));
}
await _storageDbCommand.UpdateAsync(id, storage);
return(RedirectToAction("Index"));
}
public FacetsFilteringController(BlobStorageConfig storageConfig)
{
this._storageConfig = storageConfig;
}
public async Task <IActionResult> FilterFacets(string facetname, FacetsFilteringRequest searchRequest)
{
// Calculate the response for each value.
var response = new FacetsFilteringResponse();
response.Values = new List <FacetRecord>();
string[] commonList = new string[] { };
string[] list = new string[] { };
try
{
BlobStorageConfig config = _storageConfig.Copy();
config.ContainerName = config.FacetsFilteringContainerName;
string s = await BlobStorageClient.ReadBlobAsync(config, "commonfilters.txt");
commonList = s.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries);
s = await BlobStorageClient.ReadBlobAsync(config, string.Format("{0}.txt", facetname));
list = s.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries);
}
catch (Exception)
{
}
Trace.TraceInformation("facet: " + facetname);
Trace.TraceInformation("number of values:" + searchRequest.Values.Count);
foreach (var record in searchRequest.Values)
{
if (record == null || record.RecordId == null)
{
continue;
}
FacetRecord responseRecord = new FacetRecord();
responseRecord.RecordId = record.RecordId;
try
{
List <string> restrictionList = new List <string>(commonList);
restrictionList.AddRange(list);
var outputRecord = new FacetData();
outputRecord.Facets = new List <string>();
Trace.TraceInformation("number of facets:" + record.Data.Facets.Count);
// replace all non-alphbic characters before comparision
Regex rgx = new Regex("[^a-zA-Z0-9 ]");
for (int i = 0; i < restrictionList.Count; i++)
{
restrictionList[i] = rgx.Replace(restrictionList[i], "").ToLower();
}
foreach (string phrase in record.Data.Facets)
{
var str = rgx.Replace(phrase, "").ToLower();
if (!string.IsNullOrEmpty(str))
{
//lower case the first letter
//str = Char.ToLower(str[0]) + str.Substring(1);
if (!restrictionList.Contains(str))
{
outputRecord.Facets.Add(phrase);
}
}
}
//UpdateGraph(outputRecord.Facets, facetname);
responseRecord.Data = outputRecord;
}
catch (Exception e)
{
}
finally
{
response.Values.Add(responseRecord);
}
}
return(new OkObjectResult(response));
}
public FileStorageService(BlobStorageConfig blobStorageConfig)
{
this.blobStorageConfig = blobStorageConfig;
}
public static async Task <DataSource> GetOrCreateBlobDataSource(ISearchServiceClient serviceClient, string name, DataSourceType dataSourceType, BlobStorageConfig blobStorageConfig, string query = "")
{
if (await serviceClient.DataSources.ExistsAsync(name))
{
return(await serviceClient.DataSources.GetAsync(name));
}
var dataSource = new DataSource
{
Name = name,
Type = dataSourceType,
Credentials = new DataSourceCredentials(blobStorageConfig.ConnectionString),
Container = new DataContainer(blobStorageConfig.ContainerName)
};
return(await serviceClient.DataSources.CreateAsync(dataSource));
}
public BlobStorageService(BlobStorageConfig blobStorageConfig)
{
_blobStorageConfig = blobStorageConfig;
}
public TestService(ITestOutputHelper output)
{
config = BlobStorageConfig.GetConfig(@"C:\Users\t-chwang\source\repos\ImageServingPlatform\Core\Storage\blobstorageconfig.json");
testFolder = System.Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\";
this.output = output;
}
/// <summary>
/// Uses the documents contained in the target Blob Storage account to train the Form Recognizer model.
/// </summary>
private static async Task <string> TrainFormRecognizerModel(FormRecognizerConfig formConfig, BlobStorageConfig storageConfig)
{
var formRecognizerTrainUri = $"{formConfig.Endpoint}formrecognizer/v1.0-preview/custom/train";
var sasUri = $"https://{storageConfig.AccountName}.blob.core.windows.net/{storageConfig.ContainerName}/{storageConfig.SasToken}";
var requestBody = JsonConvert.SerializeObject(new FormRecognizerTrainRequestBody {
Source = sasUri
});
using (var client = new HttpClient())
{
using (var request = new HttpRequestMessage())
{
request.Method = HttpMethod.Post;
request.RequestUri = new Uri(formRecognizerTrainUri);
request.Content = new StringContent(requestBody, Encoding.UTF8, "application/json");
request.Headers.Add("Ocp-Apim-Subscription-Key", formConfig.Key);
var response = await client.SendAsync(request).ConfigureAwait(false);
var responseBody = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
var errorResponse = JsonConvert.DeserializeObject <FormRecognizerErrorResponse>(responseBody);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(errorResponse.Error.Message);
return("");
}
var successResponse = JsonConvert.DeserializeObject <FormRecognizerTrainSuccessResponse>(responseBody);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"Successfully trained the form recognizer model with {successResponse.TrainingDocuments.Count} forms.");
Console.ResetColor();
return(successResponse.ModelId);
}
}
}
