public void PlatformExportBackground(PlatformImportExportRequest exportRequest, PlatformExportPushNotification pushNotification) { Action <ExportImportProgressInfo> progressCallback = x => { pushNotification.InjectFrom(x); pushNotification.Errors = x.Errors; _pushNotifier.Upsert(pushNotification); }; try { var relativeUrl = "tmp/exported_data.zip"; using (var stream = _blobStorageProvider.OpenWrite(relativeUrl)) { var manifest = exportRequest.ToManifest(); _platformExportManager.Export(stream, manifest, progressCallback); //Get a download url pushNotification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); } } catch (Exception ex) { pushNotification.Errors.Add(ex.ExpandExceptionMessage()); } finally { pushNotification.Description = "Export finished"; pushNotification.Finished = DateTime.UtcNow; _pushNotifier.Upsert(pushNotification); } }
private void ImportImages(IHasImages[] haveImagesObjects, ExportImportProgressInfo progressInfo) { var allImages = haveImagesObjects.SelectMany(x => x.GetFlatObjectsListWithInterface <IHasImages>()) .SelectMany(x => x.Images).ToArray(); foreach (var image in allImages.Where(x => x.BinaryData != null)) { try { string url = image.Url != null && !image.Url.IsAbsoluteUrl() ? image.Url : image.RelativeUrl; //do not save images with external url if (!string.IsNullOrEmpty(url)) { using (var sourceStream = new MemoryStream(image.BinaryData)) using (var targetStream = _blobStorageProvider.OpenWrite(image.Url)) { sourceStream.CopyTo(targetStream); } } } catch (Exception ex) { progressInfo.Errors.Add(ex.Message); } } }
/// <summary> /// Save given image to blob storage. /// </summary> /// <param name="imageUrl">Image url.</param> /// <param name="image">Image object.</param> /// <param name="imageFormat">ImageFormat object.</param> /// <param name="quality">JpegQuality object.</param> public virtual async Task SaveImage(string imageUrl, Image image, ImageFormat imageFormat, JpegQuality quality) { using (var blobStream = _storageProvider.OpenWrite(imageUrl)) using (var stream = new MemoryStream()) { if (imageFormat.Guid == ImageFormat.Jpeg.Guid) { var codecInfo = ImageCodecInfo.GetImageEncoders().FirstOrDefault(c => c.FormatID == imageFormat.Guid); var encoderParams = new EncoderParameters { Param = new EncoderParameter[] { new EncoderParameter(Encoder.Quality, (int)quality) } }; image.Save(stream, codecInfo, encoderParams); } else { image.Save(stream, imageFormat); } stream.Position = 0; await stream.CopyToAsync(blobStream); } }
public ExportWriter(string filePath, IBlobStorageProvider blobStorageProvider, Configuration csvConfiguration, DynamicProperty[] dynamicProperties = null) { var stream = blobStorageProvider.OpenWrite(filePath); _streamWriter = new StreamWriter(stream); _csvWriter = new CsvWriter(_streamWriter, csvConfiguration); _csvWriter.Configuration.RegisterClassMap(new GenericClassMap <T>(dynamicProperties)); }
public CsvCustomerImportReporter(string filePath, IBlobStorageProvider blobStorageProvider, string delimiter) { _filePath = filePath; _delimiter = delimiter; _blobStorageProvider = blobStorageProvider; var stream = _blobStorageProvider.OpenWrite(filePath); _streamWriter = new StreamWriter(stream); }
public CsvPriceImportReporter(string reportFilePath, IBlobStorageProvider blobStorageProvider, string delimiter) { _reportFilePath = reportFilePath; _delimiter = delimiter; _blobStorageProvider = blobStorageProvider; var stream = _blobStorageProvider.OpenWrite(reportFilePath); _streamWriter = new StreamWriter(stream); }
// Only public methods can be invoked in the background. (Hangfire) public async Task BackgroundExport(CsvExportInfo exportInfo, ExportNotification notifyEvent) { var currencies = await _currencyService.GetAllCurrenciesAsync(); var defaultCurrency = currencies.First(x => x.IsPrimary); exportInfo.Currency ??= defaultCurrency.Code; var catalog = await _catalogService.GetByIdsAsync(new[] { exportInfo.CatalogId }); if (catalog == null) { throw new InvalidOperationException($"Cannot get catalog with id '{exportInfo.CatalogId}'"); } Action <ExportImportProgressInfo> progressCallback = async x => { notifyEvent.InjectFrom(x); await _notifier.SendAsync(notifyEvent); }; using (var stream = new MemoryStream()) { try { exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration(); await _csvExporter.DoExportAsync(stream, exportInfo, progressCallback); stream.Position = 0; var fileNameTemplate = await _settingsManager.GetValueAsync(CsvModuleConstants.Settings.General.ExportFileNameTemplate.Name, CsvModuleConstants.Settings.General.ExportFileNameTemplate.DefaultValue.ToString()); var fileName = string.Format(fileNameTemplate, DateTime.UtcNow); fileName = Path.ChangeExtension(fileName, ".csv"); var blobRelativeUrl = Path.Combine("temp", fileName); //Upload result csv to blob storage using (var blobStream = _blobStorageProvider.OpenWrite(blobRelativeUrl)) { stream.CopyTo(blobStream); } //Get a download url notifyEvent.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(blobRelativeUrl); notifyEvent.Description = "Export finished"; } catch (Exception ex) { notifyEvent.Description = "Export failed"; notifyEvent.Errors.Add(ex.ExpandExceptionMessage()); } finally { notifyEvent.Finished = DateTime.UtcNow; await _notifier.SendAsync(notifyEvent); } } }
public void SaveLicense(License license) { using (var stream = _blobStorageProvider.OpenWrite(_blobUrlResolver.GetAbsoluteUrl(_platformOptions.LicenseBlobPath))) { var streamWriter = new StreamWriter(stream); streamWriter.Write(license.RawLicense); streamWriter.Flush(); } }
/// <summary> /// Save given image to blob storage. /// </summary> /// <param name="imageUrl">Image url.</param> /// <param name="image">Image object.</param> /// <param name="format">Image object format.</param> public virtual async Task SaveImage(string imageUrl, Image image, ImageFormat format) { using (var blobStream = _storageProvider.OpenWrite(imageUrl)) using (var stream = new MemoryStream()) { image.Save(stream, format); stream.Position = 0; await stream.CopyToAsync(blobStream); } }
/// <summary> /// Add icon to given product /// </summary> private void AddIcon(CatalogProduct product, string extention, byte[] icon) { if (icon != null) { using (MemoryStream ms = new MemoryStream(icon)) { var blobRelativeUrl = Path.Combine("catalog", product.Code, HttpUtility.UrlDecode(Path.ChangeExtension("icon", extention))); using (var targetStream = _blobStorageProvider.OpenWrite(blobRelativeUrl)) { ms.CopyTo(targetStream); } product.Images = new List <Image> { new Image { Url = blobRelativeUrl } }; } } }
// Only public methods can be invoked in the background. (Hangfire) public void BackgroundExport(Data.Model.CsvExportInfo exportInfo, ExportNotification notifyEvent) { var currencies = _commerceService.GetAllCurrencies(); var defaultCurrency = currencies.First(x => x.IsPrimary); exportInfo.Currency = exportInfo.Currency ?? defaultCurrency.Code; var catalog = _catalogService.GetById(exportInfo.CatalogId); if (catalog == null) { throw new NullReferenceException("catalog"); } Action <ExportImportProgressInfo> progressCallback = x => { notifyEvent.InjectFrom(x); _notifier.Upsert(notifyEvent); }; using (var stream = new MemoryStream()) { try { exportInfo.Configuration = Data.Model.CsvProductMappingConfiguration.GetDefaultConfiguration(); _csvExporter.DoExport(stream, exportInfo, progressCallback); stream.Position = 0; var fileNameTemplate = _settingsManager.GetValue("CsvCatalogImport.ExportFileNameTemplate", string.Empty); var fileName = string.Format(fileNameTemplate, DateTime.UtcNow); fileName = Path.ChangeExtension(fileName, ".csv"); var blobRelativeUrl = Path.Combine("temp", fileName); //Upload result csv to blob storage using (var blobStream = _blobStorageProvider.OpenWrite(blobRelativeUrl)) { stream.CopyTo(blobStream); } //Get a download url notifyEvent.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(blobRelativeUrl); notifyEvent.Description = "Export finished"; } catch (Exception ex) { notifyEvent.Description = "Export failed"; notifyEvent.Errors.Add(ex.ExpandExceptionMessage()); } finally { notifyEvent.Finished = DateTime.UtcNow; _notifier.Upsert(notifyEvent); } } }
public async Task ExportBackgroundAsync(ExportDataRequest request, ExportPushNotification notification, IJobCancellationToken cancellationToken, PerformContext context) { void progressCallback(ExportProgressInfo x) { notification.Patch(x); notification.JobId = context.BackgroundJob.Id; _pushNotificationManager.Send(notification); } try { if (string.IsNullOrEmpty(_platformOptions.DefaultExportFolder)) { throw new PlatformException($"{nameof(_platformOptions.DefaultExportFolder)} should be set."); } var fileName = string.Format(FileNameTemplate, DateTime.UtcNow); // Do not like provider creation here to get file extension, maybe need to pass created provider to Exporter. // Create stream inside Exporter is not good as it is not Exporter resposibility to decide where to write. var provider = _exportProviderFactory.CreateProvider(request); if (!string.IsNullOrEmpty(provider.ExportedFileExtension)) { fileName = Path.ChangeExtension(fileName, provider.ExportedFileExtension); } var url = UrlHelperExtensions.Combine(_platformOptions.DefaultExportFolder, fileName); using (var blobStream = _blobStorageProvider.OpenWrite(url)) { _dataExporter.Export(blobStream, request, progressCallback, new JobCancellationTokenWrapper(cancellationToken)); } notification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(url); } catch (JobAbortedException) { //do nothing } catch (Exception ex) { notification.Errors.Add(ex.ExpandExceptionMessage()); } finally { notification.Description = "Export finished"; notification.Finished = DateTime.UtcNow; await _pushNotificationManager.SendAsync(notification); } }
// Only public methods can be invoked in the background. (Hangfire) public void BackgroundExport(CsvExportInfo exportInfo, ExportNotification notifyEvent) { var curencySetting = _settingsManager.GetSettingByName("VirtoCommerce.Core.General.Currencies"); var defaultCurrency = EnumUtility.SafeParse <CurrencyCodes>(curencySetting.DefaultValue, CurrencyCodes.USD); exportInfo.Currency = exportInfo.Currency ?? defaultCurrency; var catalog = _catalogService.GetById(exportInfo.CatalogId); if (catalog == null) { throw new NullReferenceException("catalog"); } Action <ExportImportProgressInfo> progressCallback = (x) => { notifyEvent.InjectFrom(x); _notifier.Upsert(notifyEvent); }; using (var stream = new MemoryStream()) { try { exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration(); _csvExporter.DoExport(stream, exportInfo, progressCallback); stream.Position = 0; var blobRelativeUrl = "temp/Catalog-" + catalog.Name + "-export.csv"; //Upload result csv to blob storage using (var blobStream = _blobStorageProvider.OpenWrite(blobRelativeUrl)) { stream.CopyTo(blobStream); } //Get a download url notifyEvent.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(blobRelativeUrl); notifyEvent.Description = "Export finished"; } catch (Exception ex) { notifyEvent.Description = "Export failed"; notifyEvent.Errors.Add(ex.ExpandExceptionMessage()); } finally { notifyEvent.Finished = DateTime.UtcNow; _notifier.Upsert(notifyEvent); } } }
public async Task <IHttpActionResult> UploadAsset([FromUri] string folderUrl, [FromUri] string url = null) { if (url == null && !Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } if (url != null) { var fileName = HttpUtility.UrlDecode(System.IO.Path.GetFileName(url)); var fileUrl = folderUrl + "/" + fileName; using (var client = new WebClient()) using (var blobStream = _blobProvider.OpenWrite(fileUrl)) using (var remoteStream = client.OpenRead(url)) { remoteStream.CopyTo(blobStream); var retVal = new webModel.BlobInfo { Name = fileName, RelativeUrl = fileUrl, Url = _urlResolver.GetAbsoluteUrl(fileUrl) }; return(Ok(retVal)); } } else { var blobMultipartProvider = new BlobStorageMultipartProvider(_blobProvider, _urlResolver, folderUrl); await Request.Content.ReadAsMultipartAsync(blobMultipartProvider); var retVal = new List <webModel.BlobInfo>(); foreach (var blobInfo in blobMultipartProvider.BlobInfos) { retVal.Add(new webModel.BlobInfo { Name = blobInfo.FileName, Size = blobInfo.Size.ToString(), MimeType = blobInfo.ContentType, RelativeUrl = blobInfo.Key, Url = _urlResolver.GetAbsoluteUrl(blobInfo.Key) }); } return(Ok(retVal.ToArray())); } }
public async Task PlatformExportBackgroundAsync(PlatformImportExportRequest exportRequest, PlatformExportPushNotification pushNotification, IJobCancellationToken cancellationToken, PerformContext context) { Action <ExportImportProgressInfo> progressCallback = x => { pushNotification.Path(x); pushNotification.JobId = context.BackgroundJob.Id; _pushNotifier.SendAsync(pushNotification); }; try { const string relativeUrl = "tmp/exported_data.zip"; var localTmpFolder = _hostEnv.MapPath(Path.Combine(_platformOptions.LocalUploadFolderPath, "tmp")); var localTmpPath = Path.Combine(localTmpFolder, "exported_data.zip"); if (!Directory.Exists(localTmpFolder)) { Directory.CreateDirectory(localTmpFolder); } //Import first to local tmp folder because Azure blob storage doesn't support some special file access mode using (var stream = System.IO.File.Open(localTmpPath, FileMode.OpenOrCreate)) { var manifest = exportRequest.ToManifest(); await _platformExportManager.ExportAsync(stream, manifest, progressCallback, new JobCancellationTokenWrapper(cancellationToken)); } //Copy export data to blob provider for get public download url using (var localStream = System.IO.File.Open(localTmpPath, FileMode.Open)) using (var blobStream = _blobStorageProvider.OpenWrite(relativeUrl)) { localStream.CopyTo(blobStream); //Get a download url pushNotification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); } } catch (JobAbortedException) { //do nothing } catch (Exception ex) { pushNotification.Errors.Add(ex.ExpandExceptionMessage()); } finally { pushNotification.Description = "Export finished"; pushNotification.Finished = DateTime.UtcNow; await _pushNotifier.SendAsync(pushNotification); } }
public override Stream GetStream(HttpContent parent, HttpContentHeaders headers) { var fileName = Path.GetFileName((headers.ContentDisposition.FileName ?? headers.ContentDisposition.Name).Replace("\"", string.Empty)); var relativeUrl = _rootPath + "/" + fileName; var absoluteUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); BlobInfos.Add(new BlobInfo { ContentType = MimeTypeResolver.ResolveContentType(fileName), FileName = fileName, Key = relativeUrl, Url = absoluteUrl }); return(_blobProvider.OpenWrite(_rootPath + "/" + fileName)); }
private void ImportImages(IHasImages[] haveImagesObjects) { var allImages = haveImagesObjects.SelectMany(x => x.GetFlatObjectsListWithInterface <IHasImages>()) .SelectMany(x => x.Images).ToArray(); foreach (var image in allImages.Where(x => x.BinaryData != null)) { //do not save images with external url if (image.Url != null && !image.Url.IsAbsoluteUrl()) { using (var sourceStream = new MemoryStream(image.BinaryData)) using (var targetStream = _blobStorageProvider.OpenWrite(image.Url)) { sourceStream.CopyTo(targetStream); } } } }
public void PlatformExportBackground(PlatformImportExportRequest exportRequest, PlatformExportPushNotification pushNotification) { Action <ExportImportProgressInfo> progressCallback = x => { pushNotification.InjectFrom(x); pushNotification.Errors = x.Errors; _pushNotifier.Upsert(pushNotification); }; try { const string relativeUrl = "tmp/exported_data.zip"; var localTmpFolder = HostingEnvironment.MapPath("~/App_Data/Uploads/tmp"); var localTmpPath = Path.Combine(localTmpFolder, "exported_data.zip"); if (!Directory.Exists(localTmpFolder)) { Directory.CreateDirectory(localTmpFolder); } //Import first to local tmp folder because Azure blob storage doesn't support some special file access mode using (var stream = File.Open(localTmpPath, FileMode.OpenOrCreate)) { var manifest = exportRequest.ToManifest(); _platformExportManager.Export(stream, manifest, progressCallback); } //Copy export data to blob provider for get public download url using (var localStream = File.Open(localTmpPath, FileMode.Open)) using (var blobStream = _blobStorageProvider.OpenWrite(relativeUrl)) { localStream.CopyTo(blobStream); //Get a download url pushNotification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); } } catch (Exception ex) { pushNotification.Errors.Add(ex.ExpandExceptionMessage()); } finally { pushNotification.Description = "Export finished"; pushNotification.Finished = DateTime.UtcNow; _pushNotifier.Upsert(pushNotification); } }
public async Task <IHttpActionResult> UploadAsset([FromUri] string folderUrl, [FromUri] string url = null, [FromUri] string name = null) { if (url == null && !Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } var retVal = new List <BlobInfo>(); if (url != null) { var fileName = name ?? HttpUtility.UrlDecode(Path.GetFileName(url)); var fileUrl = folderUrl + "/" + fileName; using (var client = new WebClient()) using (var blobStream = _blobProvider.OpenWrite(fileUrl)) using (var remoteStream = client.OpenRead(url)) { remoteStream.CopyTo(blobStream); var blobInfo = AbstractTypeFactory <BlobInfo> .TryCreateInstance(); blobInfo.FileName = fileName; blobInfo.RelativeUrl = fileUrl; blobInfo.Url = _urlResolver.GetAbsoluteUrl(fileUrl); retVal.Add(blobInfo); } } else { var blobMultipartProvider = new BlobStorageMultipartProvider(_blobProvider, _urlResolver, folderUrl); await Request.Content.ReadAsMultipartAsync(blobMultipartProvider); foreach (var blobInfo in blobMultipartProvider.BlobInfos) { blobInfo.RelativeUrl = blobInfo.Key; blobInfo.Url = _urlResolver.GetAbsoluteUrl(blobInfo.Key); retVal.Add(blobInfo); } } return(Ok(retVal.ToArray())); }
/// <inheritdoc /> /// <summary> /// Save given image to blob storage. /// </summary> /// <param name="imageUrl">Image url.</param> /// <param name="image">Image object.</param> /// <param name="format">Image object format.</param> /// <param name="jpegQuality">Target image quality.</param> public virtual async Task SaveImageAsync(string imageUrl, Image <Rgba32> image, IImageFormat format, JpegQuality jpegQuality) { using (var blobStream = _storageProvider.OpenWrite(imageUrl)) using (var stream = new MemoryStream()) { if (format.DefaultMimeType == "image/jpeg") { var options = new JpegEncoder { Quality = (int)jpegQuality }; image.Save(stream, options); } else { image.Save(stream, format); } stream.Position = 0; await stream.CopyToAsync(blobStream); } }
/// <summary> /// Save given image to blob storage. /// </summary> /// <param name="imageUrl">Image url.</param> /// <param name="image">Image object.</param> /// <param name="format">Image object format.</param> public virtual async Task SaveImage(string imageUrl, Image image, ImageFormat format) { ImageCodecInfo jpgEncoder = GetEncoder(ImageFormat.Jpeg); // Create an Encoder object based on the GUID // for the Quality parameter category. System.Drawing.Imaging.Encoder myEncoder = System.Drawing.Imaging.Encoder.Quality; // Create an EncoderParameters object. // An EncoderParameters object has an array of EncoderParameter // objects. In this case, there is only one // EncoderParameter object in the array. EncoderParameters myEncoderParameters = new EncoderParameters(1); EncoderParameter myEncoderParameter = new EncoderParameter(myEncoder, 87L); myEncoderParameters.Param[0] = myEncoderParameter; using (var blobStream = _storageProvider.OpenWrite(imageUrl)) using (var stream = new MemoryStream()) { image.Save(stream, jpgEncoder, myEncoderParameters); stream.Position = 0; await stream.CopyToAsync(blobStream); } }
public void DoImport(Stream backupStream, PlatformExportManifest manifest, Action <ExportImportProgressInfo> progressCallback) { var progressInfo = new ExportImportProgressInfo(); var backupObject = backupStream.DeserializeJson <BackupObject>(); foreach (var category in backupObject.Categories) { category.Catalog = backupObject.Catalogs.FirstOrDefault(x => x.Id == category.CatalogId); if (category.Parents != null) { category.Level = category.Parents.Count(); } } var originalObject = GetBackupObject(progressCallback, false); progressInfo.Description = String.Format("{0} catalogs importing...", backupObject.Catalogs.Count()); progressCallback(progressInfo); UpdateCatalogs(originalObject.Catalogs, backupObject.Catalogs); progressInfo.Description = String.Format("{0} categories importing...", backupObject.Categories.Count()); progressCallback(progressInfo); backupObject.Products = backupObject.Products.OrderBy(x => x.MainProductId).ToList(); UpdateCategories(originalObject.Categories, backupObject.Categories); UpdateProperties(originalObject.Properties, backupObject.Properties); //Binary data if (manifest.HandleBinaryData) { var allBackupImages = backupObject.Products.SelectMany(x => x.Images); allBackupImages = allBackupImages.Concat(backupObject.Categories.SelectMany(x => x.Images)); allBackupImages = allBackupImages.Concat(backupObject.Products.SelectMany(x => x.Variations).SelectMany(x => x.Images)); var allOrigImages = originalObject.Products.SelectMany(x => x.Images); allOrigImages = allOrigImages.Concat(originalObject.Categories.SelectMany(x => x.Images)); allOrigImages = allOrigImages.Concat(originalObject.Products.SelectMany(x => x.Variations).SelectMany(x => x.Images)); //Import only new images var allNewImages = allBackupImages.Where(x => !allOrigImages.Contains(x)); var index = 0; var progressTemplate = "{0} of " + allNewImages.Count() + " images uploading"; foreach (var image in allNewImages) { progressInfo.Description = String.Format(progressTemplate, index); progressCallback(progressInfo); try { //do not save images with external url if (image.Url != null && !image.Url.IsAbsoluteUrl()) { using (var sourceStream = new MemoryStream(image.BinaryData)) using (var targetStream = _blobStorageProvider.OpenWrite(image.Url)) { sourceStream.CopyTo(targetStream); } } } catch (Exception ex) { progressInfo.Errors.Add(String.Format("{0}: {1}", "CatalogModule", ex.ExpandExceptionMessage())); progressCallback(progressInfo); } index++; } } progressInfo.Description = String.Format("{0} products importing...", backupObject.Products.Count()); progressCallback(progressInfo); UpdateCatalogProducts(originalObject.Products, backupObject.Products); }
public async Task<IActionResult> UploadAssetAsync([FromQuery] string folderUrl, [FromQuery]string url = null, [FromQuery]string name = null) { if (url == null && !MultipartRequestHelper.IsMultipartContentType(Request.ContentType)) { return BadRequest($"Expected a multipart request, but got {Request.ContentType}"); } var retVal = new List<BlobInfo>(); if (url != null) { var fileName = name ?? HttpUtility.UrlDecode(Path.GetFileName(url)); var fileUrl = folderUrl + "/" + fileName; using (var client = new WebClient()) using (var blobStream = _blobProvider.OpenWrite(fileUrl)) using (var remoteStream = client.OpenRead(url)) { remoteStream.CopyTo(blobStream); var blobInfo = AbstractTypeFactory<BlobInfo>.TryCreateInstance(); blobInfo.Name = fileName; blobInfo.RelativeUrl = fileUrl; blobInfo.Url = _urlResolver.GetAbsoluteUrl(fileUrl); retVal.Add(blobInfo); } } else { string targetFilePath = null; var boundary = MultipartRequestHelper.GetBoundary(MediaTypeHeaderValue.Parse(Request.ContentType), _defaultFormOptions.MultipartBoundaryLengthLimit); var reader = new MultipartReader(boundary, HttpContext.Request.Body); var section = await reader.ReadNextSectionAsync(); if (section != null) { var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out ContentDispositionHeaderValue contentDisposition); if (hasContentDispositionHeader) { if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition)) { var fileName = contentDisposition.FileName.Value; targetFilePath = folderUrl + "/" + fileName; using (var targetStream = _blobProvider.OpenWrite(targetFilePath)) { await section.Body.CopyToAsync(targetStream); } var blobInfo = AbstractTypeFactory<BlobInfo>.TryCreateInstance(); blobInfo.Name = fileName; blobInfo.RelativeUrl = targetFilePath; blobInfo.Url = _urlResolver.GetAbsoluteUrl(targetFilePath); blobInfo.ContentType = MimeTypeResolver.ResolveContentType(fileName); retVal.Add(blobInfo); } } } } return Ok(retVal.ToArray()); }
public async Task BackgroundDownload(string storeId, string baseUrl, SitemapDownloadNotification notification) { void SendNotificationWithProgressInfo(ExportImportProgressInfo c) { // TODO: is there a better way to copy ExportImportProgressInfo properties to SitemapDownloadNotification without using ValueInjecter? notification.Description = c.Description; notification.ProcessedCount = c.ProcessedCount; notification.TotalCount = c.TotalCount; notification.Errors = c.Errors?.ToList() ?? new List <string>(); _notifier.Send(notification); } try { var relativeUrl = $"tmp/sitemap-{storeId}.zip"; var localTmpFolder = MapPath(_hostingEnvironment, "~/App_Data/Uploads/tmp"); var localTmpPath = Path.Combine(localTmpFolder, $"sitemap-{storeId}.zip"); if (!Directory.Exists(localTmpFolder)) { Directory.CreateDirectory(localTmpFolder); } //Import first to local tmp folder because Azure blob storage doesn't support some special file access mode using (var stream = SystemFile.Open(localTmpPath, FileMode.OpenOrCreate)) { using (var zipPackage = ZipPackage.Open(stream, FileMode.Create)) { await CreateSitemapPartAsync(zipPackage, storeId, baseUrl, "sitemap.xml", SendNotificationWithProgressInfo); var sitemapUrls = await _sitemapXmlGenerator.GetSitemapUrlsAsync(storeId); foreach (var sitemapUrl in sitemapUrls) { if (!string.IsNullOrEmpty(sitemapUrl)) { await CreateSitemapPartAsync(zipPackage, storeId, baseUrl, sitemapUrl, SendNotificationWithProgressInfo); } } } } //Copy export data to blob provider for get public download url using (var localStream = SystemFile.Open(localTmpPath, FileMode.Open)) using (var blobStream = _blobStorageProvider.OpenWrite(relativeUrl)) { localStream.CopyTo(blobStream); notification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); notification.Description = "Sitemap download finished"; } } catch (Exception exception) { notification.Description = "Sitemap download failed"; notification.Errors.Add(exception.ExpandExceptionMessage()); } finally { notification.Finished = DateTime.UtcNow; await _notifier.SendAsync(notification); } }
public void DoImport(Stream backupStream, PlatformExportManifest manifest, Action <ExportImportProgressInfo> progressCallback) { var progressInfo = new ExportImportProgressInfo(); var backupObject = backupStream.DeserializeJson <BackupObject>(); foreach (var category in backupObject.Categories) { category.Catalog = backupObject.Catalogs.FirstOrDefault(x => x.Id == category.CatalogId); if (category.Parents != null) { category.Level = category.Parents.Count(); } } var originalObject = GetBackupObject(progressCallback, false); progressInfo.Description = String.Format("{0} catalogs importing...", backupObject.Catalogs.Count()); progressCallback(progressInfo); UpdateCatalogs(originalObject.Catalogs, backupObject.Catalogs); progressInfo.Description = String.Format("{0} categories importing...", backupObject.Categories.Count()); progressCallback(progressInfo); //Categories should be sorted right way //first need to create virtual categories var orderedCategories = backupObject.Categories.Where(x => x.Catalog.Virtual) .OrderBy(x => x.Level) .ToList(); //second need to create physical categories orderedCategories.AddRange(backupObject.Categories.Where(x => !x.Catalog.Virtual) .OrderBy(x => x.Level)); backupObject.Products = backupObject.Products.OrderBy(x => x.MainProductId).ToList(); UpdateCategories(originalObject.Categories, orderedCategories); UpdateProperties(originalObject.Properties, backupObject.Properties); //Binary data if (manifest.HandleBinaryData) { var allBackupImages = backupObject.Products.SelectMany(x => x.Images); allBackupImages = allBackupImages.Concat(backupObject.Categories.SelectMany(x => x.Images)); allBackupImages = allBackupImages.Concat(backupObject.Products.SelectMany(x => x.Variations).SelectMany(x => x.Images)); var allOrigImages = originalObject.Products.SelectMany(x => x.Images); allOrigImages = allOrigImages.Concat(originalObject.Categories.SelectMany(x => x.Images)); allOrigImages = allOrigImages.Concat(originalObject.Products.SelectMany(x => x.Variations).SelectMany(x => x.Images)); var allNewImages = allBackupImages.Where(x => !allOrigImages.Contains(x)).Where(x => x.BinaryData != null); var index = 0; var progressTemplate = "{0} of " + allNewImages.Count() + " images uploading"; foreach (var image in allNewImages) { progressInfo.Description = String.Format(progressTemplate, index); progressCallback(progressInfo); image.Url = "catalog/" + image.Name; using (var sourceStream = new MemoryStream(image.BinaryData)) using (var targetStream = _blobStorageProvider.OpenWrite(image.Url)) { sourceStream.CopyTo(targetStream); } index++; } } progressInfo.Description = String.Format("{0} products importing...", backupObject.Products.Count()); progressCallback(progressInfo); UpdateCatalogProducts(originalObject.Products, backupObject.Products); }
protected void DoExport <TCsvClass, TClass>(string fileName, int chunkSize, string entitiesType, Func <ICollection <TCsvClass> > entityFactory, CsvClassMap <TClass> entityClassMap, ExportPushNotification notification) { Action <ExportImportProgressInfo> progressCallback = x => { notification.Description = x.Description; notification.TotalCount = x.TotalCount; notification.ProcessedCount = x.ProcessedCount; notification.Errors = x.Errors; _pushNotifier.Upsert(notification); }; var progressInfo = new ExportImportProgressInfo { Description = string.Format("Loading {0}...", entitiesType) }; progressCallback(progressInfo); var updateProgress = new Action(() => { progressInfo.Description = string.Format("{0} of {1} {2} processed", progressInfo.ProcessedCount, progressInfo.TotalCount, entitiesType); progressCallback(progressInfo); }); var updateProgressWithThrottling = updateProgress.Throttle(TimeSpan.FromSeconds(1)); var relativeUrl = "temp/" + fileName + ".zip"; using (var blobStream = _blobStorageProvider.OpenWrite(relativeUrl)) { try { // Because of Recommendation API file uploading limit, we need to split our csv file to parts with size no more than this limit using (var archive = new ZipArchive(blobStream, ZipArchiveMode.Create, true, new UTF8Encoding(false))) { var partIndex = 1; using (var stream = new MemoryStream()) { using (var streamWriter = new StreamWriter(stream, new UTF8Encoding(false), 1024, true) { AutoFlush = true }) { using (var csvWriter = new CsvWriter(streamWriter)) { progressCallback(progressInfo); var entities = entityFactory().ToArray(); csvWriter.Configuration.Delimiter = ","; csvWriter.Configuration.RegisterClassMap(entityClassMap); progressInfo.TotalCount = entities.Length; for (var index = 0; index < entities.Length; index++) { try { var previousSize = (int)stream.Length; csvWriter.WriteRecord(entities[index]); if (stream.Length > chunkSize) { WriteEntry(archive, fileName, ref partIndex, x => x.Write(stream.GetBuffer(), 0, previousSize - 2)); stream.SetLength(0); --index; } } catch (Exception ex) { progressInfo.Errors.Add(ex.ToString()); progressCallback(progressInfo); } progressInfo.ProcessedCount = index + 1; updateProgressWithThrottling(); } } } WriteEntry(archive, fileName, ref partIndex, stream.WriteTo, true); } } updateProgress(); notification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); } catch (Exception ex) { notification.Description = "Export failed"; notification.Errors.Add(ex.ExpandExceptionMessage()); } finally { notification.Description = "Export finished"; notification.Finished = DateTime.UtcNow; _pushNotifier.Upsert(notification); } } }