public void MigrateMediaFiles_Old(SmartObjectContext ctx)
{
var query = ctx.Set <MediaFile>();
//.Where(x => x.Version == 0)
//.Include(x => x.MediaStorage);
var pager = new FastPager <MediaFile>(query, 1000);
using (var scope = new DbContextScope(ctx,
hooksEnabled: false,
autoCommit: false,
proxyCreation: false,
validateOnSave: false,
lazyLoading: false))
{
while (pager.ReadNextPage(out var files))
{
foreach (var file in files)
{
if (file.Version > 0)
{
continue;
}
if (file.Extension.IsEmpty())
{
file.Extension = MimeTypes.MapMimeTypeToExtension(file.MimeType);
}
var name = file.Name;
if (name.IsEmpty())
{
name = file.Id.ToString(System.Globalization.CultureInfo.InvariantCulture);
}
else
{
name = MediaHelper.NormalizeFileName(file.Name.Truncate(292));
}
file.Name = name + "." + file.Extension;
file.CreatedOnUtc = file.UpdatedOnUtc;
file.Version = 1;
ProcessMediaFile(file);
}
// Save to DB
int num = scope.Commit();
// Breathe
ctx.DetachEntities <MediaFile>(deep: true);
}
}
}
public void MigrateMediaFiles(SmartObjectContext ctx)
{
var query = ctx.Set <MediaFile>()
//.Where(x => x.Version == 0)
.Include(x => x.MediaStorage);
var pager = new FastPager <MediaFile>(query, 1000);
using (var scope = new DbContextScope(ctx,
hooksEnabled: false,
autoCommit: false,
proxyCreation: false,
validateOnSave: false,
lazyLoading: false))
{
while (pager.ReadNextPage(out var files))
{
foreach (var file in files)
{
if (file.Version > 0)
{
continue;
}
if (file.Extension.IsEmpty())
{
file.Extension = MimeTypes.MapMimeTypeToExtension(file.MimeType);
}
file.Name = file.Name + "." + file.Extension;
file.CreatedOnUtc = file.UpdatedOnUtc;
file.Version = 1;
ProcessMediaFile(file);
}
// Save to DB
int num = scope.Commit();
// Breathe
ctx.DetachEntities <MediaFile>(deep: true);
}
}
}
public void MigrateUploadedFiles(SmartObjectContext ctx)
{
var fileSet = ctx.Set <MediaFile>();
var folderSet = ctx.Set <MediaFolder>();
using (var scope = new DbContextScope(ctx,
hooksEnabled: false,
autoCommit: false,
validateOnSave: false,
lazyLoading: false,
autoDetectChanges: false))
{
var albumId = _albumRegistry.GetAlbumByName(SystemAlbumProvider.Files)?.Id;
var rootFolder = _mediaFileSystem.GetFolder("Uploaded");
if (!rootFolder.Exists)
{
return;
}
ProcessFolder(rootFolder, albumId.Value);
void ProcessFolder(IFolder folder, int mediaFolderId)
{
var newFiles = new List <FilePair>();
foreach (var uploadedFile in _mediaFileSystem.ListFiles(folder.Path))
{
var file = new MediaFile
{
CreatedOnUtc = uploadedFile.LastUpdated,
UpdatedOnUtc = uploadedFile.LastUpdated,
Extension = uploadedFile.Extension.TrimStart('.'),
Name = uploadedFile.Name,
MimeType = MimeTypes.MapNameToMimeType(uploadedFile.Name),
Size = Convert.ToInt32(uploadedFile.Size),
FolderId = mediaFolderId,
Version = 2
};
ProcessMediaFile(file);
newFiles.Add(new FilePair {
MediaFile = file, UploadedFile = uploadedFile
});
fileSet.Add(file);
}
// Process/save files of current folder
try
{
// Save files to DB
int num = scope.Commit();
// Copy/Move files
foreach (var newFile in newFiles)
{
if (_isFsProvider)
{
var newPath = GetStoragePath(newFile.MediaFile);
if (!_mediaFileSystem.FileExists(newPath))
{
// TODO: (mm) (mc) should we actually MOVE the file?
_mediaFileSystem.CopyFile(newFile.UploadedFile.Path, newPath);
}
}
else
{
_mediaStorageProvider.Save(newFile.MediaFile, MediaStorageItem.FromFile(newFile.UploadedFile));
}
}
if (!_isFsProvider)
{
// MediaFile.MediaStorageId has been updated, we need to save again.
num = scope.Commit();
}
}
catch
{
throw;
}
finally
{
newFiles.Clear();
// Breathe
ctx.DetachEntities <MediaFile>(deep: true);
}
foreach (var uploadedFolder in _mediaFileSystem.ListFolders(folder.Path))
{
var mediaFolder = new MediaFolder
{
Name = uploadedFolder.Name,
ParentId = mediaFolderId
};
// Add folder and save ASAP, we need the folder id
folderSet.Add(mediaFolder);
ctx.SaveChanges();
ProcessFolder(uploadedFolder, mediaFolder.Id);
}
}
}
}
public void MigrateMediaFiles(SmartObjectContext ctx)
{
string prevName = null;
int fileIndex = 0;
var query = ctx.Set <MediaFile>().OrderBy(x => x.Name);
var totalCount = query.Count();
var pageSize = Math.Max(1000, Math.Min(5000, totalCount / 200));
var pageIndex = 0;
using (var scope = new DbContextScope(ctx,
hooksEnabled: false,
autoCommit: false,
proxyCreation: false,
validateOnSave: false,
lazyLoading: false))
{
while (true)
{
var files = new PagedList <MediaFile>(query, pageIndex, pageSize);
if (files.Count == 0)
{
break;
}
foreach (var file in files)
{
if (file.Version > 0)
{
continue;
}
if (file.Extension.IsEmpty())
{
file.Extension = MimeTypes.MapMimeTypeToExtension(file.MimeType);
}
var name = file.Name;
var fixedName = name;
if (name.IsEmpty())
{
name = fixedName = file.Id.ToString(System.Globalization.CultureInfo.InvariantCulture);
}
else
{
name = fixedName = MediaHelper.NormalizeFileName(file.Name.Truncate(292));
if (name == prevName)
{
// Make file name unique
fixedName = name + "-" + ++fileIndex;
}
else
{
fileIndex = 0;
}
}
prevName = name;
file.Name = fixedName + "." + file.Extension;
file.CreatedOnUtc = file.UpdatedOnUtc;
file.Version = 1;
ProcessMediaFile(file);
}
// Save to DB
int num = scope.Commit();
// Breathe
ctx.DetachEntities <MediaFile>(deep: true);
if (!files.HasNextPage)
{
break;
}
pageIndex++;
}
}
}
public void MigrateDownloads(SmartObjectContext ctx)
{
var sql = "SELECT * FROM [Download] WHERE [MediaFileId] IS NULL AND [UseDownloadUrl] = 0";
var downloadStubs = ctx.SqlQuery <DownloadStub>(sql).ToDictionary(x => x.Id);
var downloadsFolderId = _albumRegistry.GetAlbumByName(SystemAlbumProvider.Downloads)?.Id;
var messagesFolderId = _albumRegistry.GetAlbumByName(SystemAlbumProvider.Messages)?.Id;
var newFiles = new List <MediaFile>();
using (var scope = new DbContextScope(ctx,
validateOnSave: false,
hooksEnabled: false,
autoCommit: false,
autoDetectChanges: false))
{
var messageTemplates = ctx.Set <MessageTemplate>()
.Where(x => x.Attachment1FileId.HasValue || x.Attachment2FileId.HasValue || x.Attachment3FileId.HasValue)
.ToList();
// Key = Download.Id
var messageTemplatesDict = new Dictionary <int, MessageTemplate>();
foreach (var mt in messageTemplates)
{
if (mt.Attachment1FileId.HasValue)
{
messageTemplatesDict[mt.Attachment1FileId.Value] = mt;
}
if (mt.Attachment2FileId.HasValue)
{
messageTemplatesDict[mt.Attachment2FileId.Value] = mt;
}
if (mt.Attachment3FileId.HasValue)
{
messageTemplatesDict[mt.Attachment3FileId.Value] = mt;
}
}
var hasPostProcessor = _isFsProvider || messageTemplatesDict.Count > 0;
var query = ctx.Set <Download>().Where(x => x.MediaFileId == null);
var pager = new FastPager <Download>(query, 1000);
while (pager.ReadNextPage(out var downloads))
{
foreach (var d in downloads)
{
var stub = downloadStubs.Get(d.Id);
if (stub == null)
{
continue;
}
if (stub.UseDownloadUrl || string.IsNullOrEmpty(stub.Extension))
{
// Something weird has happened in the past
continue;
}
if (stub.Filename == "undefined" || string.IsNullOrEmpty(stub.Filename))
{
stub.Filename = stub.Id.ToString(CultureInfo.InvariantCulture);
}
var isMailAttachment = false;
if (messageTemplatesDict.TryGetValue(stub.Id, out var mt))
{
isMailAttachment = true;
}
// Create and insert new MediaFile entity for the download
var file = new MediaFile
{
CreatedOnUtc = stub.UpdatedOnUtc,
UpdatedOnUtc = stub.UpdatedOnUtc,
Extension = stub.Extension.TrimStart('.'),
Name = stub.Filename.Truncate(292), // Extension appended later in MigrateFiles()
MimeType = stub.ContentType,
MediaType = MediaType.Image, // Resolved later in MigrateFiles()
FolderId = isMailAttachment ? messagesFolderId : downloadsFolderId,
IsTransient = stub.IsTransient,
MediaStorageId = stub.MediaStorageId,
Version = 0 // Ensure that this record gets processed by MigrateFiles()
};
// Assign new file to download
d.MediaFile = file;
// To be able to move files later
if (hasPostProcessor)
{
newFiles.Add(file);
}
}
// Save to DB
int num = scope.Commit();
if (hasPostProcessor)
{
var downloadsDict = downloads.ToDictionary(x => x.Id);
if (_isFsProvider)
{
// Copy files from "Media/Downloads" to "Media/Storage" folder
MoveDownloadFiles(newFiles.ToDictionary(x => x.Id), downloadsDict, downloadStubs);
}
// MessageTemplate attachments (Download > MediaFile)
if (messageTemplatesDict.Count > 0)
{
ReRefMessageTemplateAttachments(ctx, messageTemplatesDict, downloadsDict);
}
newFiles.Clear();
}
// Breathe
ctx.DetachEntities <MessageTemplate>();
ctx.DetachEntities <Download>(deep: true);
}
}
}
public void Migrate(IEnumerable <LocaleResourceEntry> entries, bool updateTouchedResources = false)
{
Guard.NotNull(entries, nameof(entries));
if (!entries.Any() || !_languages.Any())
{
return;
}
using (var scope = new DbContextScope(_ctx, autoDetectChanges: false, hooksEnabled: false))
{
var langMap = _languages.ToDictionarySafe(x => x.LanguageCulture.EmptyNull().ToLower());
var toDelete = new List <LocaleStringResource>();
var toUpdate = new List <LocaleStringResource>();
var toAdd = new List <LocaleStringResource>();
bool IsEntryValid(LocaleResourceEntry entry, Language targetLang)
{
if (entry.Lang == null)
{
return(true);
}
var sourceLangCode = entry.Lang.ToLower();
if (targetLang != null)
{
var culture = targetLang.LanguageCulture;
if (culture == sourceLangCode || culture.StartsWith(sourceLangCode + "-"))
{
return(true);
}
}
else
{
if (langMap.ContainsKey(sourceLangCode))
{
return(true);
}
if (langMap.Keys.Any(k => k.StartsWith(sourceLangCode + "-", StringComparison.OrdinalIgnoreCase)))
{
return(true);
}
}
return(false);
}
// Remove all entries with invalid lang identifier
var invalidEntries = entries.Where(x => !IsEntryValid(x, null));
if (invalidEntries.Any())
{
entries = entries.Except(invalidEntries).ToArray();
}
foreach (var lang in langMap)
{
var validEntries = entries.Where(x => IsEntryValid(x, lang.Value)).ToArray();
foreach (var entry in validEntries)
{
var db = GetResource(entry.Key, lang.Value.Id, toAdd, out bool isLocal);
if (db == null && entry.Value.HasValue() && !entry.UpdateOnly)
{
// ADD action
toAdd.Add(new LocaleStringResource {
LanguageId = lang.Value.Id, ResourceName = entry.Key, ResourceValue = entry.Value
});
}
if (db == null)
{
continue;
}
if (entry.Value == null)
{
// DELETE action
if (isLocal)
{
toAdd.Remove(db);
}
else
{
toDelete.Add(db);
}
}
else
{
if (isLocal)
{
db.ResourceValue = entry.Value;
continue;
}
// UPDATE action
if (updateTouchedResources || !db.IsTouched.GetValueOrDefault())
{
db.ResourceValue = entry.Value;
toUpdate.Add(db);
if (toDelete.Contains(db))
{
toDelete.Remove(db);
}
}
}
}
}
// add new resources to context
_resources.AddRange(toAdd);
// remove deleted resources
_resources.RemoveRange(toDelete);
// save now
int affectedRows = _ctx.SaveChanges();
_ctx.DetachEntities <Language>();
_ctx.DetachEntities <LocaleStringResource>();
}
}
public void Migrate(IEnumerable <LocaleResourceEntry> entries, bool updateTouchedResources = false)
{
Guard.NotNull(entries, nameof(entries));
if (!entries.Any() || !_languages.Any())
{
return;
}
using (var scope = new DbContextScope(_ctx, autoDetectChanges: false, hooksEnabled: false))
{
var langMap = _languages.ToDictionarySafe(x => x.UniqueSeoCode.EmptyNull().ToLower());
var toDelete = new List <LocaleStringResource>();
var toUpdate = new List <LocaleStringResource>();
var toAdd = new List <LocaleStringResource>();
// remove all entries with invalid lang identifier
var invalidEntries = entries.Where(x => x.Lang != null && !langMap.ContainsKey(x.Lang.ToLower()));
if (invalidEntries.Any())
{
entries = entries.Except(invalidEntries);
}
foreach (var lang in langMap)
{
var validEntries = entries.Where(x => x.Lang == null || langMap[x.Lang.ToLower()].Id == lang.Value.Id);
foreach (var entry in validEntries)
{
var db = GetResource(entry.Key, lang.Value.Id, toAdd, out bool isLocal);
if (db == null && entry.Value.HasValue() && !entry.UpdateOnly)
{
// ADD action
toAdd.Add(new LocaleStringResource {
LanguageId = lang.Value.Id, ResourceName = entry.Key, ResourceValue = entry.Value
});
}
if (db == null)
{
continue;
}
if (entry.Value == null)
{
// DELETE action
if (isLocal)
{
toAdd.Remove(db);
}
else
{
toDelete.Add(db);
}
}
else
{
if (isLocal)
{
db.ResourceValue = entry.Value;
continue;
}
// UPDATE action
if (updateTouchedResources || !db.IsTouched.GetValueOrDefault())
{
db.ResourceValue = entry.Value;
toUpdate.Add(db);
if (toDelete.Contains(db))
{
toDelete.Remove(db);
}
}
}
}
}
// add new resources to context
_resources.AddRange(toAdd);
// remove deleted resources
_resources.RemoveRange(toDelete);
// save now
int affectedRows = _ctx.SaveChanges();
_ctx.DetachEntities <LocaleStringResource>();
}
}