/// <summary> /// Saves the files. /// </summary> /// <param name="newFileList">The new file list.</param> private static void SaveFiles(List <DocumentKeys> newFileList, ProviderComponent storageProvider) { var rockContext = new RockContext(); rockContext.WrapTransaction(() => { rockContext.BinaryFiles.AddRange(newFileList.Select(f => f.File)); rockContext.SaveChanges(); foreach (var entry in newFileList) { if (entry.File != null) { if (storageProvider != null) { storageProvider.SaveContent(entry.File); entry.File.Path = storageProvider.GetPath(entry.File); } else { LogException("Binary File Import", string.Format("Could not load provider {0}.", storageProvider.ToString())); } } // set person attribute value to this binary file guid var attributeValue = rockContext.AttributeValues.FirstOrDefault(p => p.AttributeId == entry.AttributeId && p.EntityId == entry.PersonId); if (attributeValue == null || attributeValue.CreatedDateTime < entry.File.CreatedDateTime) { bool addToContext = attributeValue == null; attributeValue = new AttributeValue(); attributeValue.EntityId = entry.PersonId; attributeValue.AttributeId = entry.AttributeId; attributeValue.Value = entry.File.Guid.ToString(); attributeValue.IsSystem = false; if (addToContext) { rockContext.AttributeValues.Add(attributeValue); } } } rockContext.SaveChanges(DisableAuditing); }); }
/// <summary> /// Maps the specified folder. /// </summary> /// <param name="folder">The folder.</param> /// <param name="ministryFileType">Type of the ministry file.</param> /// <param name="storageProvider">The storage provider.</param> public void Map(ZipArchive folder, BinaryFileType ministryFileType, ProviderComponent storageProvider) { var lookupContext = new RockContext(); var personEntityTypeId = EntityTypeCache.GetId <Person>(); var fileFieldTypeId = FieldTypeCache.Read(Rock.SystemGuid.FieldType.FILE.AsGuid(), lookupContext).Id; var existingAttributes = new AttributeService(lookupContext).GetByFieldTypeId(fileFieldTypeId) .Where(a => a.EntityTypeId == personEntityTypeId) .ToDictionary(a => a.Key, a => a.Id); var emptyJsonObject = "{}"; var newFileList = new List <DocumentKeys>(); int completed = 0; int totalRows = folder.Entries.Count; int percentage = (totalRows - 1) / 100 + 1; ReportProgress(0, string.Format("Verifying files import ({0:N0} found.", totalRows)); foreach (var file in folder.Entries) { var fileExtension = Path.GetExtension(file.Name); var fileMimeType = Extensions.GetMIMEType(file.Name); if (BinaryFileComponent.FileTypeBlackList.Contains(fileExtension)) { LogException("Binary File Import", string.Format("{0} filetype not allowed ({1})", fileExtension, file.Name)); continue; } else if (fileMimeType == null) { LogException("Binary File Import", string.Format("{0} filetype not recognized ({1})", fileExtension, file.Name)); continue; } string[] parsedFileName = file.Name.Split('_'); // Ministry docs should follow this pattern: // 0. Firstname // 1. Lastname // 2. ForeignId // 3. Filename var personForeignId = parsedFileName[2].AsType <int?>(); var personKeys = BinaryFileComponent.ImportedPeople.FirstOrDefault(p => p.IndividualId == personForeignId); if (personKeys != null) { var rockFile = new Rock.Model.BinaryFile(); rockFile.IsSystem = false; rockFile.IsTemporary = false; rockFile.FileName = file.Name; rockFile.MimeType = fileMimeType; rockFile.BinaryFileTypeId = ministryFileType.Id; rockFile.CreatedDateTime = file.LastWriteTime.DateTime; rockFile.ModifiedDateTime = ImportDateTime; rockFile.Description = string.Format("Imported as {0}", file.Name); rockFile.SetStorageEntityTypeId(ministryFileType.StorageEntityTypeId); rockFile.StorageEntitySettings = emptyJsonObject; if (ministryFileType.AttributeValues.Any()) { rockFile.StorageEntitySettings = ministryFileType.AttributeValues .ToDictionary(a => a.Key, v => v.Value.Value).ToJson(); } // use base stream instead of file stream to keep the byte[] // NOTE: if byte[] converts to a string it will corrupt the stream using (var fileContent = new StreamReader(file.Open())) { rockFile.ContentStream = new MemoryStream(fileContent.BaseStream.ReadBytesToEnd()); } var attributePattern = "[A-Za-z0-9-]+"; var attributeName = Regex.Match(parsedFileName[3].RemoveWhitespace(), attributePattern); var attributeKey = attributeName.Value.RemoveWhitespace(); // change key to default key for Background Check Documents if (attributeKey == "BackgroundCheck") { attributeKey = "BackgroundCheckDocument"; } if (!existingAttributes.ContainsKey(attributeKey)) { var newAttribute = new Attribute(); newAttribute.FieldTypeId = fileFieldTypeId; newAttribute.EntityTypeId = personEntityTypeId; newAttribute.EntityTypeQualifierColumn = string.Empty; newAttribute.EntityTypeQualifierValue = string.Empty; newAttribute.Key = attributeKey; newAttribute.Name = attributeName.Value; newAttribute.Description = attributeName.Value + " created by binary file import"; newAttribute.CreatedDateTime = ImportDateTime; newAttribute.ModifiedDateTime = ImportDateTime; newAttribute.IsGridColumn = false; newAttribute.IsMultiValue = false; newAttribute.IsRequired = false; newAttribute.AllowSearch = false; newAttribute.IsSystem = false; newAttribute.Order = 0; newAttribute.AttributeQualifiers.Add(new AttributeQualifier() { Key = "binaryFileType", Value = ministryFileType.Guid.ToString() }); lookupContext.Attributes.Add(newAttribute); lookupContext.SaveChanges(); existingAttributes.Add(newAttribute.Key, newAttribute.Id); } newFileList.Add(new DocumentKeys() { PersonId = personKeys.PersonId, AttributeId = existingAttributes[attributeKey], File = rockFile }); completed++; if (completed % percentage < 1) { int percentComplete = completed / percentage; ReportProgress(percentComplete, string.Format("{0:N0} files imported ({1}% complete).", completed, percentComplete)); } else if (completed % ReportingNumber < 1) { SaveFiles(newFileList, storageProvider); // Reset list newFileList.Clear(); ReportPartialProgress(); } } } if (newFileList.Any()) { SaveFiles(newFileList, storageProvider); } ReportProgress(100, string.Format("Finished files import: {0:N0} addresses imported.", completed)); }
/// <summary> /// Saves the files. /// </summary> /// <param name="newFileList">The new file list.</param> private static void SaveFiles(List <DocumentKeys> newFileList, ProviderComponent storageProvider) { if (storageProvider == null) { LogException("Binary File Import", string.Format("Could not load provider {0}.", storageProvider.ToString())); return; } if (newFileList.Any(f => f.File == null)) { LogException("Binary File Import", string.Format("Could not load {0} files because they were null.", newFileList.Count(f => f.File == null))); } var rockContext = new RockContext(); rockContext.WrapTransaction(() => { foreach (var entry in newFileList) { storageProvider.SaveContent(entry.File); entry.File.Path = storageProvider.GetPath(entry.File); } var list = newFileList.Select(f => f.File).ToList(); rockContext.BinaryFiles.AddRange(newFileList.Select(f => f.File)); rockContext.SaveChanges(); var currentPersonAttributes = new Dictionary <int, List <int> >(); foreach (var entry in newFileList.OrderByDescending(f => f.File.CreatedDateTime)) { List <int> attributeList = null; if (currentPersonAttributes.ContainsKey(entry.PersonId) && currentPersonAttributes[entry.PersonId] != null) { attributeList = currentPersonAttributes[entry.PersonId]; } else { // first document for this person in the current zip file, start a list attributeList = new List <int>(); currentPersonAttributes.Add(entry.PersonId, attributeList); } if (!attributeList.Contains(entry.AttributeId)) { var attributeValue = rockContext.AttributeValues.FirstOrDefault(p => p.AttributeId == entry.AttributeId && p.EntityId == entry.PersonId); // set person attribute value to this binary file guid if (attributeValue == null) { attributeValue = new AttributeValue(); attributeValue.IsSystem = false; attributeValue.EntityId = entry.PersonId; attributeValue.AttributeId = entry.AttributeId; attributeValue.Value = entry.File.Guid.ToString(); rockContext.AttributeValues.Add(attributeValue); } else if (attributeValue.CreatedDateTime < entry.File.CreatedDateTime) { attributeValue.Value = entry.File.Guid.ToString(); rockContext.Entry(attributeValue).State = EntityState.Modified; } attributeList.Add(entry.AttributeId); } } rockContext.SaveChanges(DisableAuditing); }); }
/// <summary> /// Saves the files. /// </summary> /// <param name="newFileList">The new file list.</param> private static void SaveFiles(Dictionary <int, Rock.Model.BinaryFile> newFileList, ProviderComponent storageProvider) { var rockContext = new RockContext(); rockContext.WrapTransaction(() => { rockContext.BinaryFiles.AddRange(newFileList.Values); rockContext.SaveChanges(); foreach (var entry in newFileList) { if (entry.Value != null) { if (storageProvider != null) { storageProvider.SaveContent(entry.Value); entry.Value.Path = storageProvider.GetPath(entry.Value); } else { LogException("Binary File Import", string.Format("Could not load provider {0}.", storageProvider.ToString())); } } // associate the image with the right transaction var transactionImage = new FinancialTransactionImage(); transactionImage.TransactionId = entry.Key; transactionImage.BinaryFileId = entry.Value.Id; transactionImage.Order = 0; rockContext.FinancialTransactions.FirstOrDefault(t => t.Id == entry.Key) .Images.Add(transactionImage); } rockContext.SaveChanges(DisableAuditing); }); }
/// <summary> /// Saves the files. /// </summary> /// <param name="newFileList">The new file list.</param> /// <param name="storageProvider">The storage provider.</param> private static void SaveFiles(Dictionary <int, Rock.Model.BinaryFile> newFileList, ProviderComponent storageProvider) { var rockContext = new RockContext(); rockContext.WrapTransaction(() => { rockContext.BinaryFiles.AddRange(newFileList.Values); rockContext.SaveChanges(); foreach (var file in newFileList) { if (storageProvider != null) { storageProvider.SaveContent(file.Value); file.Value.Path = storageProvider.GetPath(file.Value); } else { LogException("Binary File Import", string.Format("Could not load provider {0}.", storageProvider.ToString())); } // associate the person with this photo rockContext.People.FirstOrDefault(p => p.Id == file.Key).PhotoId = file.Value.Id; } rockContext.SaveChanges(DisableAuditing); }); }
/// <summary> /// Maps the specified folder. /// </summary> /// <param name="folder">The folder.</param> /// <param name="transactionImageType">Type of the transaction image file.</param> /// <param name="storageProvider">The storage provider.</param> public void Map(ZipArchive folder, BinaryFileType transactionImageType, ProviderComponent storageProvider) { var lookupContext = new RockContext(); var emptyJsonObject = "{}"; var newFileList = new Dictionary <int, Rock.Model.BinaryFile>(); var transactionIdList = new FinancialTransactionService(lookupContext) .Queryable().AsNoTracking().Where(t => t.ForeignId != null) .ToDictionary(t => (int)t.ForeignId, t => t.Id); int completed = 0; int totalRows = folder.Entries.Count; int percentage = (totalRows - 1) / 100 + 1; ReportProgress(0, string.Format("Verifying files import ({0:N0} found.", totalRows)); foreach (var file in folder.Entries) { var fileExtension = Path.GetExtension(file.Name); var fileMimeType = Extensions.GetMIMEType(file.Name); if (BinaryFileComponent.FileTypeBlackList.Contains(fileExtension)) { LogException("Binary File Import", string.Format("{0} filetype not allowed ({1})", fileExtension, file.Name)); continue; } else if (fileMimeType == null) { LogException("Binary File Import", string.Format("{0} filetype not recognized ({1})", fileExtension, file.Name)); continue; } int?transactionId = Path.GetFileNameWithoutExtension(file.Name).AsType <int?>(); if (transactionId != null && transactionIdList.ContainsKey((int)transactionId)) { var rockFile = new Rock.Model.BinaryFile(); rockFile.IsSystem = false; rockFile.IsTemporary = false; rockFile.FileName = file.Name; rockFile.MimeType = fileMimeType; rockFile.BinaryFileTypeId = transactionImageType.Id; rockFile.CreatedDateTime = file.LastWriteTime.DateTime; rockFile.ModifiedDateTime = ImportDateTime; rockFile.Description = string.Format("Imported as {0}", file.Name); rockFile.SetStorageEntityTypeId(transactionImageType.StorageEntityTypeId); rockFile.StorageEntitySettings = emptyJsonObject; if (transactionImageType.AttributeValues.Any()) { rockFile.StorageEntitySettings = transactionImageType.AttributeValues .ToDictionary(a => a.Key, v => v.Value.Value).ToJson(); } // use base stream instead of file stream to keep the byte[] // NOTE: if byte[] converts to a string it will corrupt the stream using (var fileContent = new StreamReader(file.Open())) { rockFile.ContentStream = new MemoryStream(fileContent.BaseStream.ReadBytesToEnd()); } newFileList.Add(transactionIdList[(int)transactionId], rockFile); completed++; if (completed % percentage < 1) { int percentComplete = completed / percentage; ReportProgress(percentComplete, string.Format("{0:N0} files imported ({1}% complete).", completed, percentComplete)); } else if (completed % ReportingNumber < 1) { SaveFiles(newFileList, storageProvider); // Reset list newFileList.Clear(); ReportPartialProgress(); } } } if (newFileList.Any()) { SaveFiles(newFileList, storageProvider); } ReportProgress(100, string.Format("Finished files import: {0:N0} addresses imported.", completed)); }
/// <summary> /// Saves the files. /// </summary> /// <param name="newFileList">The new file list.</param> /// <param name="storageProvider">The storage provider.</param> private static void SaveFiles(Dictionary <KeyValuePair <int, int>, Rock.Model.BinaryFile> newFileList, ProviderComponent storageProvider) { var rockContext = new RockContext(); rockContext.WrapTransaction(() => { rockContext.BinaryFiles.AddRange(newFileList.Values); rockContext.SaveChanges(DisableAuditing); foreach (var entry in newFileList) { if (entry.Value != null) { if (storageProvider != null) { storageProvider.SaveContent(entry.Value); entry.Value.Path = storageProvider.GetPath(entry.Value); } else { LogException("Binary File Import", string.Format("Could not load provider {0}.", storageProvider.ToString())); } } // associate the document with the correct BenevolenceRequest var benevolenceDocument = new BenevolenceRequestDocument { BenevolenceRequestId = entry.Key.Key, BinaryFileId = entry.Value.Id, Order = 0 }; if (entry.Key.Value > 0) { benevolenceDocument.ForeignKey = entry.Key.Value.ToString(); benevolenceDocument.ForeignId = entry.Key.Value; } rockContext.BenevolenceRequests.FirstOrDefault(r => r.Id == entry.Key.Key) .Documents.Add(benevolenceDocument); } rockContext.SaveChanges(DisableAuditing); }); }
/// <summary> /// Maps the specified folder. /// </summary> /// <param name="folder">The folder.</param> /// <param name="personImageType">Type of the person image file.</param> /// <param name="storageProvider">The storage provider.</param> public void Map(ZipArchive folder, BinaryFileType personImageType, ProviderComponent storageProvider) { // check for existing images var lookupContext = new RockContext(); var existingImageList = new PersonService(lookupContext).Queryable().AsNoTracking() .Where(p => p.Photo != null) .ToDictionary(p => p.Id, p => p.Photo.CreatedDateTime); var emptyJsonObject = "{}"; var newFileList = new Dictionary <int, Rock.Model.BinaryFile>(); int completed = 0; int totalRows = folder.Entries.Count; int percentage = (totalRows - 1) / 100 + 1; ReportProgress(0, string.Format("Verifying files import ({0:N0} found.", totalRows)); foreach (var file in folder.Entries) { var fileExtension = Path.GetExtension(file.Name); if (BinaryFileComponent.FileTypeBlackList.Contains(fileExtension)) { LogException("Binary File Import", string.Format("{0} filetype not allowed ({1})", fileExtension, file.Name)); continue; } var personForeignId = Path.GetFileNameWithoutExtension(file.Name).AsType <int?>(); var personKeys = BinaryFileComponent.ImportedPeople.FirstOrDefault(p => p.IndividualId == personForeignId); if (personKeys != null) { // only import the most recent profile photo if (!existingImageList.ContainsKey(personKeys.PersonId) || existingImageList[personKeys.PersonId].Value < file.LastWriteTime.DateTime) { var rockFile = new Rock.Model.BinaryFile(); rockFile.IsSystem = false; rockFile.IsTemporary = false; rockFile.FileName = file.Name; rockFile.BinaryFileTypeId = personImageType.Id; rockFile.MimeType = Extensions.GetMIMEType(file.Name); rockFile.CreatedDateTime = file.LastWriteTime.DateTime; rockFile.ModifiedDateTime = ImportDateTime; rockFile.Description = string.Format("Imported as {0}", file.Name); rockFile.SetStorageEntityTypeId(personImageType.StorageEntityTypeId); rockFile.StorageEntitySettings = emptyJsonObject; if (personImageType.AttributeValues.Any()) { rockFile.StorageEntitySettings = personImageType.AttributeValues .ToDictionary(a => a.Key, v => v.Value.Value).ToJson(); } // use base stream instead of file stream to keep the byte[] // NOTE: if byte[] converts to a string it will corrupt the stream using (var fileContent = new StreamReader(file.Open())) { rockFile.ContentStream = new MemoryStream(fileContent.BaseStream.ReadBytesToEnd()); } newFileList.Add(personKeys.PersonId, rockFile); } completed++; if (completed % percentage < 1) { int percentComplete = completed / percentage; ReportProgress(percentComplete, string.Format("{0:N0} files imported ({1}% complete).", completed, percentComplete)); } else if (completed % ReportingNumber < 1) { SaveFiles(newFileList, storageProvider); // add image keys to master list foreach (var newFile in newFileList) { existingImageList.AddOrReplace(newFile.Key, newFile.Value.CreatedDateTime); } // Reset batch list newFileList.Clear(); ReportPartialProgress(); } } } if (newFileList.Any()) { SaveFiles(newFileList, storageProvider); } ReportProgress(100, string.Format("Finished files import: {0:N0} addresses imported.", completed)); }