Exemplo n.º 1
0
        /// <summary>
        /// Saves all family changes.
        /// </summary>
        private void SaveFamilies(List <Group> newFamilyList, Dictionary <GroupLocation, string> newGroupLocations)
        {
            var rockContext = new RockContext();

            // First save any unsaved families
            if (newFamilyList.Any())
            {
                rockContext.WrapTransaction(() =>
                {
                    rockContext.Groups.AddRange(newFamilyList);
                    rockContext.SaveChanges(DisableAuditing);
                });

                // Add these new families to the global list
                ImportedPeople.AddRange(newFamilyList);
            }

            // Now save locations
            if (newGroupLocations.Any())
            {
                // Set updated family id on locations
                foreach (var locationPair in newGroupLocations)
                {
                    int?familyGroupId = ImportedPeople.Where(g => g.ForeignKey == locationPair.Value).Select(g => (int?)g.Id).FirstOrDefault();
                    if (familyGroupId != null)
                    {
                        locationPair.Key.GroupId = (int)familyGroupId;
                    }
                }

                // Save locations
                rockContext.WrapTransaction(() =>
                {
                    rockContext.Configuration.AutoDetectChangesEnabled = false;
                    rockContext.GroupLocations.AddRange(newGroupLocations.Keys);
                    rockContext.ChangeTracker.DetectChanges();
                    rockContext.SaveChanges(DisableAuditing);
                });
            }
        }
Exemplo n.º 2
0
        /// <summary>
        /// Saves the people.
        /// </summary>
        /// <param name="familyList">The family list.</param>
        /// <param name="visitorList">The visitor list.</param>
        /// <param name="previousNamesList">The previous names list.</param>
        private static void SavePeople(List <Group> familyList, List <Group> visitorList, Dictionary <Guid, string> previousNamesList)
        {
            var rockContext        = new RockContext();
            var groupMemberService = new GroupMemberService(rockContext);

            rockContext.WrapTransaction(() =>
            {
                rockContext.Configuration.AutoDetectChangesEnabled = false;
                rockContext.Groups.AddRange(familyList);
                rockContext.SaveChanges(DisableAuditing);

                foreach (var familyGroups in familyList.GroupBy(g => g.ForeignId))
                {
                    var visitorsExist = familyGroups.Count() > 1;
                    foreach (var newFamilyGroup in familyGroups)
                    {
                        foreach (var groupMember in newFamilyGroup.Members)
                        {
                            // don't call LoadAttributes, it only rewrites existing cache objects
                            // groupMember.Person.LoadAttributes( rockContext );

                            var memberPersonAttributeValues = groupMember.Person.Attributes.Select(a => a.Value)
                                                              .Select(a => new AttributeValue
                            {
                                AttributeId = a.Id,
                                EntityId    = groupMember.Person.Id,
                                Value       = groupMember.Person.AttributeValues[a.Key].Value
                            }).ToList();

                            rockContext.AttributeValues.AddRange(memberPersonAttributeValues);

                            // add a default person alias
                            if (!groupMember.Person.Aliases.Any(a => a.AliasPersonId == groupMember.Person.Id))
                            {
                                groupMember.Person.Aliases.Add(new PersonAlias
                                {
                                    AliasPersonId   = groupMember.Person.Id,
                                    AliasPersonGuid = groupMember.Person.Guid,
                                    ForeignId       = groupMember.Person.ForeignId,
                                    ForeignKey      = groupMember.Person.ForeignKey
                                });
                            }

                            // assign the previous name
                            if (previousNamesList.Any(l => l.Key.Equals(groupMember.Person.Guid)))
                            {
                                var newPreviousName = new PersonPreviousName
                                {
                                    LastName    = previousNamesList[groupMember.Person.Guid],
                                    PersonAlias = groupMember.Person.Aliases.FirstOrDefault()
                                };

                                rockContext.PersonPreviousNames.Add(newPreviousName);
                            }

                            // assign the giving group
                            if (groupMember.GroupRoleId != FamilyChildRoleId)
                            {
                                groupMember.Person.GivingGroupId = newFamilyGroup.Id;
                            }

                            // Add known relationship group
                            var knownGroupMember = new GroupMember
                            {
                                PersonId    = groupMember.Person.Id,
                                GroupRoleId = KnownRelationshipOwnerRoleId
                            };

                            var knownRelationshipGroup = new Group
                            {
                                Name        = KnownRelationshipGroupType.Name,
                                GroupTypeId = KnownRelationshipGroupType.Id,
                                IsPublic    = true
                            };

                            knownRelationshipGroup.Members.Add(knownGroupMember);
                            rockContext.Groups.Add(knownRelationshipGroup);

                            // Add implied relationship group
                            var impliedGroupMember = new GroupMember
                            {
                                PersonId    = groupMember.Person.Id,
                                GroupRoleId = ImpliedRelationshipOwnerRoleId
                            };

                            var impliedGroup = new Group
                            {
                                Name        = ImpliedRelationshipGroupType.Name,
                                GroupTypeId = ImpliedRelationshipGroupType.Id,
                                IsPublic    = true
                            };

                            impliedGroup.Members.Add(impliedGroupMember);
                            rockContext.Groups.Add(impliedGroup);

                            if (visitorsExist)
                            {
                                // if this is a visitor, then add relationships to the family member(s)
                                if (visitorList.Where(v => v.ForeignId == newFamilyGroup.ForeignId)
                                    .Any(v => v.Members.Any(m => m.Person.ForeignId.Equals(groupMember.Person.ForeignId))))
                                {
                                    var familyMembers = familyGroups.Except(visitorList).SelectMany(g => g.Members);
                                    foreach (var familyMember in familyMembers.Select(m => m.Person))
                                    {
                                        var invitedByMember = new GroupMember
                                        {
                                            PersonId    = familyMember.Id,
                                            GroupRoleId = InvitedByKnownRelationshipId
                                        };

                                        knownRelationshipGroup.Members.Add(invitedByMember);

                                        if (groupMember.Person.Age < 18 && familyMember.Age > 18)
                                        {
                                            var allowCheckinMember = new GroupMember
                                            {
                                                PersonId    = familyMember.Id,
                                                GroupRoleId = AllowCheckInByKnownRelationshipId
                                            };

                                            knownRelationshipGroup.Members.Add(allowCheckinMember);
                                        }
                                    }
                                }
                                else
                                {   // not a visitor, add the visitors to the family member's known relationship
                                    var visitors = visitorList.Where(v => v.ForeignId == newFamilyGroup.ForeignId)
                                                   .SelectMany(g => g.Members).ToList();
                                    foreach (var visitor in visitors.Select(g => g.Person))
                                    {
                                        var inviteeMember = new GroupMember
                                        {
                                            PersonId    = visitor.Id,
                                            GroupRoleId = InviteeKnownRelationshipId
                                        };

                                        knownRelationshipGroup.Members.Add(inviteeMember);

                                        // if visitor can be checked in and this person is considered an adult
                                        if (visitor.Age < 18 && groupMember.Person.Age > 18)
                                        {
                                            var canCheckInMember = new GroupMember
                                            {
                                                PersonId    = visitor.Id,
                                                GroupRoleId = CanCheckInKnownRelationshipId
                                            };

                                            knownRelationshipGroup.Members.Add(canCheckInMember);
                                        }
                                    }
                                }
                            }
                        }
                    }
                }

                rockContext.ChangeTracker.DetectChanges();
                rockContext.SaveChanges(DisableAuditing);
            });  // end wrap transaction

            // add the new people to our tracking list
            if (familyList.Any())
            {
                var familyMembers = familyList.SelectMany(gm => gm.Members);
                ImportedPeople.AddRange(familyMembers.Select(m => new PersonKeys
                {
                    PersonAliasId   = (int)m.Person.PrimaryAliasId,
                    PersonId        = m.Person.Id,
                    PersonForeignId = m.Person.ForeignId,
                    GroupForeignId  = m.Group.ForeignId,
                    FamilyRoleId    = m.Person.ReviewReasonNote.ConvertToEnum <FamilyRole>()
                }).ToList()
                                        );
            }

            if (visitorList.Any())
            {
                var visitors = visitorList.SelectMany(gm => gm.Members);
                ImportedPeople.AddRange(visitors.Select(m => new PersonKeys
                {
                    PersonAliasId   = (int)m.Person.PrimaryAliasId,
                    PersonId        = m.Person.Id,
                    PersonForeignId = m.Person.ForeignId,
                    GroupForeignId  = m.Group.ForeignId,
                    FamilyRoleId    = m.Person.ReviewReasonNote.ConvertToEnum <FamilyRole>()
                }).ToList()
                                        );
            }
        }
Exemplo n.º 3
0
        /// <summary>
        /// Saves the companies.
        /// </summary>
        /// <param name="businessList">The business list.</param>
        private static void SaveCompanies(List <Group> businessList)
        {
            var rockContext = new RockContext();

            rockContext.WrapTransaction(() =>
            {
                rockContext.Configuration.AutoDetectChangesEnabled = false;
                rockContext.Groups.AddRange(businessList);
                rockContext.SaveChanges(DisableAuditing);

                foreach (var newBusiness in businessList)
                {
                    foreach (var groupMember in newBusiness.Members)
                    {
                        // don't call LoadAttributes, it only rewrites existing cache objects
                        // groupMember.Person.LoadAttributes( rockContext );

                        foreach (var attributeCache in groupMember.Person.Attributes.Select(a => a.Value))
                        {
                            var existingValue     = rockContext.AttributeValues.FirstOrDefault(v => v.Attribute.Key == attributeCache.Key && v.EntityId == groupMember.Person.Id);
                            var newAttributeValue = groupMember.Person.AttributeValues[attributeCache.Key];

                            // set the new value and add it to the database
                            if (existingValue == null)
                            {
                                existingValue = new AttributeValue
                                {
                                    AttributeId = newAttributeValue.AttributeId,
                                    EntityId    = groupMember.Person.Id,
                                    Value       = newAttributeValue.Value
                                };

                                rockContext.AttributeValues.Add(existingValue);
                            }
                            else
                            {
                                existingValue.Value = newAttributeValue.Value;
                                rockContext.Entry(existingValue).State = EntityState.Modified;
                            }
                        }

                        if (!groupMember.Person.Aliases.Any(a => a.AliasPersonId == groupMember.Person.Id))
                        {
                            groupMember.Person.Aliases.Add(new PersonAlias {
                                AliasPersonId = groupMember.Person.Id, AliasPersonGuid = groupMember.Person.Guid
                            });
                        }

                        groupMember.Person.GivingGroupId = newBusiness.Id;
                    }
                }

                rockContext.ChangeTracker.DetectChanges();
                rockContext.SaveChanges(DisableAuditing);

                if (businessList.Any())
                {
                    var groupMembers = businessList.SelectMany(gm => gm.Members);
                    ImportedPeople.AddRange(groupMembers.Select(m => new PersonKeys
                    {
                        PersonAliasId   = (int)m.Person.PrimaryAliasId,
                        PersonId        = m.Person.Id,
                        PersonForeignId = null,
                        GroupForeignId  = m.Group.ForeignId,
                        FamilyRoleId    = FamilyRole.Adult
                    }).ToList()
                                            );
                }
            });
        }
Exemplo n.º 4
0
        /// <summary>
        /// Loads the family data.
        /// </summary>
        /// <param name="csvData">The CSV data.</param>
        private int LoadFamily(CSVInstance csvData)
        {
            // Required variables
            var lookupContext     = new RockContext();
            var locationService   = new LocationService(lookupContext);
            int familyGroupTypeId = GroupTypeCache.GetFamilyGroupType().Id;

            int numImportedFamilies = ImportedPeople.Select(p => p.ForeignId).Distinct().Count();

            int homeLocationTypeId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.GROUP_LOCATION_TYPE_HOME)).Id;
            int workLocationTypeId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.GROUP_LOCATION_TYPE_WORK)).Id;

            var newGroupLocations  = new Dictionary <GroupLocation, string>();
            var currentFamilyGroup = new Group();
            var newFamilyList      = new List <Group>();
            var updatedFamilyList  = new List <Group>();

            var dateFormats = new[] { "yyyy-MM-dd", "MM/dd/yyyy", "MM/dd/yy" };

            string currentFamilyKey = string.Empty;
            int    completed        = 0;

            ReportProgress(0, string.Format("Starting family import ({0:N0} already exist).", numImportedFamilies));

            string[] row;
            // Uses a look-ahead enumerator: this call will move to the next record immediately
            while ((row = csvData.Database.FirstOrDefault()) != null)
            {
                string rowFamilyKey  = row[FamilyId];
                int?   rowFamilyId   = rowFamilyKey.AsType <int?>();
                string rowFamilyName = row[FamilyName];

                if (rowFamilyKey != null && rowFamilyKey != currentFamilyGroup.ForeignKey)
                {
                    currentFamilyGroup = ImportedPeople.FirstOrDefault(p => p.ForeignKey == rowFamilyKey);
                    if (currentFamilyGroup == null)
                    {
                        currentFamilyGroup                        = new Group();
                        currentFamilyGroup.ForeignKey             = rowFamilyKey;
                        currentFamilyGroup.ForeignId              = rowFamilyId;
                        currentFamilyGroup.Name                   = row[FamilyName];
                        currentFamilyGroup.CreatedByPersonAliasId = ImportPersonAliasId;
                        currentFamilyGroup.GroupTypeId            = familyGroupTypeId;
                        newFamilyList.Add(currentFamilyGroup);
                    }
                    else
                    {
                        lookupContext.Groups.Attach(currentFamilyGroup);
                    }

                    // Set the family campus
                    string campusName = row[Campus];
                    if (!string.IsNullOrWhiteSpace(campusName))
                    {
                        var familyCampus = CampusList.Where(c => c.Name.Equals(campusName, StringComparison.InvariantCultureIgnoreCase) ||
                                                            c.ShortCode.Equals(campusName, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
                        if (familyCampus == null)
                        {
                            familyCampus          = new Campus();
                            familyCampus.IsSystem = false;
                            familyCampus.Name     = campusName;
                            lookupContext.Campuses.Add(familyCampus);
                            lookupContext.SaveChanges(DisableAuditing);
                            CampusList.Add(familyCampus);
                        }

                        currentFamilyGroup.CampusId = familyCampus.Id;
                    }

                    // Add the family addresses since they exist in this file
                    string famAddress  = row[Address];
                    string famAddress2 = row[Address2];
                    string famCity     = row[City];
                    string famState    = row[State];
                    string famZip      = row[Zip];
                    string famCountry  = row[Country];

                    Location primaryAddress = locationService.Get(famAddress, famAddress2, famCity, famState, famZip, famCountry, verifyLocation: false);

                    if (primaryAddress != null)
                    {
                        var primaryLocation = new GroupLocation();
                        primaryLocation.LocationId               = primaryAddress.Id;
                        primaryLocation.IsMailingLocation        = true;
                        primaryLocation.IsMappedLocation         = true;
                        primaryLocation.GroupLocationTypeValueId = homeLocationTypeId;
                        newGroupLocations.Add(primaryLocation, rowFamilyKey);
                    }

                    string famSecondAddress  = row[SecondaryAddress];
                    string famSecondAddress2 = row[SecondaryAddress2];
                    string famSecondCity     = row[SecondaryCity];
                    string famSecondState    = row[SecondaryState];
                    string famSecondZip      = row[SecondaryZip];
                    string famSecondCountry  = row[SecondaryCountry];

                    Location secondaryAddress = locationService.Get(famSecondAddress, famSecondAddress2, famSecondCity, famSecondState, famSecondZip, famSecondCountry, verifyLocation: false);

                    if (secondaryAddress != null)
                    {
                        var secondaryLocation = new GroupLocation();
                        secondaryLocation.LocationId               = secondaryAddress.Id;
                        secondaryLocation.IsMailingLocation        = true;
                        secondaryLocation.IsMappedLocation         = true;
                        secondaryLocation.GroupLocationTypeValueId = workLocationTypeId;
                        newGroupLocations.Add(secondaryLocation, rowFamilyKey);
                    }

                    DateTime createdDateValue;
                    if (DateTime.TryParseExact(row[CreatedDate], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out createdDateValue))
                    {
                        currentFamilyGroup.CreatedDateTime  = createdDateValue;
                        currentFamilyGroup.ModifiedDateTime = ImportDateTime;
                    }
                    else
                    {
                        currentFamilyGroup.CreatedDateTime  = ImportDateTime;
                        currentFamilyGroup.ModifiedDateTime = ImportDateTime;
                    }

                    completed++;
                    if (completed % (ReportingNumber * 10) < 1)
                    {
                        ReportProgress(0, string.Format("{0:N0} families imported.", completed));
                    }
                    else if (completed % ReportingNumber < 1)
                    {
                        SaveFamilies(newFamilyList, newGroupLocations);
                        ReportPartialProgress();

                        // Reset lookup context
                        lookupContext.SaveChanges();
                        lookupContext   = new RockContext();
                        locationService = new LocationService(lookupContext);
                        newFamilyList.Clear();
                        newGroupLocations.Clear();
                    }
                }
            }

            // Check to see if any rows didn't get saved to the database
            if (newGroupLocations.Any())
            {
                SaveFamilies(newFamilyList, newGroupLocations);
            }

            lookupContext.SaveChanges();
            lookupContext.Dispose();

            ReportProgress(0, string.Format("Finished family import: {0:N0} families added or updated.", completed));
            return(completed);
        }
Exemplo n.º 5
0
        /// <summary>
        /// Saves the individuals.
        /// </summary>
        /// <param name="newFamilyList">The family list.</param>
        /// <param name="visitorList">The optional visitor list.</param>
        private void SaveIndividuals(List <Group> newFamilyList, List <Group> visitorList = null)
        {
            if (newFamilyList.Any())
            {
                var rockContext = new RockContext();
                rockContext.WrapTransaction(() =>
                {
                    rockContext.Groups.AddRange(newFamilyList);
                    rockContext.SaveChanges(true);

                    ImportedPeople.AddRange(newFamilyList);

                    foreach (var familyGroups in newFamilyList.GroupBy <Group, string>(g => g.ForeignId))
                    {
                        bool visitorsExist = visitorList.Any() && familyGroups.Count() > 1;
                        foreach (var newFamilyGroup in familyGroups)
                        {
                            foreach (var person in newFamilyGroup.Members.Select(m => m.Person))
                            {
                                foreach (var attributeCache in person.Attributes.Select(a => a.Value))
                                {
                                    var newAttributeValue = person.AttributeValues[attributeCache.Key];

                                    if (newAttributeValue != null)
                                    {
                                        newAttributeValue.EntityId = person.Id;
                                        rockContext.AttributeValues.Add(newAttributeValue);
                                    }
                                }

                                if (!person.Aliases.Any(a => a.AliasPersonId == person.Id))
                                {
                                    person.Aliases.Add(new PersonAlias
                                    {
                                        AliasPersonId   = person.Id,
                                        AliasPersonGuid = person.Guid
                                    });
                                }

                                person.GivingGroupId = newFamilyGroup.Id;

                                if (visitorsExist)
                                {
                                    var groupTypeRoleService = new GroupTypeRoleService(rockContext);
                                    var ownerRole            = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_OWNER));
                                    int inviteeRoleId        = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_INVITED)).Id;
                                    int invitedByRoleId      = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_INVITED_BY)).Id;
                                    int canCheckInRoleId     = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_CAN_CHECK_IN)).Id;
                                    int allowCheckInByRoleId = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_ALLOW_CHECK_IN_BY)).Id;

                                    // Retrieve or create the group this person is an owner of
                                    var ownerGroup = new GroupMemberService(rockContext).Queryable()
                                                     .Where(m => m.PersonId == person.Id && m.GroupRoleId == ownerRole.Id)
                                                     .Select(m => m.Group).FirstOrDefault();
                                    if (ownerGroup == null)
                                    {
                                        var ownerGroupMember         = new GroupMember();
                                        ownerGroupMember.PersonId    = person.Id;
                                        ownerGroupMember.GroupRoleId = ownerRole.Id;

                                        ownerGroup             = new Group();
                                        ownerGroup.Name        = ownerRole.GroupType.Name;
                                        ownerGroup.GroupTypeId = ownerRole.GroupTypeId.Value;
                                        ownerGroup.Members.Add(ownerGroupMember);
                                        rockContext.Groups.Add(ownerGroup);
                                    }

                                    // Visitor, add relationships to the family members
                                    if (visitorList.Where(v => v.ForeignId == newFamilyGroup.ForeignId)
                                        .Any(v => v.Members.Any(m => m.Person.ForeignId.Equals(person.ForeignId))))
                                    {
                                        var familyMembers = familyGroups.Except(visitorList).SelectMany(g => g.Members);
                                        foreach (var familyMember in familyMembers)
                                        {
                                            // Add visitor invitedBy relationship
                                            var invitedByMember         = new GroupMember();
                                            invitedByMember.PersonId    = familyMember.Person.Id;
                                            invitedByMember.GroupRoleId = invitedByRoleId;
                                            ownerGroup.Members.Add(invitedByMember);

                                            if (person.Age < 18 && familyMember.Person.Age > 15)
                                            {
                                                // Add visitor allowCheckInBy relationship
                                                var allowCheckinMember         = new GroupMember();
                                                allowCheckinMember.PersonId    = familyMember.Person.Id;
                                                allowCheckinMember.GroupRoleId = allowCheckInByRoleId;
                                                ownerGroup.Members.Add(allowCheckinMember);
                                            }
                                        }
                                    }
                                    else
                                    {   // Family member, add relationships to the visitor(s)
                                        var familyVisitors = visitorList.Where(v => v.ForeignId == newFamilyGroup.ForeignId).SelectMany(g => g.Members).ToList();
                                        foreach (var visitor in familyVisitors)
                                        {
                                            // Add invited visitor relationship
                                            var inviteeMember         = new GroupMember();
                                            inviteeMember.PersonId    = visitor.Person.Id;
                                            inviteeMember.GroupRoleId = inviteeRoleId;
                                            ownerGroup.Members.Add(inviteeMember);

                                            if (visitor.Person.Age < 18 && person.Age > 15)
                                            {
                                                // Add canCheckIn visitor relationship
                                                var canCheckInMember         = new GroupMember();
                                                canCheckInMember.PersonId    = visitor.Person.Id;
                                                canCheckInMember.GroupRoleId = canCheckInRoleId;
                                                ownerGroup.Members.Add(canCheckInMember);
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }

                    rockContext.SaveChanges(true);
                });
            }
        }
Exemplo n.º 6
0
        /// <summary>
        /// Maps the specified folder.
        /// </summary>
        /// <param name="folder">The folder.</param>
        /// <param name="personImageType">Type of the person image file.</param>
        public void Map(ZipArchive folder, BinaryFileType personImageType)
        {
            // check for existing images
            var lookupContext     = new RockContext();
            var existingImageList = new PersonService(lookupContext).Queryable().AsNoTracking()
                                    .Where(p => p.Photo != null)
                                    .ToDictionary(p => p.Id, p => p.Photo.CreatedDateTime);

            var emptyJsonObject = "{}";
            var newFileList     = new Dictionary <int, Rock.Model.BinaryFile>();

            var storageProvider = personImageType.StorageEntityTypeId == DatabaseProvider.EntityType.Id
                ? (ProviderComponent)DatabaseProvider
                : (ProviderComponent)FileSystemProvider;

            var completedItems = 0;
            var totalEntries   = folder.Entries.Count;
            var percentage     = (totalEntries - 1) / 100 + 1;

            ReportProgress(0, string.Format("Verifying person images import ({0:N0} found.", totalEntries));

            foreach (var file in folder.Entries)
            {
                var fileExtension = Path.GetExtension(file.Name);
                if (FileTypeBlackList.Contains(fileExtension))
                {
                    LogException("Binary File Import", string.Format("{0} filetype not allowed ({1})", fileExtension, file.Name));
                    continue;
                }

                var personForeignId = Path.GetFileNameWithoutExtension(file.Name).AsType <int?>();
                var personKeys      = ImportedPeople.FirstOrDefault(p => p.PersonForeignId == personForeignId);
                if (personKeys != null)
                {
                    // only import the most recent profile photo
                    if (!existingImageList.ContainsKey(personKeys.PersonId) || existingImageList[personKeys.PersonId].Value < file.LastWriteTime.DateTime)
                    {
                        var rockFile = new Rock.Model.BinaryFile
                        {
                            IsSystem         = false,
                            IsTemporary      = false,
                            FileName         = file.Name,
                            BinaryFileTypeId = personImageType.Id,
                            MimeType         = GetMIMEType(file.Name),
                            CreatedDateTime  = file.LastWriteTime.DateTime,
                            Description      = string.Format("Imported as {0}", file.Name)
                        };

                        rockFile.SetStorageEntityTypeId(personImageType.StorageEntityTypeId);
                        rockFile.StorageEntitySettings = emptyJsonObject;

                        if (personImageType.AttributeValues.Any())
                        {
                            rockFile.StorageEntitySettings = personImageType.AttributeValues
                                                             .ToDictionary(a => a.Key, v => v.Value.Value).ToJson();
                        }

                        // use base stream instead of file stream to keep the byte[]
                        // NOTE: if byte[] converts to a string it will corrupt the stream
                        using (var fileContent = new StreamReader(file.Open()))
                        {
                            rockFile.ContentStream = new MemoryStream(fileContent.BaseStream.ReadBytesToEnd());
                        }

                        newFileList.Add(personKeys.PersonId, rockFile);
                    }

                    completedItems++;
                    if (completedItems % percentage < 1)
                    {
                        var percentComplete = completedItems / percentage;
                        ReportProgress(percentComplete, string.Format("{0:N0} person image files imported ({1}% complete).", completedItems, percentComplete));
                    }
                    else if (completedItems % ReportingNumber < 1)
                    {
                        SaveFiles(newFileList, storageProvider);

                        // add image keys to master list
                        foreach (var newFile in newFileList)
                        {
                            existingImageList.AddOrReplace(newFile.Key, newFile.Value.CreatedDateTime);
                        }

                        // Reset batch list
                        newFileList.Clear();
                        ReportPartialProgress();
                    }
                }
            }

            if (newFileList.Any())
            {
                SaveFiles(newFileList, storageProvider);
            }

            lookupContext.Dispose();
            ReportProgress(100, string.Format("Finished files import: {0:N0} person images imported.", completedItems));
        }
Exemplo n.º 7
0
        /// <summary>
        /// Loads the individual data.
        /// </summary>
        /// <param name="csvData">The CSV data.</param>
        private int LoadIndividuals(CsvDataModel csvData)
        {
            var lookupContext        = new RockContext();
            var groupTypeRoleService = new GroupTypeRoleService(lookupContext);
            var groupMemberService   = new GroupMemberService(lookupContext);

            // Marital statuses: Married, Single, Separated, etc
            var maritalStatusTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_MARITAL_STATUS), lookupContext).DefinedValues;

            // Connection statuses: Member, Visitor, Attendee, etc
            var connectionStatusTypes      = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_CONNECTION_STATUS), lookupContext).DefinedValues;
            int memberConnectionStatusId   = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_MEMBER)).Id;
            int visitorConnectionStatusId  = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_VISITOR)).Id;
            int attendeeConnectionStatusId = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_ATTENDEE)).Id;

            // Suffix type: Dr., Jr., II, etc
            var suffixTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_SUFFIX), lookupContext).DefinedValues;

            // Title type: Mr., Mrs. Dr., etc
            var titleTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_TITLE), lookupContext).DefinedValues;

            // Record statuses: Active, Inactive, Pending
            int?recordStatusActiveId   = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_ACTIVE), lookupContext).Id;
            int?recordStatusInactiveId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_INACTIVE), lookupContext).Id;
            int?recordStatusPendingId  = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_PENDING), lookupContext).Id;

            // Deceased record status reason (others available: No Activity, Moved, etc)
            var recordStatusDeceasedId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_REASON_DECEASED)).Id;

            // Record type: Person
            int?personRecordTypeId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_TYPE_PERSON), lookupContext).Id;

            // Group roles: Owner, Adult, Child, others
            GroupTypeRole ownerRole   = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_OWNER));
            int           adultRoleId = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_FAMILY_MEMBER_ADULT)).Id;
            int           childRoleId = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_FAMILY_MEMBER_CHILD)).Id;

            // Phone types: Home, Work, Mobile
            var numberTypeValues = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_PHONE_TYPE), lookupContext).DefinedValues;

            // Timeline note type id
            var noteTimelineTypeId = new NoteTypeService(lookupContext).Get(new Guid("7E53487C-D650-4D85-97E2-350EB8332763")).Id;

            // School defined type
            var schoolDefinedType = DefinedTypeCache.Read(new Guid("576FF1E2-6225-4565-A16D-230E26167A3D"));

            // Look up additional Person attributes (existing)
            var personAttributes = new AttributeService(lookupContext).GetByEntityTypeId(PersonEntityTypeId).ToList();

            // Core attributes: PreviousChurch, Position, Employer, School, etc
            var previousChurchAttribute = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "PreviousChurch"));
            var employerAttribute       = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Employer"));
            var positionAttribute       = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Position"));
            var firstVisitAttribute     = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "FirstVisit"));
            var schoolAttribute         = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "School"));
            var membershipDateAttribute = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "MembershipDate"));
            var baptismDateAttribute    = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "BaptismDate"));
            var facebookAttribute       = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Facebook"));
            var twitterAttribute        = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Twitter"));
            var instagramAttribute      = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Instagram"));

            // Text field type id
            int textFieldTypeId = FieldTypeCache.Read(new Guid(Rock.SystemGuid.FieldType.TEXT), lookupContext).Id;

            // Attribute entity type id
            int attributeEntityTypeId = EntityTypeCache.Read("Rock.Model.Attribute").Id;

            // Visit info category
            var visitInfoCategory = new CategoryService(lookupContext).GetByEntityTypeId(attributeEntityTypeId)
                                    .Where(c => c.Name == "Visit Information").FirstOrDefault();

            // Add a Secondary Email attribute if it doesn't exist
            var secondaryEmail = personAttributes.FirstOrDefault(a => a.Key == "SecondaryEmail");

            if (secondaryEmail == null)
            {
                secondaryEmail                           = new Rock.Model.Attribute();
                secondaryEmail.Key                       = "SecondaryEmail";
                secondaryEmail.Name                      = "Secondary Email";
                secondaryEmail.FieldTypeId               = textFieldTypeId;
                secondaryEmail.EntityTypeId              = PersonEntityTypeId;
                secondaryEmail.EntityTypeQualifierValue  = string.Empty;
                secondaryEmail.EntityTypeQualifierColumn = string.Empty;
                secondaryEmail.Description               = "The secondary email for this person";
                secondaryEmail.DefaultValue              = string.Empty;
                secondaryEmail.IsMultiValue              = false;
                secondaryEmail.IsRequired                = false;
                secondaryEmail.Order                     = 0;

                lookupContext.Attributes.Add(secondaryEmail);
                secondaryEmail.Categories.Add(visitInfoCategory);
                lookupContext.SaveChanges(true);
            }

            var secondaryEmailAttribute = AttributeCache.Read(secondaryEmail.Id, lookupContext);

            // Add a former name attribute
            var formerName = personAttributes.FirstOrDefault(a => a.Key == "FormerName");

            if (formerName == null)
            {
                formerName                           = new Rock.Model.Attribute();
                formerName.Key                       = "FormerName";
                formerName.Name                      = "Former Name";
                formerName.FieldTypeId               = textFieldTypeId;
                formerName.EntityTypeId              = PersonEntityTypeId;
                formerName.EntityTypeQualifierValue  = string.Empty;
                formerName.EntityTypeQualifierColumn = string.Empty;
                formerName.Description               = "The former name for this person";
                formerName.DefaultValue              = string.Empty;
                formerName.IsMultiValue              = false;
                formerName.IsRequired                = false;
                formerName.Order                     = 0;

                lookupContext.Attributes.Add(formerName);
                secondaryEmail.Categories.Add(visitInfoCategory);
                lookupContext.SaveChanges(true);
            }

            var formerNameAttribute = AttributeCache.Read(formerName.Id, lookupContext);

            // Look for custom attributes in the Individual file
            var allFields = csvData.TableNodes.FirstOrDefault().Columns.Select((node, index) => new { node = node, index = index }).ToList();
            Dictionary <int, string> customAttributes = allFields.Where(f => f.index > Twitter).ToDictionary(f => f.index, f => f.node.Name);

            // Add any if they don't already exist
            if (customAttributes.Any())
            {
                var newAttributes = new List <Rock.Model.Attribute>();
                foreach (var newAttributePair in customAttributes.Where(ca => !personAttributes.Any(a => a.Name == ca.Value)))
                {
                    var newAttribute = new Rock.Model.Attribute();
                    newAttribute.Name        = newAttributePair.Value;
                    newAttribute.Key         = newAttributePair.Value.RemoveWhitespace();
                    newAttribute.Description = newAttributePair.Value + " created by CSV import";
                    newAttribute.EntityTypeQualifierValue  = string.Empty;
                    newAttribute.EntityTypeQualifierColumn = string.Empty;
                    newAttribute.EntityTypeId = PersonEntityTypeId;
                    newAttribute.FieldTypeId  = textFieldTypeId;
                    newAttribute.DefaultValue = string.Empty;
                    newAttribute.IsMultiValue = false;
                    newAttribute.IsGridColumn = false;
                    newAttribute.IsRequired   = false;
                    newAttribute.Order        = 0;
                    newAttributes.Add(newAttribute);
                }

                lookupContext.Attributes.AddRange(newAttributes);
                lookupContext.SaveChanges(true);
                personAttributes.AddRange(newAttributes);
            }

            var dateFormats = new[] { "MM/dd/yyyy", "MM/dd/yy" };

            var currentFamilyGroup = new Group();
            var newFamilyList      = new List <Group>();
            var newVisitorList     = new List <Group>();
            var importDate         = DateTime.Now;

            int completed = 0;

            ReportProgress(0, string.Format("Starting Individual import ({0:N0} already exist).", ImportedPeople.Count(p => p.Members.Any(m => m.Person.ForeignId != null))));

            string[] row;
            // Uses a look-ahead enumerator: this call will move to the next record immediately
            while ((row = csvData.Database.FirstOrDefault()) != null)
            {
                int  groupRoleId          = adultRoleId;
                bool isFamilyRelationship = true;

                string rowFamilyId   = row[FamilyId];
                string rowPersonId   = row[PersonId];
                string rowFamilyName = row[FamilyName];

                if (!string.IsNullOrWhiteSpace(rowFamilyId) && rowFamilyId != currentFamilyGroup.ForeignId)
                {
                    currentFamilyGroup = ImportedPeople.FirstOrDefault(p => p.ForeignId == rowFamilyId);
                    if (currentFamilyGroup == null)
                    {
                        currentFamilyGroup           = new Group();
                        currentFamilyGroup.ForeignId = rowFamilyId;
                        currentFamilyGroup.Name      = row[FamilyName];
                        currentFamilyGroup.CreatedByPersonAliasId = ImportPersonAlias.Id;
                        currentFamilyGroup.GroupTypeId            = FamilyGroupTypeId;
                    }
                }

                // Verify this person isn't already in our data
                var personExists = ImportedPeople.Any(p => p.Members.Any(m => m.Person.ForeignId == rowPersonId));
                if (!personExists)
                {
                    var person = new Person();
                    person.ForeignId              = rowPersonId;
                    person.SystemNote             = string.Format("Imported via Excavator on {0}", importDate.ToString());
                    person.RecordTypeValueId      = personRecordTypeId;
                    person.CreatedByPersonAliasId = ImportPersonAlias.Id;
                    string firstName = row[FirstName];
                    person.FirstName  = firstName;
                    person.NickName   = row[NickName] ?? firstName;
                    person.MiddleName = row[MiddleName];
                    person.LastName   = row[LastName];

                    #region Assign values to the Person record

                    DateTime birthDate;
                    if (DateTime.TryParseExact(row[DateOfBirth], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out birthDate))
                    {
                        person.BirthDate = birthDate;
                    }

                    DateTime anniversary;
                    if (DateTime.TryParseExact(row[Anniversary], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out anniversary))
                    {
                        person.AnniversaryDate = anniversary;
                    }

                    string gender = row[Gender];
                    if (gender != null)
                    {
                        switch (gender.Trim().ToLower())
                        {
                        case "m":
                        case "male":
                            person.Gender = Rock.Model.Gender.Male;
                            break;

                        case "f":
                        case "female":
                            person.Gender = Rock.Model.Gender.Female;
                            break;

                        default:
                            person.Gender = Rock.Model.Gender.Unknown;
                            break;
                        }
                    }

                    string prefix = row[Prefix];
                    if (!string.IsNullOrWhiteSpace(prefix))
                    {
                        prefix = prefix.RemoveSpecialCharacters().Trim();
                        person.TitleValueId = titleTypes.Where(s => prefix == s.Value.RemoveSpecialCharacters())
                                              .Select(s => (int?)s.Id).FirstOrDefault();
                    }

                    string suffix = row[Suffix];
                    if (!string.IsNullOrWhiteSpace(suffix))
                    {
                        suffix = suffix.RemoveSpecialCharacters().Trim();
                        person.SuffixValueId = suffixTypes.Where(s => suffix == s.Value.RemoveSpecialCharacters())
                                               .Select(s => (int?)s.Id).FirstOrDefault();
                    }

                    string maritalStatus = row[MaritalStatus];
                    if (!string.IsNullOrWhiteSpace(maritalStatus))
                    {
                        person.MaritalStatusValueId = maritalStatusTypes.Where(dv => dv.Value == maritalStatus)
                                                      .Select(dv => (int?)dv.Id).FirstOrDefault();
                    }
                    else
                    {
                        person.MaritalStatusValueId = maritalStatusTypes.Where(dv => dv.Value == "Unknown")
                                                      .Select(dv => (int?)dv.Id).FirstOrDefault();
                    }

                    string familyRole = row[FamilyRole];
                    if (!string.IsNullOrWhiteSpace(familyRole))
                    {
                        if (familyRole == "Visitor")
                        {
                            isFamilyRelationship = false;
                        }

                        if (familyRole == "Child" || person.Age < 18)
                        {
                            groupRoleId = childRoleId;
                        }
                    }

                    string connectionStatus = row[ConnectionStatus];
                    if (!string.IsNullOrWhiteSpace(connectionStatus))
                    {
                        if (connectionStatus == "Member")
                        {
                            person.ConnectionStatusValueId = memberConnectionStatusId;
                        }
                        else if (connectionStatus == "Visitor")
                        {
                            person.ConnectionStatusValueId = visitorConnectionStatusId;
                        }
                        else if (connectionStatus == "Deceased")
                        {
                            person.IsDeceased = true;
                            person.RecordStatusReasonValueId = recordStatusDeceasedId;
                        }
                        else
                        {
                            // look for user-defined connection type or default to Attendee
                            var customConnectionType = connectionStatusTypes.Where(dv => dv.Value == connectionStatus)
                                                       .Select(dv => (int?)dv.Id).FirstOrDefault();

                            person.ConnectionStatusValueId = customConnectionType ?? attendeeConnectionStatusId;
                            person.RecordStatusValueId     = recordStatusActiveId;
                        }
                    }

                    string recordStatus = row[RecordStatus];
                    if (!string.IsNullOrWhiteSpace(recordStatus))
                    {
                        switch (recordStatus.Trim())
                        {
                        case "Active":
                            person.RecordStatusValueId = recordStatusActiveId;
                            break;

                        case "Inactive":
                            person.RecordStatusValueId = recordStatusInactiveId;
                            break;

                        default:
                            person.RecordStatusValueId = recordStatusPendingId;
                            break;
                        }
                    }

                    var personNumbers = new Dictionary <string, string>();
                    personNumbers.Add("Home", row[HomePhone]);
                    personNumbers.Add("Mobile", row[MobilePhone]);
                    personNumbers.Add("Work", row[WorkPhone]);
                    string smsAllowed = row[AllowSMS];

                    foreach (var numberPair in personNumbers.Where(n => !string.IsNullOrWhiteSpace(n.Value)))
                    {
                        var extension        = string.Empty;
                        var countryCode      = Rock.Model.PhoneNumber.DefaultCountryCode();
                        var normalizedNumber = string.Empty;
                        var countryIndex     = numberPair.Value.IndexOf('+');
                        int extensionIndex   = numberPair.Value.LastIndexOf('x') > 0 ? numberPair.Value.LastIndexOf('x') : numberPair.Value.Length;
                        if (countryIndex >= 0)
                        {
                            countryCode      = numberPair.Value.Substring(countryIndex, countryIndex + 3).AsNumeric();
                            normalizedNumber = numberPair.Value.Substring(countryIndex + 3, extensionIndex - 3).AsNumeric();
                            extension        = numberPair.Value.Substring(extensionIndex);
                        }
                        else if (extensionIndex > 0)
                        {
                            normalizedNumber = numberPair.Value.Substring(0, extensionIndex).AsNumeric();
                            extension        = numberPair.Value.Substring(extensionIndex).AsNumeric();
                        }
                        else
                        {
                            normalizedNumber = numberPair.Value.AsNumeric();
                        }

                        if (!string.IsNullOrWhiteSpace(normalizedNumber))
                        {
                            var currentNumber = new PhoneNumber();
                            currentNumber.CountryCode            = countryCode;
                            currentNumber.CreatedByPersonAliasId = ImportPersonAlias.Id;
                            currentNumber.Extension         = extension.Left(20);
                            currentNumber.Number            = normalizedNumber.Left(20);
                            currentNumber.NumberTypeValueId = numberTypeValues.Where(v => v.Value.Equals(numberPair.Key))
                                                              .Select(v => (int?)v.Id).FirstOrDefault();
                            if (numberPair.Key == "Mobile")
                            {
                                switch (smsAllowed.Trim().ToLower())
                                {
                                case "y":
                                case "yes":
                                case "active":
                                    currentNumber.IsMessagingEnabled = true;
                                    break;

                                default:
                                    currentNumber.IsMessagingEnabled = false;
                                    break;
                                }
                            }

                            person.PhoneNumbers.Add(currentNumber);
                        }
                    }

                    // Map Person attributes
                    person.Attributes      = new Dictionary <string, AttributeCache>();
                    person.AttributeValues = new Dictionary <string, AttributeValue>();

                    string formerNameValue = row[FormerName];
                    if (!string.IsNullOrWhiteSpace(formerNameValue))
                    {
                        AddPersonAttribute(formerNameAttribute, person, formerNameValue);
                    }

                    bool isEmailActive;
                    switch (row[IsEmailActive].Trim().ToLower())
                    {
                    case "n":
                    case "no":
                    case "inactive":
                        isEmailActive = false;
                        break;

                    default:
                        isEmailActive = true;
                        break;
                    }

                    EmailPreference emailPreference;
                    switch (row[AllowBulkEmail].Trim().ToLower())
                    {
                    case "n":
                    case "no":
                    case "inactive":
                        emailPreference = EmailPreference.NoMassEmails;
                        break;

                    default:
                        emailPreference = EmailPreference.EmailAllowed;
                        break;
                    }

                    string primaryEmail = row[Email];
                    if (!string.IsNullOrWhiteSpace(primaryEmail))
                    {
                        person.Email           = primaryEmail;
                        person.IsEmailActive   = isEmailActive;
                        person.EmailPreference = emailPreference;
                    }

                    string secondaryEmailValue = row[SecondaryEmail];
                    if (!string.IsNullOrWhiteSpace(secondaryEmailValue))
                    {
                        AddPersonAttribute(secondaryEmailAttribute, person, secondaryEmailValue);
                    }

                    DateTime membershipDateValue;
                    if (DateTime.TryParseExact(row[MembershipDate], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out membershipDateValue))
                    {
                        AddPersonAttribute(membershipDateAttribute, person, membershipDateValue.ToString());
                    }

                    DateTime baptismDateValue;
                    if (DateTime.TryParseExact(row[BaptismDate], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out baptismDateValue))
                    {
                        AddPersonAttribute(baptismDateAttribute, person, baptismDateValue.ToString());
                    }

                    DateTime firstVisitValue;
                    if (DateTime.TryParseExact(row[FirstVisit], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out firstVisitValue))
                    {
                        AddPersonAttribute(firstVisitAttribute, person, firstVisitValue.ToString());
                    }

                    string previousChurchValue = row[PreviousChurch];
                    if (!string.IsNullOrWhiteSpace(previousChurchValue))
                    {
                        AddPersonAttribute(previousChurchAttribute, person, previousChurchValue);
                    }

                    string positionValue = row[Occupation];
                    if (!string.IsNullOrWhiteSpace(positionValue))
                    {
                        AddPersonAttribute(positionAttribute, person, positionValue);
                    }

                    string employerValue = row[Employer];
                    if (!string.IsNullOrWhiteSpace(employerValue))
                    {
                        AddPersonAttribute(employerAttribute, person, employerValue);
                    }

                    string schoolName = row[School];
                    if (!string.IsNullOrWhiteSpace(schoolName))
                    {
                        // Add school if it doesn't exist
                        Guid schoolGuid;
                        var  schoolExists = schoolDefinedType.DefinedValues.Any(s => s.Value.Equals(schoolName));
                        if (!schoolExists)
                        {
                            var newSchool = new DefinedValue();
                            newSchool.DefinedTypeId = schoolDefinedType.Id;
                            newSchool.Value         = schoolName;
                            newSchool.Order         = 0;

                            lookupContext.DefinedValues.Add(newSchool);
                            lookupContext.SaveChanges();

                            schoolGuid = newSchool.Guid;
                        }
                        else
                        {
                            schoolGuid = schoolDefinedType.DefinedValues.FirstOrDefault(s => s.Value.Equals(schoolName)).Guid;
                        }

                        AddPersonAttribute(schoolAttribute, person, schoolGuid.ToString());
                    }

                    string facebookValue = row[Facebook];
                    if (!string.IsNullOrWhiteSpace(facebookValue))
                    {
                        AddPersonAttribute(facebookAttribute, person, facebookValue);
                    }

                    string twitterValue = row[Twitter];
                    if (!string.IsNullOrWhiteSpace(twitterValue))
                    {
                        AddPersonAttribute(twitterAttribute, person, twitterValue);
                    }

                    string instagramValue = row[Instagram];
                    if (!string.IsNullOrWhiteSpace(instagramValue))
                    {
                        AddPersonAttribute(instagramAttribute, person, instagramValue);
                    }

                    foreach (var attributePair in customAttributes)
                    {
                        string newAttributeValue = row[attributePair.Key];
                        if (!string.IsNullOrWhiteSpace(newAttributeValue))
                        {
                            int?newAttributeId = personAttributes.Where(a => a.Key == attributePair.Value)
                                                 .Select(a => (int?)a.Id).FirstOrDefault();
                            if (newAttributeId != null)
                            {
                                var newAttribute = AttributeCache.Read((int)newAttributeId);
                                AddPersonAttribute(newAttribute, person, newAttributeValue);
                            }
                        }
                    }

                    // Add notes to timeline
                    var notePairs = new Dictionary <string, string>();
                    notePairs.Add("General", row[GeneralNote]);
                    notePairs.Add("Medical", row[MedicalNote]);
                    notePairs.Add("Security", row[SecurityNote]);

                    var newNoteList = new List <Note>();
                    foreach (var notePair in notePairs.Where(n => !string.IsNullOrWhiteSpace(n.Value)))
                    {
                        var newNote = new Note();
                        newNote.CreatedByPersonAliasId = ImportPersonAlias.Id;
                        newNote.CreatedDateTime        = importDate;
                        newNote.EntityId   = person.Id;
                        newNote.Text       = notePair.Value;
                        newNote.NoteTypeId = noteTimelineTypeId;
                        newNote.Caption    = string.Format("{0} Note", notePair.Key);

                        if (!notePair.Key.Equals("General"))
                        {
                            newNote.IsAlert = true;
                        }

                        newNoteList.Add(newNote);
                    }

                    if (newNoteList.Any())
                    {
                        lookupContext.Notes.AddRange(newNoteList);
                        lookupContext.SaveChanges(true);
                    }

                    #endregion

                    var groupMember = new GroupMember();
                    groupMember.Person            = person;
                    groupMember.GroupRoleId       = groupRoleId;
                    groupMember.GroupMemberStatus = GroupMemberStatus.Active;

                    if (isFamilyRelationship || currentFamilyGroup.Members.Count() < 1)
                    {
                        currentFamilyGroup.Members.Add(groupMember);
                        newFamilyList.Add(currentFamilyGroup);
                        completed++;
                    }
                    else
                    {
                        var visitorGroup = new Group();
                        visitorGroup.ForeignId = rowFamilyId.ToString();
                        visitorGroup.Members.Add(groupMember);
                        visitorGroup.GroupTypeId = FamilyGroupTypeId;
                        visitorGroup.Name        = person.LastName + " Family";
                        newFamilyList.Add(visitorGroup);
                        completed++;

                        newVisitorList.Add(visitorGroup);
                    }

                    if (completed % (ReportingNumber * 10) < 1)
                    {
                        ReportProgress(0, string.Format("{0:N0} people imported.", completed));
                    }
                    else if (completed % ReportingNumber < 1)
                    {
                        SaveIndividuals(newFamilyList, newVisitorList);
                        ReportPartialProgress();
                        newFamilyList.Clear();
                    }
                }
            }

            if (newFamilyList.Any())
            {
                SaveIndividuals(newFamilyList, newVisitorList);
            }

            ReportProgress(0, string.Format("Finished individual import: {0:N0} people imported.", completed));
            return(completed);
        }
        /// <summary>
        /// Loads the individual data.
        /// </summary>
        /// <param name="csvData">The CSV data.</param>
        private int LoadIndividuals(CSVInstance csvData)
        {
            var lookupContext        = new RockContext();
            var groupTypeRoleService = new GroupTypeRoleService(lookupContext);
            var groupMemberService   = new GroupMemberService(lookupContext);

            // Marital statuses: Married, Single, Separated, etc
            var maritalStatusTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_MARITAL_STATUS), lookupContext).DefinedValues;

            // Connection statuses: Member, Visitor, Attendee, etc
            var connectionStatusTypes      = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_CONNECTION_STATUS), lookupContext).DefinedValues;
            int memberConnectionStatusId   = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_MEMBER)).Id;
            int visitorConnectionStatusId  = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_VISITOR)).Id;
            int attendeeConnectionStatusId = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_ATTENDEE)).Id;

            // Suffix type: Dr., Jr., II, etc
            var suffixTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_SUFFIX), lookupContext).DefinedValues;

            // Title type: Mr., Mrs. Dr., etc
            var titleTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_TITLE), lookupContext).DefinedValues;

            // Record statuses: Active, Inactive, Pending
            int?recordStatusActiveId   = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_ACTIVE), lookupContext).Id;
            int?recordStatusInactiveId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_INACTIVE), lookupContext).Id;
            int?recordStatusPendingId  = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_PENDING), lookupContext).Id;

            // Deceased record status reason (others available: No Activity, Moved, etc)
            var recordStatusDeceasedId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_REASON_DECEASED)).Id;

            // Record type: Person
            int?personRecordTypeId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_TYPE_PERSON), lookupContext).Id;

            // Group roles: Owner, Adult, Child, others
            GroupTypeRole ownerRole   = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_OWNER));
            int           adultRoleId = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_FAMILY_MEMBER_ADULT)).Id;
            int           childRoleId = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_FAMILY_MEMBER_CHILD)).Id;

            // Phone types: Home, Work, Mobile
            var numberTypeValues = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_PHONE_TYPE), lookupContext).DefinedValues;

            // Personal note type id
            var personalNoteTypeId = new NoteTypeService(lookupContext).Get(new Guid(Rock.SystemGuid.NoteType.PERSON_TIMELINE_NOTE)).Id;

            // School defined type
            var schoolDefinedType = DefinedTypeCache.Read(new Guid("576FF1E2-6225-4565-A16D-230E26167A3D"));

            // Look up existing Person attributes
            var personAttributes = new AttributeService(lookupContext).GetByEntityTypeId(PersonEntityTypeId).ToList();
            var schoolAttribute  = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "School"));

            // Text field type id
            int textFieldTypeId = FieldTypeCache.Read(new Guid(Rock.SystemGuid.FieldType.TEXT), lookupContext).Id;
            int dateFieldTypeId = FieldTypeCache.Read(new Guid(Rock.SystemGuid.FieldType.DATE), lookupContext).Id;

            // Attribute entity type id
            int attributeEntityTypeId = EntityTypeCache.Read("Rock.Model.Attribute").Id;

            // Visit info category
            var visitInfoCategory = new CategoryService(lookupContext).GetByEntityTypeId(attributeEntityTypeId)
                                    .Where(c => c.Name == "Visit Information").FirstOrDefault();

            // Look for custom attributes in the Individual file
            var allFields = csvData.TableNodes.FirstOrDefault().Children.Select((node, index) => new { node = node, index = index }).ToList();
            Dictionary <int, string> customAttributes = allFields
                                                        .Where(f => f.index > SecurityNote)
                                                        .ToDictionary(f => f.index, f => f.node.Name.RemoveWhitespace());

            // Add any attributes if they don't already exist
            if (customAttributes.Any())
            {
                var newAttributes = new List <Rock.Model.Attribute>();
                foreach (var newAttributePair in customAttributes.Where(ca => !personAttributes.Any(a => a.Key == ca.Value)))
                {
                    var newAttribute = new Rock.Model.Attribute();
                    newAttribute.Name        = newAttributePair.Value;
                    newAttribute.Key         = newAttributePair.Value.RemoveWhitespace();
                    newAttribute.Description = newAttributePair.Value + " created by CSV import";
                    newAttribute.EntityTypeQualifierValue  = string.Empty;
                    newAttribute.EntityTypeQualifierColumn = string.Empty;
                    newAttribute.EntityTypeId = PersonEntityTypeId;
                    newAttribute.FieldTypeId  = textFieldTypeId;
                    newAttribute.DefaultValue = string.Empty;
                    newAttribute.IsMultiValue = false;
                    newAttribute.IsGridColumn = false;
                    newAttribute.IsRequired   = false;
                    newAttribute.Order        = 0;
                    newAttributes.Add(newAttribute);
                }

                lookupContext.Attributes.AddRange(newAttributes);
                lookupContext.SaveChanges(DisableAuditing);
                personAttributes.AddRange(newAttributes);
            }

            // Set the supported date formats
            var dateFormats = new[] { "yyyy-MM-dd", "MM/dd/yyyy", "MM/dd/yy" };

            var currentFamilyGroup = new Group();
            var newFamilyList      = new List <Group>();
            var newVisitorList     = new List <Group>();
            var newNoteList        = new List <Note>();

            int completed   = 0;
            int newFamilies = 0;

            ReportProgress(0, string.Format("Starting Individual import ({0:N0} already exist).", ImportedPeople.Count(p => p.Members.Any(m => m.Person.ForeignKey != null))));

            string[] row;
            // Uses a look-ahead enumerator: this call will move to the next record immediately
            while ((row = csvData.Database.FirstOrDefault()) != null)
            {
                int  groupRoleId          = adultRoleId;
                bool isFamilyRelationship = true;

                string rowFamilyName = row[FamilyName];
                string rowFamilyKey  = row[FamilyId];
                string rowPersonKey  = row[PersonId];
                int?   rowFamilyId   = rowFamilyKey.AsType <int?>();
                int?   rowPersonId   = rowPersonKey.AsType <int?>();

                // Check that this person isn't already in our data
                var personExists = ImportedPeople.Any(p => p.Members.Any(m => m.Person.ForeignKey == rowPersonKey));
                if (!personExists)
                {
                    #region person create

                    var person = new Person();
                    person.ForeignKey             = rowPersonKey;
                    person.ForeignId              = rowPersonId;
                    person.SystemNote             = string.Format("Imported via Excavator on {0}", ImportDateTime);
                    person.RecordTypeValueId      = personRecordTypeId;
                    person.CreatedByPersonAliasId = ImportPersonAliasId;
                    string firstName = row[FirstName].Left(50);
                    string nickName  = row[NickName].Left(50);
                    person.FirstName  = firstName;
                    person.NickName   = string.IsNullOrWhiteSpace(nickName) ? firstName : nickName;
                    person.MiddleName = row[MiddleName].Left(50);
                    person.LastName   = row[LastName].Left(50);

                    DateTime createdDateValue;
                    if (DateTime.TryParseExact(row[CreatedDate], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out createdDateValue))
                    {
                        person.CreatedDateTime  = createdDateValue;
                        person.ModifiedDateTime = ImportDateTime;
                    }
                    else
                    {
                        person.CreatedDateTime  = ImportDateTime;
                        person.ModifiedDateTime = ImportDateTime;
                    }

                    DateTime birthDate;
                    if (DateTime.TryParseExact(row[DateOfBirth], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out birthDate))
                    {
                        person.BirthDay   = birthDate.Day;
                        person.BirthMonth = birthDate.Month;
                        person.BirthYear  = birthDate.Year;
                    }

                    DateTime graduationDate;
                    if (DateTime.TryParseExact(row[GraduationDate], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out graduationDate))
                    {
                        person.GraduationYear = graduationDate.Year;
                    }

                    DateTime anniversary;
                    if (DateTime.TryParseExact(row[Anniversary], dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out anniversary))
                    {
                        person.AnniversaryDate = anniversary;
                    }

                    string gender = row[Gender];
                    if (gender != null)
                    {
                        switch (gender.Trim().ToLower())
                        {
                        case "m":
                        case "male":
                            person.Gender = Rock.Model.Gender.Male;
                            break;

                        case "f":
                        case "female":
                            person.Gender = Rock.Model.Gender.Female;
                            break;

                        default:
                            person.Gender = Rock.Model.Gender.Unknown;
                            break;
                        }
                    }

                    string prefix = row[Prefix];
                    if (!string.IsNullOrWhiteSpace(prefix))
                    {
                        prefix = prefix.RemoveSpecialCharacters().Trim();
                        person.TitleValueId = titleTypes.Where(s => prefix == s.Value.RemoveSpecialCharacters())
                                              .Select(s => (int?)s.Id).FirstOrDefault();
                    }

                    string suffix = row[Suffix];
                    if (!string.IsNullOrWhiteSpace(suffix))
                    {
                        suffix = suffix.RemoveSpecialCharacters().Trim();
                        person.SuffixValueId = suffixTypes.Where(s => suffix == s.Value.RemoveSpecialCharacters())
                                               .Select(s => (int?)s.Id).FirstOrDefault();
                    }

                    string maritalStatus = row[MaritalStatus];
                    if (!string.IsNullOrWhiteSpace(maritalStatus))
                    {
                        person.MaritalStatusValueId = maritalStatusTypes.Where(dv => dv.Value == maritalStatus)
                                                      .Select(dv => (int?)dv.Id).FirstOrDefault();
                    }
                    else
                    {
                        person.MaritalStatusValueId = maritalStatusTypes.Where(dv => dv.Value == "Unknown")
                                                      .Select(dv => (int?)dv.Id).FirstOrDefault();
                    }

                    string familyRole = row[FamilyRole];
                    if (!string.IsNullOrWhiteSpace(familyRole))
                    {
                        if (familyRole == "Visitor")
                        {
                            isFamilyRelationship = false;
                        }

                        if (familyRole == "Child" || person.Age < 18)
                        {
                            groupRoleId = childRoleId;
                        }
                    }

                    string connectionStatus = row[ConnectionStatus];
                    if (!string.IsNullOrWhiteSpace(connectionStatus))
                    {
                        if (connectionStatus == "Member")
                        {
                            person.ConnectionStatusValueId = memberConnectionStatusId;
                        }
                        else if (connectionStatus == "Visitor")
                        {
                            person.ConnectionStatusValueId = visitorConnectionStatusId;
                        }
                        else
                        {
                            // look for user-defined connection type or default to Attendee
                            var customConnectionType = connectionStatusTypes.Where(dv => dv.Value == connectionStatus)
                                                       .Select(dv => (int?)dv.Id).FirstOrDefault();

                            person.ConnectionStatusValueId = customConnectionType ?? attendeeConnectionStatusId;
                            person.RecordStatusValueId     = recordStatusActiveId;
                        }
                    }

                    string recordStatus = row[RecordStatus];
                    if (!string.IsNullOrWhiteSpace(recordStatus))
                    {
                        switch (recordStatus.Trim().ToLower())
                        {
                        case "active":
                            person.RecordStatusValueId = recordStatusActiveId;
                            break;

                        case "inactive":
                            person.RecordStatusValueId = recordStatusInactiveId;
                            break;

                        default:
                            person.RecordStatusValueId = recordStatusPendingId;
                            break;
                        }
                    }

                    string isDeceasedValue = row[IsDeceased];
                    if (!string.IsNullOrWhiteSpace(isDeceasedValue))
                    {
                        switch (isDeceasedValue.Trim().ToLower())
                        {
                        case "y":
                        case "yes":
                            person.IsDeceased = true;
                            person.RecordStatusReasonValueId = recordStatusDeceasedId;
                            person.RecordStatusValueId       = recordStatusInactiveId;
                            break;

                        default:
                            person.IsDeceased = false;
                            break;
                        }
                    }

                    var personNumbers = new Dictionary <string, string>();
                    personNumbers.Add("Home", row[HomePhone]);
                    personNumbers.Add("Mobile", row[MobilePhone]);
                    personNumbers.Add("Work", row[WorkPhone]);
                    string smsAllowed = row[AllowSMS];

                    foreach (var numberPair in personNumbers.Where(n => !string.IsNullOrWhiteSpace(n.Value)))
                    {
                        var extension        = string.Empty;
                        var countryCode      = Rock.Model.PhoneNumber.DefaultCountryCode();
                        var normalizedNumber = string.Empty;
                        var countryIndex     = numberPair.Value.IndexOf('+');
                        int extensionIndex   = numberPair.Value.LastIndexOf('x') > 0 ? numberPair.Value.LastIndexOf('x') : numberPair.Value.Length;
                        if (countryIndex >= 0)
                        {
                            countryCode      = numberPair.Value.Substring(countryIndex, countryIndex + 3).AsNumeric();
                            normalizedNumber = numberPair.Value.Substring(countryIndex + 3, extensionIndex - 3).AsNumeric().TrimStart(new Char[] { '0' });
                            extension        = numberPair.Value.Substring(extensionIndex);
                        }
                        else if (extensionIndex > 0)
                        {
                            normalizedNumber = numberPair.Value.Substring(0, extensionIndex).AsNumeric();
                            extension        = numberPair.Value.Substring(extensionIndex).AsNumeric();
                        }
                        else
                        {
                            normalizedNumber = numberPair.Value.AsNumeric();
                        }

                        if (!string.IsNullOrWhiteSpace(normalizedNumber))
                        {
                            var currentNumber = new PhoneNumber();
                            currentNumber.CountryCode            = countryCode;
                            currentNumber.CreatedByPersonAliasId = ImportPersonAliasId;
                            currentNumber.Extension         = extension.Left(20);
                            currentNumber.Number            = normalizedNumber.TrimStart(new Char[] { '0' }).Left(20);
                            currentNumber.NumberTypeValueId = numberTypeValues.Where(v => v.Value.Equals(numberPair.Key))
                                                              .Select(v => (int?)v.Id).FirstOrDefault();
                            if (numberPair.Key == "Mobile")
                            {
                                switch (smsAllowed.Trim().ToLower())
                                {
                                case "y":
                                case "yes":
                                case "active":
                                    currentNumber.IsMessagingEnabled = true;
                                    break;

                                default:
                                    currentNumber.IsMessagingEnabled = false;
                                    break;
                                }
                            }

                            person.PhoneNumbers.Add(currentNumber);
                        }
                    }

                    // Map Person attributes
                    person.Attributes      = new Dictionary <string, AttributeCache>();
                    person.AttributeValues = new Dictionary <string, AttributeValueCache>();

                    bool isEmailActive;
                    switch (row[IsEmailActive].Trim().ToLower())
                    {
                    case "n":
                    case "no":
                    case "inactive":
                        isEmailActive = false;
                        break;

                    default:
                        isEmailActive = true;
                        break;
                    }

                    EmailPreference emailPreference;
                    switch (row[AllowBulkEmail].Trim().ToLower())
                    {
                    case "n":
                    case "no":
                    case "inactive":
                        emailPreference = EmailPreference.NoMassEmails;
                        break;

                    default:
                        emailPreference = EmailPreference.EmailAllowed;
                        break;
                    }

                    person.EmailPreference = emailPreference;
                    string primaryEmail = row[Email].Trim().Left(75);
                    if (!string.IsNullOrWhiteSpace(primaryEmail))
                    {
                        if (primaryEmail.IsEmail())
                        {
                            person.Email         = primaryEmail;
                            person.IsEmailActive = isEmailActive;
                        }
                        else
                        {
                            LogException("InvalidPrimaryEmail", string.Format("PersonId: {0} - Email: {1}", rowPersonKey, primaryEmail));
                        }
                    }

                    string schoolName = row[School];
                    if (!string.IsNullOrWhiteSpace(schoolName))
                    {
                        // Add school if it doesn't exist
                        Guid schoolGuid;
                        var  schoolExists = schoolDefinedType.DefinedValues.Any(s => s.Value.Equals(schoolName));
                        if (!schoolExists)
                        {
                            var newSchool = new DefinedValue();
                            newSchool.DefinedTypeId = schoolDefinedType.Id;
                            newSchool.Value         = schoolName;
                            newSchool.Order         = 0;

                            lookupContext.DefinedValues.Add(newSchool);
                            lookupContext.SaveChanges();

                            schoolGuid = newSchool.Guid;
                        }
                        else
                        {
                            schoolGuid = schoolDefinedType.DefinedValues.FirstOrDefault(s => s.Value.Equals(schoolName)).Guid;
                        }

                        AddPersonAttribute(schoolAttribute, person, schoolGuid.ToString().ToUpper());
                    }

                    foreach (var attributePair in customAttributes)
                    {
                        string newAttributeValue = row[attributePair.Key];
                        if (!string.IsNullOrWhiteSpace(newAttributeValue))
                        {
                            // check if this attribute value is a date
                            DateTime valueAsDateTime;
                            if (DateTime.TryParseExact(newAttributeValue, dateFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out valueAsDateTime))
                            {
                                newAttributeValue = valueAsDateTime.ToString("yyyy-MM-dd");
                            }

                            int?newAttributeId = personAttributes.Where(a => a.Key == attributePair.Value.RemoveWhitespace())
                                                 .Select(a => (int?)a.Id).FirstOrDefault();
                            if (newAttributeId != null)
                            {
                                var newAttribute = AttributeCache.Read((int)newAttributeId);
                                AddPersonAttribute(newAttribute, person, newAttributeValue);
                            }
                        }
                    }

                    // Add notes to timeline
                    var notePairs = new Dictionary <string, string>();
                    notePairs.Add("General", row[GeneralNote]);
                    notePairs.Add("Medical", row[MedicalNote]);
                    notePairs.Add("Security", row[SecurityNote]);

                    foreach (var notePair in notePairs.Where(n => !string.IsNullOrWhiteSpace(n.Value)))
                    {
                        var newNote = new Note();
                        newNote.NoteTypeId             = personalNoteTypeId;
                        newNote.CreatedByPersonAliasId = ImportPersonAliasId;
                        newNote.CreatedDateTime        = ImportDateTime;
                        newNote.Text       = notePair.Value;
                        newNote.ForeignKey = rowPersonKey;
                        newNote.ForeignId  = rowPersonId;
                        newNote.Caption    = string.Format("{0} Note", notePair.Key);

                        if (!notePair.Key.Equals("General"))
                        {
                            newNote.IsAlert = true;
                        }

                        newNoteList.Add(newNote);
                    }

                    #endregion person create

                    var groupMember = new GroupMember();
                    groupMember.Person                 = person;
                    groupMember.GroupRoleId            = groupRoleId;
                    groupMember.CreatedDateTime        = ImportDateTime;
                    groupMember.ModifiedDateTime       = ImportDateTime;
                    groupMember.CreatedByPersonAliasId = ImportPersonAliasId;
                    groupMember.GroupMemberStatus      = GroupMemberStatus.Active;

                    if (rowFamilyKey != currentFamilyGroup.ForeignKey)
                    {
                        // person not part of the previous family, see if that family exists or create a new one
                        currentFamilyGroup = ImportedPeople.FirstOrDefault(p => p.ForeignKey == rowFamilyKey);
                        if (currentFamilyGroup == null)
                        {
                            currentFamilyGroup = CreateFamilyGroup(row[FamilyName], rowFamilyKey);
                            newFamilyList.Add(currentFamilyGroup);
                            newFamilies++;
                        }
                        else
                        {
                            lookupContext.Groups.Attach(currentFamilyGroup);
                            lookupContext.Entry(currentFamilyGroup).State = EntityState.Modified;
                        }

                        currentFamilyGroup.Members.Add(groupMember);
                    }
                    else
                    {
                        // person is part of this family group, check if they're a visitor
                        if (isFamilyRelationship || currentFamilyGroup.Members.Count() < 1)
                        {
                            currentFamilyGroup.Members.Add(groupMember);
                        }
                        else
                        {
                            var visitorFamily = CreateFamilyGroup(person.LastName + " Family", rowFamilyKey);
                            visitorFamily.Members.Add(groupMember);
                            newFamilyList.Add(visitorFamily);
                            newVisitorList.Add(visitorFamily);
                            newFamilies++;
                        }
                    }

                    completed++;
                    if (completed % (ReportingNumber * 10) < 1)
                    {
                        ReportProgress(0, string.Format("{0:N0} people imported.", completed));
                    }
                    else if (completed % ReportingNumber < 1)
                    {
                        SaveIndividuals(newFamilyList, newVisitorList, newNoteList);
                        lookupContext.SaveChanges();
                        ReportPartialProgress();

                        // Clear out variables
                        currentFamilyGroup = new Group();
                        newFamilyList.Clear();
                        newVisitorList.Clear();
                        newNoteList.Clear();
                    }
                }
            }

            // Save any changes to new families
            if (newFamilyList.Any())
            {
                SaveIndividuals(newFamilyList, newVisitorList, newNoteList);
            }

            // Save any changes to existing families
            lookupContext.SaveChanges();
            lookupContext.Dispose();

            ReportProgress(0, string.Format("Finished individual import: {0:N0} families and {1:N0} people added.", newFamilies, completed));
            return(completed);
        }
Exemplo n.º 9
0
        /// <summary>
        /// Maps the specified folder.
        /// </summary>
        /// <param name="folder">The folder.</param>
        /// <param name="ministryFileType">Type of the ministry file.</param>
        public int Map(ZipArchive folder, BinaryFileType ministryFileType)
        {
            var lookupContext         = new RockContext();
            var personEntityTypeId    = EntityTypeCache.GetId <Person>();
            var binaryFileTypeService = new BinaryFileTypeService(lookupContext);
            var fileFieldTypeId       = FieldTypeCache.Get(Rock.SystemGuid.FieldType.FILE.AsGuid(), lookupContext).Id;
            var backgroundFieldTypeId = FieldTypeCache.Get(Rock.SystemGuid.FieldType.BACKGROUNDCHECK.AsGuid(), lookupContext).Id;

            var existingAttributes = new AttributeService(lookupContext).GetByFieldTypeId(fileFieldTypeId)
                                     .Where(a => a.EntityTypeId == personEntityTypeId)
                                     .ToDictionary(a => a.Key, a => a);

            var backgroundCheckFileAttributes = new AttributeService(lookupContext).GetByFieldTypeId(backgroundFieldTypeId)
                                                .Where(a => a.EntityTypeId == personEntityTypeId)
                                                .ToDictionary(a => a.Key, a => a);

            foreach (var backgroundCheckFileAttribute in backgroundCheckFileAttributes)
            {
                if (!existingAttributes.ContainsKey(backgroundCheckFileAttribute.Key))
                {
                    existingAttributes.Add(backgroundCheckFileAttribute.Key, backgroundCheckFileAttribute.Value);
                }
            }

            var emptyJsonObject = "{}";
            var newFileList     = new List <DocumentKeys>();

            var completedItems = 0;
            var totalRows      = folder.Entries.Count;
            var percentage     = (totalRows - 1) / 100 + 1;

            ReportProgress(0, string.Format("Verifying ministry document import ({0:N0} found)", totalRows));

            foreach (var file in folder.Entries.OrderBy(f => f.Name))
            {
                var fileExtension = Path.GetExtension(file.Name);
                if (FileTypeBlackList.Contains(fileExtension))
                {
                    LogException("Binary File Import", string.Format("{0} filetype not allowed ({1})", fileExtension, file.Name));
                    continue;
                }

                var nameWithoutExtension = file.Name.ReplaceLastOccurrence(fileExtension, string.Empty);
                var parsedFileName       = nameWithoutExtension.Split('_');
                // Ministry docs should follow this pattern:
                // 0. Firstname
                // 1. Lastname
                // 2. ForeignId
                // 3. Filename
                // 4. Doc Id
                if (parsedFileName.Length < 3)
                {
                    break;
                }

                var personForeignId = parsedFileName[2].AsType <int?>();
                var personKeys      = ImportedPeople.FirstOrDefault(p => p.PersonForeignId == personForeignId);
                if (personKeys != null)
                {
                    var attributeName     = string.Empty;
                    var documentForeignId = string.Empty;
                    if (parsedFileName.Count() > 4)
                    {
                        attributeName     = parsedFileName[3];
                        documentForeignId = parsedFileName[4];
                    }
                    else
                    {
                        var filename = parsedFileName[3].ReplaceLastOccurrence(fileExtension, string.Empty);
                        attributeName     = Regex.Replace(filename, "\\d{4,}[.\\w]+$", string.Empty);
                        documentForeignId = Regex.Match(filename, "\\d+$").Value;
                    }

                    // append "Document" to attribute name to create unique attributes
                    // this matches core attribute "Background Check Document"
                    attributeName = !attributeName.EndsWith("Document", StringComparison.OrdinalIgnoreCase) ? string.Format("{0} Document", attributeName) : attributeName;
                    var attributeKey = attributeName.RemoveSpecialCharacters();

                    Attribute fileAttribute           = null;
                    var       attributeBinaryFileType = ministryFileType;
                    if (!existingAttributes.ContainsKey(attributeKey))
                    {
                        fileAttribute = new Attribute
                        {
                            FieldTypeId  = fileFieldTypeId,
                            EntityTypeId = personEntityTypeId,
                            EntityTypeQualifierColumn = string.Empty,
                            EntityTypeQualifierValue  = string.Empty,
                            Key          = attributeKey,
                            Name         = attributeName,
                            Description  = string.Format("{0} created by binary file import", attributeName),
                            IsGridColumn = false,
                            IsMultiValue = false,
                            IsRequired   = false,
                            AllowSearch  = false,
                            IsSystem     = false,
                            Order        = 0
                        };

                        fileAttribute.AttributeQualifiers.Add(new AttributeQualifier()
                        {
                            Key   = "binaryFileType",
                            Value = ministryFileType.Guid.ToString()
                        });

                        lookupContext.Attributes.Add(fileAttribute);
                        lookupContext.SaveChanges();

                        existingAttributes.Add(fileAttribute.Key, fileAttribute);
                    }
                    else
                    {
                        // if attribute already exists in Rock, override default file type with the Rock-specified file type
                        fileAttribute = existingAttributes[attributeKey];
                        var attributeBinaryFileTypeGuid = fileAttribute.AttributeQualifiers.FirstOrDefault(q => q.Key.Equals("binaryFileType"));
                        if (attributeBinaryFileTypeGuid != null)
                        {
                            attributeBinaryFileType = binaryFileTypeService.Get(attributeBinaryFileTypeGuid.Value.AsGuid());
                        }
                    }

                    var rockFile = new Rock.Model.BinaryFile
                    {
                        IsSystem               = false,
                        IsTemporary            = false,
                        MimeType               = GetMIMEType(file.Name),
                        BinaryFileTypeId       = attributeBinaryFileType.Id,
                        FileName               = file.Name,
                        Description            = string.Format("Imported as {0}", file.Name),
                        CreatedDateTime        = file.LastWriteTime.DateTime,
                        ModifiedDateTime       = file.LastWriteTime.DateTime,
                        CreatedByPersonAliasId = ImportPersonAliasId,
                        ForeignKey             = documentForeignId,
                        ForeignId              = documentForeignId.AsIntegerOrNull()
                    };

                    rockFile.SetStorageEntityTypeId(attributeBinaryFileType.StorageEntityTypeId);
                    rockFile.StorageEntitySettings = emptyJsonObject;

                    if (attributeBinaryFileType.AttributeValues != null)
                    {
                        rockFile.StorageEntitySettings = attributeBinaryFileType.AttributeValues
                                                         .ToDictionary(a => a.Key, v => v.Value.Value).ToJson();
                    }

                    // use base stream instead of file stream to keep the byte[]
                    // NOTE: if byte[] converts to a string it will corrupt the stream
                    using (var fileContent = new StreamReader(file.Open()))
                    {
                        rockFile.ContentStream = new MemoryStream(fileContent.BaseStream.ReadBytesToEnd());
                    }

                    newFileList.Add(new DocumentKeys()
                    {
                        PersonId    = personKeys.PersonId,
                        AttributeId = fileAttribute.Id,
                        File        = rockFile
                    });

                    completedItems++;

                    if (completedItems % percentage < 1)
                    {
                        var percentComplete = completedItems / percentage;
                        ReportProgress(percentComplete, string.Format("{0:N0} ministry document files imported ({1}% complete).", completedItems, percentComplete));
                    }

                    if (completedItems % ReportingNumber < 1)
                    {
                        SaveFiles(newFileList);

                        // Reset list
                        newFileList.Clear();
                        ReportPartialProgress();
                    }
                }
            }

            if (newFileList.Any())
            {
                SaveFiles(newFileList);
            }

            ReportProgress(100, string.Format("Finished documents import: {0:N0} ministry documents imported.", completedItems));
            return(completedItems);
        }
Exemplo n.º 10
0
        /// <summary>
        /// Maps the person.
        /// </summary>
        /// <param name="tableData">The table data.</param>
        /// <param name="selectedColumns">The selected columns.</param>
        private void MapPerson(IQueryable <Row> tableData, List <string> selectedColumns = null)
        {
            var lookupContext        = new RockContext();
            var groupTypeRoleService = new GroupTypeRoleService(lookupContext);

            // Marital statuses: Married, Single, Separated, etc
            var maritalStatusTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_MARITAL_STATUS), lookupContext).DefinedValues;

            // Connection statuses: Member, Visitor, Attendee, etc
            var connectionStatusTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_CONNECTION_STATUS), lookupContext).DefinedValues;

            // Record status reasons: No Activity, Moved, Deceased, etc
            var recordStatusReasons = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_RECORD_STATUS_REASON), lookupContext).DefinedValues;

            // Record statuses: Active, Inactive, Pending
            int?recordStatusActiveId   = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_ACTIVE), lookupContext).Id;
            int?recordStatusInactiveId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_INACTIVE), lookupContext).Id;
            int?recordStatusPendingId  = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_STATUS_PENDING), lookupContext).Id;

            // Record type: Person
            int?personRecordTypeId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.PERSON_RECORD_TYPE_PERSON), lookupContext).Id;

            // Suffix type: Dr., Jr., II, etc
            var suffixTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_SUFFIX)).DefinedValues;

            // Title type: Mr., Mrs. Dr., etc
            var titleTypes = DefinedTypeCache.Read(new Guid(Rock.SystemGuid.DefinedType.PERSON_TITLE), lookupContext).DefinedValues;

            // Note type: Comment
            int noteCommentTypeId = new NoteTypeService(lookupContext).Get(new Guid("7E53487C-D650-4D85-97E2-350EB8332763")).Id;

            // Group roles: Owner, Adult, Child, others
            GroupTypeRole ownerRole            = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_OWNER));
            int           adultRoleId          = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_FAMILY_MEMBER_ADULT)).Id;
            int           childRoleId          = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_FAMILY_MEMBER_CHILD)).Id;
            int           inviteeRoleId        = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_INVITED)).Id;
            int           invitedByRoleId      = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_INVITED_BY)).Id;
            int           canCheckInRoleId     = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_CAN_CHECK_IN)).Id;
            int           allowCheckInByRoleId = groupTypeRoleService.Get(new Guid(Rock.SystemGuid.GroupRole.GROUPROLE_KNOWN_RELATIONSHIPS_ALLOW_CHECK_IN_BY)).Id;

            // Group type: Family
            int familyGroupTypeId = new GroupTypeService(lookupContext).Get(new Guid(Rock.SystemGuid.GroupType.GROUPTYPE_FAMILY)).Id;

            // Look up additional Person attributes (existing)
            var personAttributes = new AttributeService(lookupContext).GetByEntityTypeId(PersonEntityTypeId).ToList();

            // Cached F1 attributes: IndividualId, HouseholdId
            // Core attributes: PreviousChurch, Position, Employer, School
            var individualIdAttribute   = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "F1IndividualId"));
            var householdIdAttribute    = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "F1HouseholdId"));
            var previousChurchAttribute = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "PreviousChurch"));
            var employerAttribute       = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Employer"));
            var positionAttribute       = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "Position"));
            var firstVisitAttribute     = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "FirstVisit"));
            var schoolAttribute         = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "School"));
            var membershipDateAttribute = AttributeCache.Read(personAttributes.FirstOrDefault(a => a.Key == "MembershipDate"));

            var familyList  = new List <Group>();
            var visitorList = new List <Group>();

            int completed  = 0;
            int totalRows  = tableData.Count();
            int percentage = (totalRows - 1) / 100 + 1;

            ReportProgress(0, string.Format("Verifying person import ({0:N0} found, {1:N0} already exist).", totalRows, ImportedPeople.Count()));

            foreach (var groupedRows in tableData.GroupBy <Row, int?>(r => r["Household_ID"] as int?))
            {
                var familyGroup         = new Group();
                var householdCampusList = new List <string>();

                foreach (var row in groupedRows)
                {
                    bool   isFamilyRelationship = true;
                    string currentCampus        = string.Empty;
                    int?   individualId         = row["Individual_ID"] as int?;
                    int?   householdId          = row["Household_ID"] as int?;
                    if (GetPersonAliasId(individualId, householdId) == null)
                    {
                        var person = new Person();
                        person.FirstName              = row["First_Name"] as string;
                        person.MiddleName             = row["Middle_Name"] as string;
                        person.NickName               = row["Goes_By"] as string ?? person.FirstName;
                        person.LastName               = row["Last_Name"] as string;
                        person.BirthDate              = row["Date_Of_Birth"] as DateTime?;
                        person.CreatedByPersonAliasId = ImportPersonAlias.Id;
                        person.RecordTypeValueId      = personRecordTypeId;
                        person.ForeignId              = individualId.ToString();
                        int groupRoleId = adultRoleId;

                        var gender = row["Gender"] as string;
                        if (gender != null)
                        {
                            person.Gender = (Gender)Enum.Parse(typeof(Gender), gender);
                        }

                        string prefix = row["Prefix"] as string;
                        if (prefix != null)
                        {
                            prefix = prefix.RemoveSpecialCharacters().Trim();
                            person.TitleValueId = titleTypes.Where(s => prefix == s.Value.RemoveSpecialCharacters())
                                                  .Select(s => (int?)s.Id).FirstOrDefault();
                        }

                        string suffix = row["Suffix"] as string;
                        if (suffix != null)
                        {
                            suffix = suffix.RemoveSpecialCharacters().Trim();
                            person.SuffixValueId = suffixTypes.Where(s => suffix == s.Value.RemoveSpecialCharacters())
                                                   .Select(s => (int?)s.Id).FirstOrDefault();
                        }

                        string maritalStatus = row["Marital_Status"] as string;
                        if (maritalStatus != null)
                        {
                            person.MaritalStatusValueId = maritalStatusTypes.Where(dv => dv.Value == maritalStatus)
                                                          .Select(dv => (int?)dv.Id).FirstOrDefault();
                        }
                        else
                        {
                            person.MaritalStatusValueId = maritalStatusTypes.Where(dv => dv.Value == "Unknown")
                                                          .Select(dv => (int?)dv.Id).FirstOrDefault();
                        }

                        string familyRole = row["Household_Position"].ToString().ToLower();
                        if (familyRole != null)
                        {
                            if (familyRole == "visitor")
                            {
                                isFamilyRelationship = false;
                            }

                            if (familyRole == "child" || person.Age < 18)
                            {
                                groupRoleId = childRoleId;
                            }
                        }

                        string memberStatus = row["Status_Name"].ToString().ToLower();
                        if (memberStatus == "member")
                        {
                            person.ConnectionStatusValueId = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_MEMBER)).Id;
                            person.RecordStatusValueId     = recordStatusActiveId;
                        }
                        else if (memberStatus == "visitor")
                        {
                            person.ConnectionStatusValueId = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_VISITOR)).Id;
                            person.RecordStatusValueId     = recordStatusActiveId;
                        }
                        else if (memberStatus == "deceased")
                        {
                            person.IsDeceased = true;
                            person.RecordStatusReasonValueId = recordStatusReasons.Where(dv => dv.Value == "Deceased")
                                                               .Select(dv => dv.Id).FirstOrDefault();
                            person.RecordStatusValueId = recordStatusInactiveId;
                        }
                        else
                        {
                            // F1 defaults are Member & Visitor; all others are user-defined
                            var customConnectionType = connectionStatusTypes.Where(dv => dv.Value == memberStatus)
                                                       .Select(dv => (int?)dv.Id).FirstOrDefault();

                            int attendeeId = connectionStatusTypes.FirstOrDefault(dv => dv.Guid == new Guid(Rock.SystemGuid.DefinedValue.PERSON_CONNECTION_STATUS_ATTENDEE)).Id;
                            person.ConnectionStatusValueId = customConnectionType ?? attendeeId;
                            person.RecordStatusValueId     = recordStatusActiveId;
                        }

                        string campus = row["SubStatus_Name"] as string;
                        if (campus != null)
                        {
                            currentCampus = campus;
                        }

                        string status_comment = row["Status_Comment"] as string;
                        if (status_comment != null)
                        {
                            person.SystemNote = status_comment;
                        }

                        // Map F1 attributes
                        person.Attributes      = new Dictionary <string, AttributeCache>();
                        person.AttributeValues = new Dictionary <string, AttributeValue>();

                        // individual_id already defined in scope
                        AddPersonAttribute(individualIdAttribute, person, individualId.ToString());

                        // household_id already defined in scope
                        AddPersonAttribute(householdIdAttribute, person, householdId.ToString());

                        string previousChurch = row["Former_Church"] as string;
                        AddPersonAttribute(previousChurchAttribute, person, previousChurch);

                        string employer = row["Employer"] as string;
                        AddPersonAttribute(employerAttribute, person, employer);

                        string position = row["Occupation_Name"] as string ?? row["Occupation_Description"] as string;
                        AddPersonAttribute(positionAttribute, person, position);

                        string school = row["School_Name"] as string;
                        AddPersonAttribute(schoolAttribute, person, school);

                        DateTime?membershipDate = row["Status_Date"] as DateTime?;
                        if (membershipDate != null)
                        {
                            person.CreatedDateTime = membershipDate;
                            AddPersonAttribute(membershipDateAttribute, person, membershipDate.Value.ToString("MM/dd/yyyy"));
                        }

                        DateTime?firstVisit = row["First_Record"] as DateTime?;
                        if (firstVisit != null)
                        {
                            person.CreatedDateTime = firstVisit;
                            AddPersonAttribute(firstVisitAttribute, person, firstVisit.Value.ToString("MM/dd/yyyy"));
                        }

                        // Other Attributes to create:
                        // former name
                        // bar_code
                        // member_env_code
                        // denomination_name

                        var groupMember = new GroupMember();
                        groupMember.Person            = person;
                        groupMember.GroupRoleId       = groupRoleId;
                        groupMember.GroupMemberStatus = GroupMemberStatus.Active;

                        if (isFamilyRelationship)
                        {
                            householdCampusList.Add(currentCampus);
                            familyGroup.Members.Add(groupMember);
                            familyGroup.ForeignId = householdId.ToString();
                        }
                        else
                        {
                            var visitorGroup = new Group();
                            visitorGroup.ForeignId = householdId.ToString();
                            visitorGroup.Members.Add(groupMember);
                            visitorGroup.GroupTypeId = familyGroupTypeId;
                            visitorGroup.Name        = person.LastName + " Family";
                            visitorGroup.CampusId    = CampusList.Where(c => c.Name.StartsWith(currentCampus) || c.ShortCode == currentCampus)
                                                       .Select(c => (int?)c.Id).FirstOrDefault();
                            familyList.Add(visitorGroup);
                            completed += visitorGroup.Members.Count;

                            visitorList.Add(visitorGroup);
                        }
                    }
                }

                if (familyGroup.Members.Any())
                {
                    familyGroup.Name = familyGroup.Members.OrderByDescending(p => p.Person.Age)
                                       .FirstOrDefault().Person.LastName + " Family";
                    familyGroup.GroupTypeId = familyGroupTypeId;

                    string primaryHouseholdCampus = householdCampusList.GroupBy(c => c).OrderByDescending(c => c.Count())
                                                    .Select(c => c.Key).FirstOrDefault();
                    if (primaryHouseholdCampus != null)
                    {
                        familyGroup.CampusId = CampusList.Where(c => c.Name.StartsWith(primaryHouseholdCampus) || c.ShortCode == primaryHouseholdCampus)
                                               .Select(c => (int?)c.Id).FirstOrDefault();
                    }

                    familyList.Add(familyGroup);
                    completed += familyGroup.Members.Count;
                    if (completed % percentage < 1)
                    {
                        int percentComplete = completed / percentage;
                        ReportProgress(percentComplete, string.Format("{0:N0} people imported ({1}% complete).", completed, percentComplete));
                    }
                    else if (completed % ReportingNumber < 1)
                    {
                        SavePeople(familyList, visitorList, ownerRole, childRoleId, inviteeRoleId, invitedByRoleId, canCheckInRoleId, allowCheckInByRoleId);

                        familyList.Clear();
                        visitorList.Clear();
                        ReportPartialProgress();
                    }
                }
            }

            // Save any remaining families in the batch
            if (familyList.Any())
            {
                SavePeople(familyList, visitorList, ownerRole, childRoleId, inviteeRoleId, invitedByRoleId, canCheckInRoleId, allowCheckInByRoleId);
            }

            ReportProgress(100, string.Format("Finished person import: {0:N0} people imported.", completed));
        }
Exemplo n.º 11
0
        /// <summary>
        /// Maps the notes.
        /// </summary>
        /// <param name="tableData">The table data.</param>
        public void MapNotes(IQueryable <Row> tableData)
        {
            var lookupContext   = new RockContext();
            var categoryService = new CategoryService(lookupContext);
            var personService   = new PersonService(lookupContext);

            var noteTypes        = new NoteTypeService(lookupContext).Queryable().AsNoTracking().ToList();
            var personalNoteType = noteTypes.FirstOrDefault(nt => nt.Guid == new Guid(Rock.SystemGuid.NoteType.PERSON_TIMELINE_NOTE));

            var importedUsers = new UserLoginService(lookupContext).Queryable().AsNoTracking()
                                .Where(u => u.ForeignId != null)
                                .ToDictionary(t => t.ForeignId, t => t.PersonId);

            var noteList = new List <Note>();

            int completed  = 0;
            int totalRows  = tableData.Count();
            int percentage = (totalRows - 1) / 100 + 1;

            ReportProgress(0, string.Format("Verifying note import ({0:N0} found).", totalRows));
            foreach (var row in tableData.Where(r => r != null))
            {
                string text           = row["Note_Text"] as string;
                int?   individualId   = row["Individual_ID"] as int?;
                int?   householdId    = row["Household_ID"] as int?;
                var    noteTypeActive = row["NoteTypeActive"] as Boolean?;

                bool noteArchived = false;
                if (row.Columns.FirstOrDefault(v => v.Name.Equals("IsInactive")) != null)
                {
                    /* =====================================================================
                     *  the NoteArchived column *should* work, but OrcaMDF won't read it...
                     *  instead check for a manually added column: IsInactive int null
                     *       var noteActive = row["NoteArchived"] as Boolean?;
                     *       if ( noteActive == null ) throw new NullReferenceException();
                     * /* ===================================================================== */
                    var rowInactiveValue = row["IsInactive"] as int?;
                    noteArchived = rowInactiveValue.Equals(1);
                }

                var personKeys = GetPersonKeys(individualId, householdId);
                if (personKeys != null && !string.IsNullOrWhiteSpace(text) && noteTypeActive == true && !noteArchived)
                {
                    DateTime?dateCreated = row["NoteCreated"] as DateTime?;
                    string   noteType    = row["Note_Type_Name"] as string;

                    var note = new Note();
                    note.CreatedDateTime = dateCreated;
                    note.EntityId        = personKeys.PersonId;

                    // These replace methods don't like being chained together
                    text = Regex.Replace(text, @"\t|\&nbsp;", " ");
                    text = text.Replace("&#45;", "-");
                    text = text.Replace("&lt;", "<");
                    text = text.Replace("&gt;", ">");
                    text = text.Replace("&amp;", "&");
                    text = text.Replace("&quot;", @"""");
                    text = text.Replace("&#x0D", string.Empty);

                    note.Text = text.Trim();

                    int?userId = row["NoteCreatedByUserID"] as int?;
                    if (userId != null && importedUsers.ContainsKey(userId))
                    {
                        var userKeys = ImportedPeople.FirstOrDefault(p => p.PersonId == (int)importedUsers[userId]);
                        if (userKeys != null)
                        {
                            note.CreatedByPersonAliasId = userKeys.PersonAliasId;
                        }
                    }

                    int?matchingNoteTypeId = null;
                    if (!noteType.StartsWith("General", StringComparison.InvariantCultureIgnoreCase))
                    {
                        matchingNoteTypeId = noteTypes.Where(nt => nt.Name == noteType).Select(i => (int?)i.Id).FirstOrDefault();
                    }
                    else
                    {
                        matchingNoteTypeId = personalNoteType.Id;
                    }

                    if (matchingNoteTypeId != null)
                    {
                        note.NoteTypeId = (int)matchingNoteTypeId;
                    }
                    else
                    {
                        // create the note type
                        var newNoteType = new NoteType();
                        newNoteType.EntityTypeId = personalNoteType.EntityTypeId;
                        newNoteType.EntityTypeQualifierColumn = string.Empty;
                        newNoteType.EntityTypeQualifierValue  = string.Empty;
                        newNoteType.UserSelectable            = true;
                        newNoteType.IsSystem = false;
                        newNoteType.Name     = noteType;
                        newNoteType.Order    = 0;

                        lookupContext.NoteTypes.Add(newNoteType);
                        lookupContext.SaveChanges(DisableAuditing);

                        noteTypes.Add(newNoteType);
                        note.NoteTypeId = newNoteType.Id;
                    }

                    noteList.Add(note);
                    completed++;

                    if (completed % percentage < 1)
                    {
                        int percentComplete = completed / percentage;
                        ReportProgress(percentComplete, string.Format("{0:N0} notes imported ({1}% complete).", completed, percentComplete));
                    }
                    else if (completed % ReportingNumber < 1)
                    {
                        SaveNotes(noteList);
                        ReportPartialProgress();
                        noteList.Clear();
                    }
                }
            }

            if (noteList.Any())
            {
                SaveNotes(noteList);
            }

            ReportProgress(100, string.Format("Finished note import: {0:N0} notes imported.", completed));
        }
Exemplo n.º 12
0
        /// <summary>
        /// Loads the family data.
        /// </summary>
        /// <param name="csvData">The CSV data.</param>
        private int LoadFamily(CsvDataModel csvData)
        {
            // Required variables
            var lookupContext     = new RockContext();
            var locationService   = new LocationService(lookupContext);
            int familyGroupTypeId = GroupTypeCache.GetFamilyGroupType().Id;

            int numImportedFamilies = ImportedPeople.Select(p => p.ForeignId).Distinct().Count();

            int homeLocationTypeId = DefinedValueCache.Read(new Guid(Rock.SystemGuid.DefinedValue.GROUP_LOCATION_TYPE_HOME)).Id;
            int workLocationTypeId = DefinedValueCache.Read(new Guid("E071472A-F805-4FC4-917A-D5E3C095C35C")).Id;

            var currentFamilyGroup = new Group();
            var newFamilyList      = new List <Group>();
            var newGroupLocations  = new Dictionary <GroupLocation, string>();

            string currentFamilyId = string.Empty;
            int    completed       = 0;

            ReportProgress(0, string.Format("Starting family import ({0:N0} already exist).", numImportedFamilies));

            string[] row;
            // Uses a look-ahead enumerator: this call will move to the next record immediately
            while ((row = csvData.Database.FirstOrDefault()) != null)
            {
                string rowFamilyId   = row[FamilyId];
                string rowFamilyName = row[FamilyName];

                if (!string.IsNullOrWhiteSpace(rowFamilyId) && rowFamilyId != currentFamilyGroup.ForeignId)
                {
                    currentFamilyGroup = ImportedPeople.FirstOrDefault(p => p.ForeignId == rowFamilyId);
                    if (currentFamilyGroup == null)
                    {
                        currentFamilyGroup           = new Group();
                        currentFamilyGroup.ForeignId = rowFamilyId;
                        currentFamilyGroup.Name      = row[FamilyName];
                        currentFamilyGroup.CreatedByPersonAliasId = ImportPersonAlias.Id;
                        currentFamilyGroup.GroupTypeId            = familyGroupTypeId;
                        newFamilyList.Add(currentFamilyGroup);
                    }

                    // Set the family campus
                    string campusName = row[Campus];
                    if (!string.IsNullOrWhiteSpace(campusName))
                    {
                        var familyCampus = CampusList.Where(c => c.Name.StartsWith(campusName) || c.ShortCode.StartsWith(campusName)).FirstOrDefault();
                        if (familyCampus == null)
                        {
                            familyCampus          = new Campus();
                            familyCampus.IsSystem = false;
                            familyCampus.Name     = campusName;
                            lookupContext.Campuses.Add(familyCampus);
                            lookupContext.SaveChanges(true);
                        }

                        // This won't assign a campus if the family already exists because the context doesn't get saved
                        currentFamilyGroup.CampusId = familyCampus.Id;
                    }

                    // Add the family addresses since they exist in this file
                    string famAddress  = row[Address];
                    string famAddress2 = row[Address2];
                    string famCity     = row[City];
                    string famState    = row[State];
                    string famZip      = row[Zip];
                    string famCountry  = row[Country];

                    // Use the core Rock location service to add or lookup an address
                    Location primaryAddress = locationService.Get(famAddress, famAddress2, famCity, famState, famZip, famCountry);
                    if (primaryAddress != null)
                    {
                        primaryAddress.Name = currentFamilyGroup.Name + " Home";

                        var primaryLocation = new GroupLocation();
                        primaryLocation.LocationId               = primaryAddress.Id;
                        primaryLocation.IsMailingLocation        = true;
                        primaryLocation.IsMappedLocation         = true;
                        primaryLocation.GroupLocationTypeValueId = homeLocationTypeId;
                        newGroupLocations.Add(primaryLocation, rowFamilyId);
                    }

                    string famSecondAddress  = row[SecondaryAddress];
                    string famSecondAddress2 = row[SecondaryAddress2];
                    string famSecondCity     = row[SecondaryCity];
                    string famSecondState    = row[SecondaryState];
                    string famSecondZip      = row[SecondaryZip];
                    string famSecondCountry  = row[SecondaryCountry];

                    Location secondaryAddress = locationService.Get(famSecondAddress, famSecondAddress2, famSecondCity, famSecondState, famSecondZip, famSecondCountry);
                    if (secondaryAddress != null)
                    {
                        secondaryAddress.Name = currentFamilyGroup.Name + " Work";

                        var secondaryLocation = new GroupLocation();
                        secondaryLocation.LocationId               = primaryAddress.Id;
                        secondaryLocation.IsMailingLocation        = true;
                        secondaryLocation.IsMappedLocation         = true;
                        secondaryLocation.GroupLocationTypeValueId = workLocationTypeId;
                        newGroupLocations.Add(secondaryLocation, rowFamilyId);
                    }

                    completed++;
                    if (completed % (ReportingNumber * 10) < 1)
                    {
                        ReportProgress(0, string.Format("{0:N0} families imported.", completed));
                    }
                    else if (completed % ReportingNumber < 1)
                    {
                        SaveFamilies(newFamilyList, newGroupLocations);
                        ReportPartialProgress();

                        // Reset lookup context
                        lookupContext   = new RockContext();
                        locationService = new LocationService(lookupContext);
                        newFamilyList.Clear();
                        newGroupLocations.Clear();
                    }
                }
            }

            // Check to see if any rows didn't get saved to the database
            if (newGroupLocations.Any())
            {
                SaveFamilies(newFamilyList, newGroupLocations);
            }

            ReportProgress(0, string.Format("Finished family import: {0:N0} families imported.", completed));
            return(completed);
        }