| public LogWarning ( string file, string errorCode, string message, int line, int column ) : void | ||
| file | string | |
| errorCode | string | |
| message | string | |
| line | int | |
| column | int | |
| return | void | |
internal NameChange(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "OLD NAME":
OldName = dataItem.Value;
break;
case "NEW NAME":
NewName = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal CallRecord(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.ToUpper().Trim())
{
case "TYPE":
CallType = dataItem.Value;
break;
case "MISSED":
Missed = dataItem.Value;
break;
case "DURATION":
Duration = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Call Record:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
public Location(CommandLineLogger logger, string dataSectionName, ParseDataItem dataItems)
{
if (dataItems != null && dataItems.HasChildren)
{
foreach (ParseDataItem dataItem in dataItems.Children)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "NAME":
Name = dataItem.Value;
break;
case "ADDRESS":
Address = dataItem.Value;
break;
case "EXTERNAL ID":
ExternalId = dataItem.Value;
break;
default:
logger.LogWarning("Unknown Section - \"Photos - Location:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal LinkedAccount(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "SERVICE":
Service = dataItem.Value;
break;
case "EMAIL":
Email = dataItem.Value;
break;
case "IDENTIFIER":
ServiceId = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal Comment(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
CommentId = dataItem.Value;
break;
case "DATE CREATED":
DateCreatedUTC = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "TEXT":
Text = dataItem.Value;
break;
case "MEDIA CONTENT ID":
ContentId = dataItem.Value;
break;
case "MEDIA OWNER":
ContentOwner = new InstagramObject(dataItem.Value);
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal Share(CommandLineLogger logger, string dataSectionName, ParseDataItem shareItem)
{
if (shareItem != null && shareItem.HasChildren)
{
foreach (ParseDataItem dataItem in shareItem.Children)
{
switch (dataItem.Header.ToUpper().Trim())
{
case "DATE CREATED":
DateCreated = dataItem.Value;
break;
case "LINK":
Link = dataItem.Value;
break;
case "SUMMARY":
Summary = dataItem.Value;
break;
case "TEXT":
Text = dataItem.Value;
break;
case "TITLE":
Title = dataItem.Value;
break;
case "URL":
URL = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Share: " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
}
internal SubscriptionEvent(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.ToUpper().Trim())
{
case "TYPE":
Type = dataItem.Value;
break;
case "USERS":
List <InstagramObject> users = new List <InstagramObject>();
foreach (string user in dataItem.Values)
{
InstagramObject fo = new InstagramObject(user);
if (fo.HasData)
{
users.Add(fo);
}
}
if (users != null && users.Any(x => x.HasData))
{
Users = users;
}
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Subscription Event:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
}
internal Video(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
Id = dataItem.Value;
break;
case "TAKEN":
Taken = dataItem.Value;
break;
case "EXPIRE AT":
ExpireAtUTC = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "URL":
Url = dataItem.Value;
break;
case "SOURCE":
Source = dataItem.Value;
break;
case "FILTER":
Filter = dataItem.Value;
break;
case "IS PUBLISHED":
IsPublished = dataItem.Value;
break;
case "SHARED BY AUTHOR":
SharedByAuthor = dataItem.Value;
break;
case "UPLOAD IP":
UploadIp = dataItem.Value;
break;
case "CAROUSEL ID":
CarouselId = dataItem.Value;
break;
case "VIDEO":
if (dataItem.HasChildren && dataItem.Children.Count() == 1)
{
foreach (ParseDataItem childDataItem in dataItem.Children)
{
switch (childDataItem.Header.Trim().ToUpper())
{
case "LINKED MEDIA FILE:":
LinkedMediaFile = childDataItem.Value;
break;
default:
logger.LogWarning("Unknown Section - \"Photos - " + dataItem.Header + " - " + childDataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(childDataItem.Header);
#endif
break;
}
}
}
break;
case "COMMENTS":
List <Comment> records = new List <Comment>();
List <ParseDataItem> components = null;
if (dataItem.HasChildren && dataItem.Children.Count() == 1)
{
string firstItem = string.Empty;
foreach (ParseDataItem childDataItem in dataItem.Children)
{
if (string.IsNullOrEmpty(firstItem))
{
firstItem = childDataItem.Header;
}
if (childDataItem.Header.Equals(firstItem) && components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
components = null;
}
if (components == null)
{
components = new List <ParseDataItem>();
}
components.Add(childDataItem);
}
if (components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
}
}
if (records != null && records.Any(x => x.HasData))
{
Comments = records;
}
break;
default:
logger.LogWarning("Unknown Section - \"Videos:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal Like(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
ID = dataItem.Value;
break;
case "TAKEN":
Taken = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "URL":
URL = dataItem.Value;
break;
case "SOURCE":
Source = dataItem.Value;
break;
case "FILTER":
Filter = dataItem.Value;
break;
case "UPLOAD IP":
UploadIP = dataItem.Value;
break;
case "IS PUBLISHED":
IsPublished = dataItem.Value;
break;
case "SHARED BY AUTHOR":
SharedByAuthor = dataItem.Value;
break;
case "CAROUSEL ID":
CarouselId = dataItem.Value;
break;
case "MEDIA":
//Skip this ase because it has no data, and shows when the like has no responsive records
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal Attachment(CommandLineLogger logger, string dataSectionName, string value, IEnumerable <ParseDataItem> dataItems)
{
if (!string.IsNullOrEmpty(value))
{
InstagramObject fo = new InstagramObject(value);
if (fo.HasData)
{
Name = fo.Name;
Id = fo.Id;
}
}
foreach (ParseDataItem dataItem in dataItems)
{
string header = !string.IsNullOrEmpty(dataItem.Header) ? dataItem.Header : string.Empty;
switch (header.Trim().ToUpper())
{
case "TYPE":
MIMEType = dataItem.Value;
break;
case "SIZE":
Size = dataItem.Value;
break;
case "URL":
URL = dataItem.Value;
break;
case "":
if (dataItem.HasChildren)
{
foreach (ParseDataItem childDataItem in dataItem.Children)
{
switch (childDataItem.Header.Trim().ToUpper())
{
case "LINKED MEDIA FILE:":
LinkedMediaFile = childDataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Comment: " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Attachment:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
//_log.LogWarning("Unknown Section - \"Message:Attachment:" + headerNode.InnerText + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
internal Message(CommandLineLogger logger, string dataSectionName, int msgNum, IEnumerable <ParseDataItem> marketplaceIdItems, IEnumerable <InstagramObject> participants, IEnumerable <ParseDataItem> messageItems)
{
List <InstagramObject> recipientList = new List <InstagramObject>();
GUID = Guid.NewGuid().ToString();
Sequence = msgNum;
if (marketplaceIdItems != null && marketplaceIdItems.Any())
{
foreach (ParseDataItem marketplaceIdItem in marketplaceIdItems)
{
switch (marketplaceIdItem.Header.ToUpper().Trim())
{
case "MARKETPLACE ID":
MarketplaceId = marketplaceIdItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message (Marketplace): " + marketplaceIdItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
//_log.LogWarning("Unknown Section - \"Message:" + headerNode.InnerText + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(marketplaceIdItem.Header);
#endif
// break;
}
}
}
foreach (ParseDataItem messageItem in messageItems)
{
switch (messageItem.Header.ToUpper().Trim())
{
case "RECIPIENTS":
if (messageItem.HasValues)
{
foreach (string recipient in messageItem.Values)
{
InstagramObject fo = new InstagramObject(recipient);
if (fo.HasData)
{
recipientList.Add(fo);
}
}
}
break;
case "AUTHOR":
Author = new InstagramObject(messageItem.Value);
break;
case "SENT":
Sent = messageItem.Value;
break;
case "DELETED":
IsDeleted = messageItem.Value;
break;
case "BODY":
Body = messageItem.Value;
break;
case "IP":
IPAddress = messageItem.Value;
break;
case "CALL RECORD":
CallRecord cr = new CallRecord(logger, dataSectionName, messageItem.Children);
if (cr.HasData)
{
CallRecords = new List <CallRecord>()
{
cr
}
}
;
break;
case "ATTACHMENTS":
List <Attachment> items = new List <Attachment>();
int startIndex = 0;
foreach (ParseDataItem.ValueCount vc in messageItem.ValueCounts)
{
string value = vc.Val;
List <ParseDataItem> components = null;
for (int i = startIndex; i < startIndex + vc.AssociatedChildCount; i++)
{
if (components == null)
{
components = new List <ParseDataItem>();
}
ParseDataItem childItem = messageItem.Children.ElementAt(i);
if (childItem.HasData)
{
components.Add(childItem);
}
}
Attachment att = new Attachment(logger, dataSectionName, value, components);
if (att.HasData)
{
items.Add(att);
}
startIndex += vc.AssociatedChildCount;
}
Attachments = items;
break;
case "SHARE":
Share share = new Share(logger, dataSectionName, messageItem);
if (share.HasData)
{
Shares = new List <Share>()
{
share
}
}
;
break;
case "MARKETPLACE ID":
MarketplaceId = messageItem.Value;
break;
case "SUBSCRIPTION EVENT":
if (messageItem.HasChildren)
{
SubscriptionEvent subscriptionEvent = new SubscriptionEvent(logger, dataSectionName, messageItem.Children);
if (subscriptionEvent.HasData)
{
SubscriptionEvents = new List <SubscriptionEvent>()
{
subscriptionEvent
}
}
;
}
break;
case "REMOVED BY SENDER":
RemoveBySender = messageItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message: " + messageItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
//_log.LogWarning("Unknown Section - \"Message:" + headerNode.InnerText + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(messageItem.Header);
#endif
// break;
}
}
if (participants != null && participants.Any(x => x.HasData))
{
recipientList.AddRange(participants.Where(x => x.HasData));
}
if (recipientList.Count() > 0)
{
Recipients = recipientList;
}
}
private void ParseHTML(CommandLineLogger logger, ParserVersionEnum version, string caseName, ref bool hasPreservation)
{
Stopwatch stopWatch = new Stopwatch();
logger.LogInfo("Begin parsing files.");
stopWatch.Start();
foreach (ExtractFileInfo htmlx in _htmlToParse)
{
ParserManager parser = new ParserManager(logger)
{
CaseNumber = caseName,
HtmlToLoad = htmlx.File_Path,
HtmlToRecord = htmlx.IsTemporary ? htmlx.ParentFile_Path : htmlx.File_Path,
IsPreservation = false,
DatabasePreservationNoPrefix = string.Empty,
Version = version,
DefaultDirectory = _defaultDirectory
};
logger.LogInfo("Processing " + htmlx.File_Path);
try
{
if (htmlx.File_Path.ToLower().Contains("preservation"))
{
hasPreservation = true;
/*Preservation-1, Preservation-2, Preservation-3 */
parser.IsPreservation = true;
DirectoryInfo di = new DirectoryInfo(htmlx.File_Path);
string p = di.Parent.Name; //goes up to parent directory, preservation
if (!p.ToLower().Contains("preservation"))
{
p = di.Parent.Parent.Name; //goes up to parent directory, preservation\folderX\index.html
}
parser.DatabasePreservationNoPrefix = p.Replace("-", "_"); //sqllite doesn't like sql queries ref tables with a '-', change to '_'
}
if (!htmlx.File_Name.ToUpper().Contains("PRESERVATION"))
{
switch (htmlx.File_Name.ToUpper().Trim())
{
case "ABOUT_ME.HTML":
parser.AboutMeParse(htmlx);
break;
case "ACCOUNT_STATUS_HISTORY.HTML":
parser.AccountStatusHistoryParse(htmlx);
break;
case "COMMENTS.HTML":
parser.CommentsParse(htmlx);
break;
case "DEVICES.HTML":
parser.DevicesParse(htmlx);
break;
case "DIRECT_SHARES.HTML":
parser.DirectSharesParse(htmlx);
break;
case "DIRECT_STORIES.HTML":
parser.DirectStoriesParse(htmlx);
break;
case "FOLLOWERS.HTML":
parser.FollowersParse(htmlx);
break;
case "FOLLOWING.HTML":
parser.FollowingParse(htmlx);
break;
case "GENDER.HTML":
parser.GenderParse(htmlx);
break;
case "INCOMING_FOLLOW_REQUESTS.HTML":
parser.IncomingFollowRequestsParse(htmlx);
break;
case "INDEX.HTML":
parser.IndexParse(htmlx);
break;
case "LIKES.HTML":
parser.LikesParse(htmlx);
break;
case "LINKED_ACCOUNTS.HTML":
parser.LinkedAccountsParse(htmlx);
break;
case "LIVE_VIDEOS.HTML":
parser.LiveVideosParse(htmlx);
break;
case "UNIFIED_MESSAGES.HTML":
parser.UnifiedMessagesParse(htmlx);
break;
case "NAME_CHANGES.HTML":
parser.NameChangesParse(htmlx);
break;
case "NCMEC_REPORTS.HTML":
parser.NcmecReportsParse(htmlx);
break;
case "PHOTOS.HTML":
parser.PhotosParse(htmlx);
break;
case "POPULAR_BLOCK.HTML":
parser.PopularBlockParse(htmlx);
break;
case "PRIVACY_SETTINGS.HTML":
parser.PrivacySettingsParse(htmlx);
break;
case "PROFILE_PICTURE.HTML":
parser.ProfilePictureParse(htmlx);
break;
case "VANITY_CHANGES.HTML":
parser.VanityChangesParse(htmlx);
break;
case "VIDEOS.HTML":
parser.VideosParse(htmlx);
break;
case "WEBSITE.HTML":
parser.WebsiteParse(htmlx);
break;
default:
logger.LogWarning("Unknown Section - \"Unknown section:" + htmlx.File_Name + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(htmlx.File_Name);
#endif
break;
}
}
if (parser.LocationData != null && parser.LocationData.Any())
{
_locationData.AddRange(parser.LocationData);
}
AddSectionToPreservationList(htmlx.File_Name, parser.PreservationQueries);
}
catch (SectionEmptyException ex)
{
logger.LogWarning("Parsing " + ex.Message + " section complete - section contains no data: Excluding from database.");
}
catch (MissingTestDataException ex)
{
logger.LogWarning("Parsing " + ex.Message + " section skipped - parser not implemented: No test data available.");
}
catch (NotImplementedException ex)
{
logger.LogError("Parsing " + ex.Message + " section failed: parser not implemented.", ex);
}
}
WriteLocationData(caseName);
stopWatch.Stop();
logger.LogInfo("Parsing files complete (Time: " + stopWatch.Elapsed.GetFormattedElapsedTime() + ")... ");
}
internal DirectStory(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "MEDIA ID":
MediaId = dataItem.Value;
break;
case "TIME":
Time = dataItem.Value;
break;
case "AUTHOR":
InstagramObject fo = new InstagramObject(dataItem.Value);
if (fo.HasData)
{
Author = fo;
}
break;
case "RECIPIENTS":
if (dataItem.HasValues)
{
List <InstagramObject> recipients = null;
foreach (string value in dataItem.Values)
{
{
InstagramObject newRecipient = new InstagramObject(value);
if (newRecipient.HasData)
{
if (recipients == null)
{
recipients = new List <InstagramObject>();
}
recipients.Add(newRecipient);
}
}
}
if (recipients != null && recipients.Any())
{
Recipients = recipients;
}
}
break;
case "LINKED MEDIA FILE:":
LinkedMediaFile = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal LiveVideo(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
Id = dataItem.Value;
break;
case "VIDEO":
LiveVideos = dataItem.Value;
break;
case "LIVEVIDEOS":
LiveVideos = dataItem.Value;
break;
case "LiveVideos":
LiveVideos = dataItem.Value;
break;
case "Live Videos":
LiveVideos = dataItem.Value;
break;
// adding for LIVE VIDEOS fix
case "LIVE VIDEOS":
LiveVideos = dataItem.Value;
break;
case "TAKEN":
Taken = dataItem.Value;
break;
case "EXPIRE AT":
ExpireAtUTC = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "SOURCE":
Source = dataItem.Value;
break;
case "FILTER":
Filter = dataItem.Value;
break;
case "IS PUBLISHED":
IsPublished = dataItem.Value;
break;
case "SHARED BY AUTHOR":
SharedByAuthor = dataItem.Value;
break;
case "UPLOAD IP":
UploadIp = dataItem.Value;
break;
case "COMMENTS":
if (dataItem.HasChildren)
{
string firstItem = string.Empty;
List <Comment> records = new List <Comment>();
List <ParseDataItem> components = null;
foreach (ParseDataItem childDataItem in dataItem.Children)
{
if (string.IsNullOrEmpty(firstItem))
{
firstItem = childDataItem.Header;
}
if (childDataItem.Header.Equals(firstItem) && components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
components = null;
}
if (components == null)
{
components = new List <ParseDataItem>();
}
components.Add(childDataItem);
}
if (components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
}
if (records != null && records.Any(x => x.HasData))
{
Comments = records;
}
}
break;
case "MEDIA":
//Skip this case - it appears that this is a placeholder for a video that they can't return any information for.
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}