internal NameChange(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "OLD NAME":
OldName = dataItem.Value;
break;
case "NEW NAME":
NewName = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal LinkedAccount(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "SERVICE":
Service = dataItem.Value;
break;
case "EMAIL":
Email = dataItem.Value;
break;
case "IDENTIFIER":
ServiceId = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
public Location(CommandLineLogger logger, string dataSectionName, ParseDataItem dataItems)
{
if (dataItems != null && dataItems.HasChildren)
{
foreach (ParseDataItem dataItem in dataItems.Children)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "NAME":
Name = dataItem.Value;
break;
case "ADDRESS":
Address = dataItem.Value;
break;
case "EXTERNAL ID":
ExternalId = dataItem.Value;
break;
default:
logger.LogWarning("Unknown Section - \"Photos - Location:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
internal CallRecord(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.ToUpper().Trim())
{
case "TYPE":
CallType = dataItem.Value;
break;
case "MISSED":
Missed = dataItem.Value;
break;
case "DURATION":
Duration = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Call Record:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
private void UnzipFiles(CommandLineLogger logger, string holderFolder, string defaultDirectory, string parentZipFile, string toExtract, ref ParserVersionEnum version)
{
ZipUtility zp = new ZipUtility(logger)
{
CaseName = holderFolder,
ParentZipFilePath = parentZipFile
};
IEnumerable <ExtractFileInfo> htmlFiles = zp.ExtractZip(defaultDirectory, toExtract, ".html");
if (htmlFiles.Any())
{
foreach (ExtractFileInfo file in htmlFiles)
{
File.Copy(file.File_Path, Path.GetDirectoryName(file.File_Path) + "\\save-" + file.File_Name);
_filesToPreserve.Add(file);
}
_htmlToParse.AddRange(htmlFiles.Where(x => !x.File_Name.ToUpper().Trim().StartsWith("INDEX.") && !x.File_Name.ToUpper().Trim().StartsWith("PRESERVATION")));
IEnumerable <ExtractFileInfo> generatedHtmlFiles = zp.GenerateParsedHtml(htmlFiles);
if (generatedHtmlFiles.Any())
{
version = ParserVersionEnum.Two;
foreach (ExtractFileInfo generatedHtmlFile in generatedHtmlFiles)
{
if (!_htmlToParse.Any(x => x.File_Path.Equals(generatedHtmlFile.File_Path)) &&
!_htmlToParse.Any(x => x.File_Path.Equals(generatedHtmlFile.File_Path + @"\" + Path.GetFileNameWithoutExtension(generatedHtmlFile.File_Path))))
{
_htmlToParse.Add(generatedHtmlFile);
}
}
_tempFilesToRemove.AddRange(generatedHtmlFiles);
}
else
{
_htmlToParse.AddRange(htmlFiles.Where(x => x.File_Name.ToUpper().Trim().StartsWith("INDEX.")));
}
SaveFiles(htmlFiles);
}
IEnumerable <ExtractFileInfo> zipFiles = zp.ExtractZip(defaultDirectory, toExtract, ".zip");
if (zipFiles.Count() > 0)
{
SaveFiles(zipFiles);
_tempFilesToRemove.AddRange(zipFiles);
}
IEnumerable <ExtractFileInfo> otherFiles = zp.ExtractZip(defaultDirectory, toExtract, "");
if (otherFiles.Count() > 0)
{
SaveFiles(otherFiles);
}
foreach (ExtractFileInfo z in zipFiles)
{
UnzipFiles(logger, holderFolder, defaultDirectory, toExtract, z.File_Path, ref version);
}
}
public static Zap Create()
{
var settings = new Settings();
var logger = new CommandLineLogger();
var httpClientHelper = new HttpClientHelper(settings);
var scanner = new Scanner(settings, logger, httpClientHelper);
var reporting = new Reporting(settings, logger, httpClientHelper);
var spider = new Spider(settings, logger, httpClientHelper);
return(new Zap(settings, logger, scanner, reporting, spider));
}
public static Zap Create()
{
var settings = new Settings();
var logger = new CommandLineLogger();
var httpClientHelper = new HttpClientHelper(settings);
var scanner = new Scanner(settings, logger, httpClientHelper);
var reporting = new Reporting(settings,logger,httpClientHelper);
var spider = new Spider(settings, logger, httpClientHelper);
return new Zap( settings, logger, scanner, reporting, spider);
}
private static int Main(string[] args)
{
try {
if (args.Length == 0)
{
Install();
return(0);
}
else
{
CommandLineLogger logger = new CommandLineLogger();
List <string> compileGeneratedFiles;
List <string> embeddedResourceGeneratedFiles;
List <string> filesToDelete;
Process(
logger,
args,
/*templateFiles*/ null,
/*templateIncludeFiles*/ null,
/*templateLibraryFiles*/ null,
/*dependencyPaths*/ null,
/*absoluteOutputDir*/ null,
out compileGeneratedFiles,
out embeddedResourceGeneratedFiles,
out filesToDelete,
/*createNewAppDomain*/ true,
/*debugMode*/ false,
/*conditionalCompilationSymbols*/ null);
if (filesToDelete != null)
{
foreach (string file in filesToDelete)
{
File.Delete(file);
}
}
return(logger.Success ? 0 : 1);
}
} catch (Exception e) {
Console.Error.WriteLine(e);
return(-1);
}
}
public SectionParser(CommandLineLogger logger, ExtractFileInfo fileInfo, bool isPreservation, string preservationPrefix, ParserVersionEnum version)
{
ContainsLocationData = false;
Logger = logger;
FileInfo = fileInfo;
SourceFile = FileInfo.OriginalFile_Path;
SectionName = Path.GetFileNameWithoutExtension(FileInfo.File_Name);
List <string> sectionStrings = new List <string>();
if (SectionName.Contains("_"))
{
sectionStrings = SectionName.Split(new string[] { "_" }, StringSplitOptions.RemoveEmptyEntries).ToList();
}
else if (SectionName.Contains("_"))
{
sectionStrings = SectionName.Split(new string[] { "_" }, StringSplitOptions.RemoveEmptyEntries).ToList();
}
if (!sectionStrings.Any())
{
sectionStrings.Add(SectionName);
}
List <string> sectionStringsUpper = new List <string>();
string temp = string.Empty;
foreach (string s in sectionStrings)
{
sectionStringsUpper.Add(s.First().ToString().ToUpper() + s.Substring(1));
}
IsPreservation = isPreservation;
PreservationPrefix = preservationPrefix;
ParserVersion = version;
MainTableName = string.Join("", sectionStringsUpper);
if (isPreservation)
{
MainTableName = PreservationPrefix + "_" + MainTableName;
}
DisplaySectionName = string.Join(" ", sectionStringsUpper);
LoadHtml();
ProcessHTML();
}
internal Share(CommandLineLogger logger, string dataSectionName, ParseDataItem shareItem)
{
if (shareItem != null && shareItem.HasChildren)
{
foreach (ParseDataItem dataItem in shareItem.Children)
{
switch (dataItem.Header.ToUpper().Trim())
{
case "DATE CREATED":
DateCreated = dataItem.Value;
break;
case "LINK":
Link = dataItem.Value;
break;
case "SUMMARY":
Summary = dataItem.Value;
break;
case "TEXT":
Text = dataItem.Value;
break;
case "TITLE":
Title = dataItem.Value;
break;
case "URL":
URL = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Share: " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
}
internal Comment(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
CommentId = dataItem.Value;
break;
case "DATE CREATED":
DateCreatedUTC = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "TEXT":
Text = dataItem.Value;
break;
case "MEDIA CONTENT ID":
ContentId = dataItem.Value;
break;
case "MEDIA OWNER":
ContentOwner = new InstagramObject(dataItem.Value);
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
public AboutMeParser(CommandLineLogger logger, ExtractFileInfo fileInfo, bool isPreservation, string preservationPrefix, ParserVersionEnum version)
: base(logger, fileInfo, isPreservation, preservationPrefix, version)
{
if (isPreservation)
{
List <PreservationQuery> pq = new List <PreservationQuery>();
pq.Add(new PreservationQuery()
{
PreservationTableName = MainTableName,
QueryText = string.Format(
@"SELECT
(CASE WHEN a.[AboutMe] IS NULL THEN 'true' ELSE 'false' END) as [MissingInCurrent],
p.[AboutMe] as [P_AboutMe],
a.[AboutMe],
(case when LTRIM(RTRIM(p.[AboutMe])) <> LTRIM(RTRIM(a.[AboutMe])) THEN 'true' ELSE 'false' END) AS [AboutMe_Changed],
a.[File]
FROM {0} p
LEFT JOIN {1} a ON a.[AboutMe] = p.[AboutMe]", MainTableName, MainTableName.Replace(preservationPrefix + "_", ""))
});
PreservationQueries = pq;
}
}
internal SubscriptionEvent(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.ToUpper().Trim())
{
case "TYPE":
Type = dataItem.Value;
break;
case "USERS":
List <InstagramObject> users = new List <InstagramObject>();
foreach (string user in dataItem.Values)
{
InstagramObject fo = new InstagramObject(user);
if (fo.HasData)
{
users.Add(fo);
}
}
if (users != null && users.Any(x => x.HasData))
{
Users = users;
}
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Subscription Event:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
}
internal Like(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
ID = dataItem.Value;
break;
case "TAKEN":
Taken = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "URL":
URL = dataItem.Value;
break;
case "SOURCE":
Source = dataItem.Value;
break;
case "FILTER":
Filter = dataItem.Value;
break;
case "UPLOAD IP":
UploadIP = dataItem.Value;
break;
case "IS PUBLISHED":
IsPublished = dataItem.Value;
break;
case "SHARED BY AUTHOR":
SharedByAuthor = dataItem.Value;
break;
case "CAROUSEL ID":
CarouselId = dataItem.Value;
break;
case "MEDIA":
//Skip this ase because it has no data, and shows when the like has no responsive records
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
public VideosParser(CommandLineLogger logger, ExtractFileInfo fileInfo, bool isPreservation, string preservationPrefix, ParserVersionEnum version)
: base(logger, fileInfo, isPreservation, preservationPrefix, version)
{
}
internal LiveVideo(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
Id = dataItem.Value;
break;
case "VIDEO":
LiveVideos = dataItem.Value;
break;
case "LIVEVIDEOS":
LiveVideos = dataItem.Value;
break;
case "LiveVideos":
LiveVideos = dataItem.Value;
break;
case "Live Videos":
LiveVideos = dataItem.Value;
break;
// adding for LIVE VIDEOS fix
case "LIVE VIDEOS":
LiveVideos = dataItem.Value;
break;
case "TAKEN":
Taken = dataItem.Value;
break;
case "EXPIRE AT":
ExpireAtUTC = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "SOURCE":
Source = dataItem.Value;
break;
case "FILTER":
Filter = dataItem.Value;
break;
case "IS PUBLISHED":
IsPublished = dataItem.Value;
break;
case "SHARED BY AUTHOR":
SharedByAuthor = dataItem.Value;
break;
case "UPLOAD IP":
UploadIp = dataItem.Value;
break;
case "COMMENTS":
if (dataItem.HasChildren)
{
string firstItem = string.Empty;
List <Comment> records = new List <Comment>();
List <ParseDataItem> components = null;
foreach (ParseDataItem childDataItem in dataItem.Children)
{
if (string.IsNullOrEmpty(firstItem))
{
firstItem = childDataItem.Header;
}
if (childDataItem.Header.Equals(firstItem) && components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
components = null;
}
if (components == null)
{
components = new List <ParseDataItem>();
}
components.Add(childDataItem);
}
if (components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
}
if (records != null && records.Any(x => x.HasData))
{
Comments = records;
}
}
break;
case "MEDIA":
//Skip this case - it appears that this is a placeholder for a video that they can't return any information for.
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
static void Main(string[] args)
{
CommandLineLogger cll = new CommandLineLogger();
cll.Initialize();
String xmlInputFolder = null;
String xmlInputFile = null;
String mdOutputFolder = null;
String pageTemplatePath = null;
String pageTemplate = null;
ProjectSpace ps = new ProjectSpace();
for (int i=0; i<args.Length; i++)
{
String arg = args[i];
String argCanon = arg.ToLowerCase();
if (argCanon.StartsWith("/"))
{
argCanon = "-" + argCanon.Substring(1);
}
switch (argCanon)
{
case "-?":
case "-h":
case "-help":
OutputUsage();
break;
case "-mdpt":
case "-mdpagetemplate":
if (i < args.Length - 1)
{
i++;
pageTemplatePath = args[i];
if (pageTemplatePath != null && File.Exists(pageTemplatePath))
{
pageTemplate = FileUtilities.GetTextFromFile(pageTemplatePath);
}
}
break;
case "-xifo":
case "-xmlinfolder":
if (i < args.Length - 1)
{
i++;
xmlInputFolder = args[i];
if (xmlInputFolder != null && !Directory.Exists(xmlInputFolder))
{
Console.WriteLine("Could not find the input folder {0}", xmlInputFolder);
return;
}
if (xmlInputFolder != null)
{
ps.ImportFromXmlDocFolder(xmlInputFolder);
}
}
break;
case "-xifi":
case "-xmlinfile":
if (i < args.Length - 1)
{
i++;
xmlInputFile = args[i];
}
if (xmlInputFile != null && !File.Exists(xmlInputFile))
{
Console.WriteLine("Could not find the input file {0}", xmlInputFile);
return;
}
if (xmlInputFile != null)
{
ps.ImportFromXmlDocFile(xmlInputFile);
}
break;
case "-mdofo":
case "-mdoutfolder":
if (i < args.Length - 1)
{
i++;
mdOutputFolder = args[i];
}
break;
}
}
if (String.IsNullOrEmpty(xmlInputFolder) && String.IsNullOrEmpty(xmlInputFile))
{
Console.WriteLine("No xml input folder was specified.\r\n");
OutputUsage();
return;
}
if (String.IsNullOrEmpty(mdOutputFolder))
{
Console.WriteLine("No Markdown output folder was specified.\r\n");
OutputUsage();
return;
}
if (!Directory.Exists(mdOutputFolder))
{
Console.WriteLine("Could not find the output folder {0}", mdOutputFolder);
return;
}
try
{
ps.ExportMarkdownFiles(mdOutputFolder, pageTemplate);
}
catch (Exception e)
{
Log.Error("An error ocurrred when running this tool: " + e.Message);
}
}
internal DirectStory(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "MEDIA ID":
MediaId = dataItem.Value;
break;
case "TIME":
Time = dataItem.Value;
break;
case "AUTHOR":
InstagramObject fo = new InstagramObject(dataItem.Value);
if (fo.HasData)
{
Author = fo;
}
break;
case "RECIPIENTS":
if (dataItem.HasValues)
{
List <InstagramObject> recipients = null;
foreach (string value in dataItem.Values)
{
{
InstagramObject newRecipient = new InstagramObject(value);
if (newRecipient.HasData)
{
if (recipients == null)
{
recipients = new List <InstagramObject>();
}
recipients.Add(newRecipient);
}
}
}
if (recipients != null && recipients.Any())
{
Recipients = recipients;
}
}
break;
case "LINKED MEDIA FILE:":
LinkedMediaFile = dataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + ": " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
public ParserManager(CommandLineLogger log)
{
_log = log;
}
private static int Main(string[] args)
{
try {
if (args.Length == 0) {
Install();
return 0;
} else {
CommandLineLogger logger = new CommandLineLogger();
List<string> compileGeneratedFiles;
List<string> embeddedResourceGeneratedFiles;
List<string> filesToDelete;
Process(
logger,
args,
/*templateFiles*/null,
/*templateIncludeFiles*/null,
/*templateLibraryFiles*/null,
/*dependencyPaths*/null,
/*absoluteOutputDir*/null,
out compileGeneratedFiles,
out embeddedResourceGeneratedFiles,
out filesToDelete,
/*createNewAppDomain*/true,
/*debugMode*/false,
/*conditionalCompilationSymbols*/null);
if (filesToDelete != null) {
foreach (string file in filesToDelete) {
File.Delete(file);
}
}
return logger.Success ? 0 : 1;
}
} catch (Exception e) {
Console.Error.WriteLine(e);
return -1;
}
}
private void ParseHTML(CommandLineLogger logger, ParserVersionEnum version, string caseName, ref bool hasPreservation)
{
Stopwatch stopWatch = new Stopwatch();
logger.LogInfo("Begin parsing files.");
stopWatch.Start();
foreach (ExtractFileInfo htmlx in _htmlToParse)
{
ParserManager parser = new ParserManager(logger)
{
CaseNumber = caseName,
HtmlToLoad = htmlx.File_Path,
HtmlToRecord = htmlx.IsTemporary ? htmlx.ParentFile_Path : htmlx.File_Path,
IsPreservation = false,
DatabasePreservationNoPrefix = string.Empty,
Version = version,
DefaultDirectory = _defaultDirectory
};
logger.LogInfo("Processing " + htmlx.File_Path);
try
{
if (htmlx.File_Path.ToLower().Contains("preservation"))
{
hasPreservation = true;
/*Preservation-1, Preservation-2, Preservation-3 */
parser.IsPreservation = true;
DirectoryInfo di = new DirectoryInfo(htmlx.File_Path);
string p = di.Parent.Name; //goes up to parent directory, preservation
if (!p.ToLower().Contains("preservation"))
{
p = di.Parent.Parent.Name; //goes up to parent directory, preservation\folderX\index.html
}
parser.DatabasePreservationNoPrefix = p.Replace("-", "_"); //sqllite doesn't like sql queries ref tables with a '-', change to '_'
}
if (!htmlx.File_Name.ToUpper().Contains("PRESERVATION"))
{
switch (htmlx.File_Name.ToUpper().Trim())
{
case "ABOUT_ME.HTML":
parser.AboutMeParse(htmlx);
break;
case "ACCOUNT_STATUS_HISTORY.HTML":
parser.AccountStatusHistoryParse(htmlx);
break;
case "COMMENTS.HTML":
parser.CommentsParse(htmlx);
break;
case "DEVICES.HTML":
parser.DevicesParse(htmlx);
break;
case "DIRECT_SHARES.HTML":
parser.DirectSharesParse(htmlx);
break;
case "DIRECT_STORIES.HTML":
parser.DirectStoriesParse(htmlx);
break;
case "FOLLOWERS.HTML":
parser.FollowersParse(htmlx);
break;
case "FOLLOWING.HTML":
parser.FollowingParse(htmlx);
break;
case "GENDER.HTML":
parser.GenderParse(htmlx);
break;
case "INCOMING_FOLLOW_REQUESTS.HTML":
parser.IncomingFollowRequestsParse(htmlx);
break;
case "INDEX.HTML":
parser.IndexParse(htmlx);
break;
case "LIKES.HTML":
parser.LikesParse(htmlx);
break;
case "LINKED_ACCOUNTS.HTML":
parser.LinkedAccountsParse(htmlx);
break;
case "LIVE_VIDEOS.HTML":
parser.LiveVideosParse(htmlx);
break;
case "UNIFIED_MESSAGES.HTML":
parser.UnifiedMessagesParse(htmlx);
break;
case "NAME_CHANGES.HTML":
parser.NameChangesParse(htmlx);
break;
case "NCMEC_REPORTS.HTML":
parser.NcmecReportsParse(htmlx);
break;
case "PHOTOS.HTML":
parser.PhotosParse(htmlx);
break;
case "POPULAR_BLOCK.HTML":
parser.PopularBlockParse(htmlx);
break;
case "PRIVACY_SETTINGS.HTML":
parser.PrivacySettingsParse(htmlx);
break;
case "PROFILE_PICTURE.HTML":
parser.ProfilePictureParse(htmlx);
break;
case "VANITY_CHANGES.HTML":
parser.VanityChangesParse(htmlx);
break;
case "VIDEOS.HTML":
parser.VideosParse(htmlx);
break;
case "WEBSITE.HTML":
parser.WebsiteParse(htmlx);
break;
default:
logger.LogWarning("Unknown Section - \"Unknown section:" + htmlx.File_Name + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(htmlx.File_Name);
#endif
break;
}
}
if (parser.LocationData != null && parser.LocationData.Any())
{
_locationData.AddRange(parser.LocationData);
}
AddSectionToPreservationList(htmlx.File_Name, parser.PreservationQueries);
}
catch (SectionEmptyException ex)
{
logger.LogWarning("Parsing " + ex.Message + " section complete - section contains no data: Excluding from database.");
}
catch (MissingTestDataException ex)
{
logger.LogWarning("Parsing " + ex.Message + " section skipped - parser not implemented: No test data available.");
}
catch (NotImplementedException ex)
{
logger.LogError("Parsing " + ex.Message + " section failed: parser not implemented.", ex);
}
}
WriteLocationData(caseName);
stopWatch.Stop();
logger.LogInfo("Parsing files complete (Time: " + stopWatch.Elapsed.GetFormattedElapsedTime() + ")... ");
}
internal Attachment(CommandLineLogger logger, string dataSectionName, string value, IEnumerable <ParseDataItem> dataItems)
{
if (!string.IsNullOrEmpty(value))
{
InstagramObject fo = new InstagramObject(value);
if (fo.HasData)
{
Name = fo.Name;
Id = fo.Id;
}
}
foreach (ParseDataItem dataItem in dataItems)
{
string header = !string.IsNullOrEmpty(dataItem.Header) ? dataItem.Header : string.Empty;
switch (header.Trim().ToUpper())
{
case "TYPE":
MIMEType = dataItem.Value;
break;
case "SIZE":
Size = dataItem.Value;
break;
case "URL":
URL = dataItem.Value;
break;
case "":
if (dataItem.HasChildren)
{
foreach (ParseDataItem childDataItem in dataItem.Children)
{
switch (childDataItem.Header.Trim().ToUpper())
{
case "LINKED MEDIA FILE:":
LinkedMediaFile = childDataItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Comment: " + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message - Attachment:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
//_log.LogWarning("Unknown Section - \"Message:Attachment:" + headerNode.InnerText + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
// break;
}
}
}
public string ParseInstagramHTMLExtract(string extractZipFileNameAndPath)
{
Stopwatch totalProcessingTime = new Stopwatch();
totalProcessingTime.Start();
Stopwatch stopWatch = null;
bool hasPreservation = false;
ParserVersionEnum version = ParserVersionEnum.One;
string holderFolder = Path.GetFileNameWithoutExtension(extractZipFileNameAndPath);
CommandLineLogger logger = new CommandLineLogger(holderFolder, _defaultDirectory);
//pass case name to extract zip
ZipUtility zp = new ZipUtility(logger)
{
CaseName = holderFolder
};
logger.LogInfo("Begin extracting files.");
stopWatch = new Stopwatch();
stopWatch.Start();
UnzipFiles(logger, holderFolder, _defaultDirectory, null, extractZipFileNameAndPath, ref version);
stopWatch.Stop();
logger.LogInfo("Extract files complete (Time: " + stopWatch.Elapsed.GetFormattedElapsedTime() + ")... ");
string holderPath = AppDomain.CurrentDomain.BaseDirectory + _defaultDirectory;
string extractPath = holderPath + Path.GetFileNameWithoutExtension(holderFolder);
ParseHTML(logger, version, holderFolder, ref hasPreservation);
if (_allFiles.Count > 0)
{
DataAccess.AddSourceFiles(_defaultDirectory, _allFiles, holderFolder);
}
// TODO: Need to revisit this in the future to determine what analytics are requred or wanted by the user base.
////compare preservation to current tables if exists
//if (hasPreservation)
//{
// _log.LogInfo("Begin comparing preservation files.");
// DifPreservationTables(holderFolder);
//}
if (_tempFilesToRemove.Any())
{
logger.LogInfo("Begin removing temporary files.");
stopWatch = new Stopwatch();
stopWatch.Start();
List <string> tempPathsToRemove = new List <string>();
foreach (ExtractFileInfo tempFile in _tempFilesToRemove)
{
logger.LogInfo("Removing file: " + tempFile.File_Path.Replace(extractPath + "\\", ""));
File.Delete(tempFile.File_Path);
if (!tempPathsToRemove.Contains(Path.GetDirectoryName(tempFile.File_Path)))
{
tempPathsToRemove.Add(Path.GetDirectoryName(tempFile.File_Path));
}
}
if (tempPathsToRemove.Any())
{
foreach (string pathToRemove in tempPathsToRemove)
{
if (Directory.GetFiles(pathToRemove).Length == 0)
{
Directory.Delete(pathToRemove);
}
}
}
stopWatch.Stop();
logger.LogInfo("Removing temporary files complete (Time: " + stopWatch.Elapsed.GetFormattedElapsedTime() + ")... ");
}
if (_filesToPreserve.Any())
{
foreach (ExtractFileInfo file in _filesToPreserve)
{
string preservedFileName = Path.GetDirectoryName(file.File_Path) + "\\save-" + file.File_Name;
if (File.Exists(preservedFileName))
{
if (!File.Exists(file.File_Path))
{
File.Copy(preservedFileName, file.File_Path);
}
File.Delete(Path.GetDirectoryName(file.File_Path) + "\\save-" + file.File_Name);
}
}
}
totalProcessingTime.Stop();
logger.LogInfoAlert("Processing Complete. Total Processing time: " + totalProcessingTime.Elapsed.GetFormattedElapsedTime());
return("");
}
internal Message(CommandLineLogger logger, string dataSectionName, int msgNum, IEnumerable <ParseDataItem> marketplaceIdItems, IEnumerable <InstagramObject> participants, IEnumerable <ParseDataItem> messageItems)
{
List <InstagramObject> recipientList = new List <InstagramObject>();
GUID = Guid.NewGuid().ToString();
Sequence = msgNum;
if (marketplaceIdItems != null && marketplaceIdItems.Any())
{
foreach (ParseDataItem marketplaceIdItem in marketplaceIdItems)
{
switch (marketplaceIdItem.Header.ToUpper().Trim())
{
case "MARKETPLACE ID":
MarketplaceId = marketplaceIdItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message (Marketplace): " + marketplaceIdItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
//_log.LogWarning("Unknown Section - \"Message:" + headerNode.InnerText + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(marketplaceIdItem.Header);
#endif
// break;
}
}
}
foreach (ParseDataItem messageItem in messageItems)
{
switch (messageItem.Header.ToUpper().Trim())
{
case "RECIPIENTS":
if (messageItem.HasValues)
{
foreach (string recipient in messageItem.Values)
{
InstagramObject fo = new InstagramObject(recipient);
if (fo.HasData)
{
recipientList.Add(fo);
}
}
}
break;
case "AUTHOR":
Author = new InstagramObject(messageItem.Value);
break;
case "SENT":
Sent = messageItem.Value;
break;
case "DELETED":
IsDeleted = messageItem.Value;
break;
case "BODY":
Body = messageItem.Value;
break;
case "IP":
IPAddress = messageItem.Value;
break;
case "CALL RECORD":
CallRecord cr = new CallRecord(logger, dataSectionName, messageItem.Children);
if (cr.HasData)
{
CallRecords = new List <CallRecord>()
{
cr
}
}
;
break;
case "ATTACHMENTS":
List <Attachment> items = new List <Attachment>();
int startIndex = 0;
foreach (ParseDataItem.ValueCount vc in messageItem.ValueCounts)
{
string value = vc.Val;
List <ParseDataItem> components = null;
for (int i = startIndex; i < startIndex + vc.AssociatedChildCount; i++)
{
if (components == null)
{
components = new List <ParseDataItem>();
}
ParseDataItem childItem = messageItem.Children.ElementAt(i);
if (childItem.HasData)
{
components.Add(childItem);
}
}
Attachment att = new Attachment(logger, dataSectionName, value, components);
if (att.HasData)
{
items.Add(att);
}
startIndex += vc.AssociatedChildCount;
}
Attachments = items;
break;
case "SHARE":
Share share = new Share(logger, dataSectionName, messageItem);
if (share.HasData)
{
Shares = new List <Share>()
{
share
}
}
;
break;
case "MARKETPLACE ID":
MarketplaceId = messageItem.Value;
break;
case "SUBSCRIPTION EVENT":
if (messageItem.HasChildren)
{
SubscriptionEvent subscriptionEvent = new SubscriptionEvent(logger, dataSectionName, messageItem.Children);
if (subscriptionEvent.HasData)
{
SubscriptionEvents = new List <SubscriptionEvent>()
{
subscriptionEvent
}
}
;
}
break;
case "REMOVED BY SENDER":
RemoveBySender = messageItem.Value;
break;
default:
logger.LogWarning("Unexpected Html Element - \"" + dataSectionName + " - Message: " + messageItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
//_log.LogWarning("Unknown Section - \"Message:" + headerNode.InnerText + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(messageItem.Header);
#endif
// break;
}
}
if (participants != null && participants.Any(x => x.HasData))
{
recipientList.AddRange(participants.Where(x => x.HasData));
}
if (recipientList.Count() > 0)
{
Recipients = recipientList;
}
}
internal Thread(CommandLineLogger logger, string dataSectionName, string threadId, string threadDate, IEnumerable <ParseDataItem> marketplaceIdItems, IEnumerable <ParseDataItem> currentParticipants, IEnumerable <ParseDataItem> pastParticipants, IEnumerable <ParseDataItem> messages)
{
ID = !string.IsNullOrEmpty(threadId) ? threadId : Guid.NewGuid().ToString();
ThreadDate = threadDate;
List <InstagramObject> currentPart = new List <InstagramObject>();
List <InstagramObject> pastPart = new List <InstagramObject>();
List <InstagramObject> participants = new List <InstagramObject>();
if (currentParticipants != null && currentParticipants.Any())
{
foreach (ParseDataItem cpItem in currentParticipants)
{
if (cpItem.HasValues)
{
foreach (string val in cpItem.Values)
{
if (val.Contains("UTC"))
{
threadDate = val;
}
else
{
InstagramObject fo = new InstagramObject(val);
if (fo.HasData)
{
currentPart.Add(fo);
}
}
}
}
}
if (currentPart.Any())
{
participants.AddRange(currentPart);
}
}
if (pastParticipants != null && pastParticipants.Any())
{
foreach (ParseDataItem ppItem in pastParticipants)
{
if (ppItem.HasValues)
{
foreach (string val in ppItem.Values)
{
if (val.Contains("UTC"))
{
threadDate = val;
}
else
{
InstagramObject fo = new InstagramObject(val);
if (fo.HasData)
{
pastPart.Add(fo);
}
}
}
}
}
if (pastPart.Any())
{
participants.AddRange(currentPart);
}
}
List <Message> items = new List <Message>();
List <ParseDataItem> components = null;
int messageCount = 1;
string firstItem = string.Empty;
if (messages != null && messages.Any())
{
foreach (ParseDataItem item in messages)
{
if (string.IsNullOrEmpty(firstItem))
{
firstItem = item.Header;
}
if (item.Header.Equals(firstItem) && components != null && components.Any())
{
Message newItem = new Message(logger, dataSectionName, messageCount, marketplaceIdItems, participants, components);
if (newItem.HasData)
{
items.Add(newItem);
messageCount++;
}
components = null;
}
if (components == null)
{
components = new List <ParseDataItem>();
}
components.Add(item);
}
if (components != null && components.Any())
{
Message newItem = new Message(logger, dataSectionName, messageCount, marketplaceIdItems, participants, components);
if (newItem.HasData)
{
items.Add(newItem);
messageCount++;
}
}
if (items != null && items.Any())
{
Messages = items;
}
}
}
public AccountStatusHistoryParser(CommandLineLogger logger, ExtractFileInfo fileInfo, bool isPreservation, string preservationPrefix, ParserVersionEnum version)
: base(logger, fileInfo, isPreservation, preservationPrefix, version)
{
}
internal Video(CommandLineLogger logger, string dataSectionName, IEnumerable <ParseDataItem> dataItems)
{
if (dataItems != null && dataItems.Any())
{
foreach (ParseDataItem dataItem in dataItems)
{
switch (dataItem.Header.Trim().ToUpper())
{
case "ID":
Id = dataItem.Value;
break;
case "TAKEN":
Taken = dataItem.Value;
break;
case "EXPIRE AT":
ExpireAtUTC = dataItem.Value;
break;
case "STATUS":
Status = dataItem.Value;
break;
case "URL":
Url = dataItem.Value;
break;
case "SOURCE":
Source = dataItem.Value;
break;
case "FILTER":
Filter = dataItem.Value;
break;
case "IS PUBLISHED":
IsPublished = dataItem.Value;
break;
case "SHARED BY AUTHOR":
SharedByAuthor = dataItem.Value;
break;
case "UPLOAD IP":
UploadIp = dataItem.Value;
break;
case "CAROUSEL ID":
CarouselId = dataItem.Value;
break;
case "VIDEO":
if (dataItem.HasChildren && dataItem.Children.Count() == 1)
{
foreach (ParseDataItem childDataItem in dataItem.Children)
{
switch (childDataItem.Header.Trim().ToUpper())
{
case "LINKED MEDIA FILE:":
LinkedMediaFile = childDataItem.Value;
break;
default:
logger.LogWarning("Unknown Section - \"Photos - " + dataItem.Header + " - " + childDataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(childDataItem.Header);
#endif
break;
}
}
}
break;
case "COMMENTS":
List <Comment> records = new List <Comment>();
List <ParseDataItem> components = null;
if (dataItem.HasChildren && dataItem.Children.Count() == 1)
{
string firstItem = string.Empty;
foreach (ParseDataItem childDataItem in dataItem.Children)
{
if (string.IsNullOrEmpty(firstItem))
{
firstItem = childDataItem.Header;
}
if (childDataItem.Header.Equals(firstItem) && components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
components = null;
}
if (components == null)
{
components = new List <ParseDataItem>();
}
components.Add(childDataItem);
}
if (components != null && components.Any())
{
Comment newItem = new Comment(logger, dataSectionName, components);
if (newItem.HasData)
{
records.Add(newItem);
}
}
}
if (records != null && records.Any(x => x.HasData))
{
Comments = records;
}
break;
default:
logger.LogWarning("Unknown Section - \"Videos:" + dataItem.Header + "\". Please contact NDCAC with section name and test data to improve parsing functionality");
#if DEBUG
throw new ApplicationException(dataItem.Header);
#endif
break;
}
}
}
}
public NcmecReportsParser(CommandLineLogger logger, ExtractFileInfo fileInfo, bool isPreservation, string preservationPrefix, ParserVersionEnum version)
: base(logger, fileInfo, isPreservation, preservationPrefix, version)
{
throw new MissingTestDataException(DisplaySectionName);
}
public static void SetLogger(CommandLineLogger logger)
{
Logger = logger;
}
public ZipUtility(CommandLineLogger log)
{
_log = log;
}
