List <OdFileInfo> SelectFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
mediaQueue.ReportProgress("Selecting files");
// Determine the "after" threshold from SelectAfter and SelectIncremental
var after = sourceConfig.GetBookmarkOrAfter(m_bookmarkPath);
if (after.HasValue)
{
mediaQueue.ReportProgress($" Filter: Created after {after.Value}");
}
// Retrieve file info and enqueue
var queue = new List <OdFileInfo>();
int count = 0;
// This is a Kludge. Need a way to either detect the correct folder or to
// configure it when setting up a new named source. And, with the named
// source, we need to clean up the circumstance when credentials expire.
string nextUrl = m_sourceName.Equals("BrandtOneDrive")
? c_oneDriveSamsungCameraUrl : c_oneDriveCameraRollUrl;
do
{
var fileList = FetchJson(nextUrl);
var root = fileList.CreateNavigator();
nextUrl = root.XPVal("/root/a:item[@item='@odata.nextLink']");
foreach (XPathNavigator node in fileList.CreateNavigator().Select("/root/value/item"))
{
++count;
var odfi = new OdFileInfo(node);
if (!after.HasValue ||
(odfi.BookmarkDate.HasValue && odfi.BookmarkDate > after.Value))
{
queue.Add(odfi);
}
}
mediaQueue.ReportStatus($" Selected {queue.Count} Skipped {count - queue.Count}");
}while (!string.IsNullOrEmpty(nextUrl));
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($"Selected {queue.Count} Skipped {count - queue.Count}");
if (queue.Count > c_maxBatch)
{
// Sort so that we'll keep the oldest ones.
queue.Sort((a, b) => DateCompare(a.BookmarkDate, b.BookmarkDate));
queue.RemoveRange(c_maxBatch, queue.Count - c_maxBatch);
mediaQueue.RequestAnotherBatch = true;
mediaQueue.ReportProgress($"Batch limited to {queue.Count}");
}
return(queue);
}
public void RetrieveMediaFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
// DCIM requires destination directory
if (string.IsNullOrEmpty(sourceConfig.DestinationDirectory))
{
throw new Exception("DCIM source requires -d destination directory.");
}
var queue = SelectDcimFiles(sourceConfig, mediaQueue);
FileMover.CopyOrMoveFiles(queue, sourceConfig, mediaQueue);
CleanupDcfDirectories(mediaQueue);
}
public void RetrieveMediaFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
// if a destination directory not specified, error
if (string.IsNullOrEmpty(sourceConfig.DestinationDirectory))
{
throw new InvalidOperationException("OneDrive Source requires destination directory -d");
}
mediaQueue.ReportProgress($"Connecting to OneDrive: {m_sourceName}");
m_accessToken = NamedSource.GetOnedriveAccessToken(m_refreshToken);
var queue = SelectFiles(sourceConfig, mediaQueue);
DownloadMediaFiles(queue, sourceConfig, mediaQueue);
}
public void RetrieveMediaFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
var queue = SelectFiles(sourceConfig, mediaQueue);
// if a destination directory was specified, copy or move the files
if (sourceConfig.DestinationDirectory != null)
{
FileMover.CopyOrMoveFiles(queue, sourceConfig, mediaQueue);
}
// Else, simply put them in the mediaQueue
else
{
FileMover.EnqueueFiles(queue, mediaQueue);
}
// Save the bookmark
if (m_newestSelection > DateTime.MinValue)
{
// Only sets a bookmark if incremental is on.
sourceConfig.SetBookmark(m_path, m_newestSelection);
}
}
private List <ProcessFileInfo> SelectDcimFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
if (sourceConfig.SelectIncremental)
{
throw new Exception("DCIM source is not compatible with -selectIncremental");
}
if (sourceConfig.SelectAfter.HasValue)
{
throw new Exception("DCIM source does not support -selectAfter");
}
var sourceFolders = new List <string>();
// Process each removable drive
foreach (DriveInfo drv in DriveInfo.GetDrives())
{
if (drv.IsReady && drv.DriveType == DriveType.Removable)
{
try
{
// File system structure is according to JEITA "Design rule for Camera File System (DCF) which is JEITA specification CP-3461
// See if the DCIM folder exists
DirectoryInfo dcim = new DirectoryInfo(Path.Combine(drv.RootDirectory.FullName, c_dcimDirectory));
if (dcim.Exists)
{
// Folders containing images must be named with three digits followed
// by five alphanumeric characters. First digit cannot be zero.
foreach (DirectoryInfo di in dcim.EnumerateDirectories())
{
if (di.Name.Length == 8)
{
int dirnum;
if (int.TryParse(di.Name.Substring(0, 3), out dirnum) && dirnum >= 100 && dirnum <= 999)
{
sourceFolders.Add(di.FullName);
}
}
}
}
}
catch (Exception)
{
// Suppress the error and move to the next drive.
}
} // If drive is ready and removable
} // for each drive
var queue = new List <ProcessFileInfo>();
// Add files from each source folder
foreach (var path in sourceFolders)
{
int bookmark = queue.Count;
mediaQueue.ReportProgress($"Selecting from: {path}");
DirectoryInfo di = new DirectoryInfo(path);
foreach (var fi in di.EnumerateFiles())
{
if ((queue.Count % 100) == 0)
{
mediaQueue.ReportStatus($"Selected: {queue.Count}");
}
if (MediaFile.IsSupportedMediaType(fi.Extension))
{
queue.Add(new ProcessFileInfo(fi));
}
}
if (queue.Count > bookmark)
{
m_dcfDirectories.Add(path);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($" Selected: {queue.Count - bookmark}");
}
return(queue);
}
public static void CopyOrMoveFiles(List <ProcessFileInfo> queue, SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
string verb = sourceConfig.MoveFiles ? "Moving" : "Copying";
mediaQueue.ReportProgress($"{verb} media files to working folder: {sourceConfig.DestinationDirectory}.");
// Sum up the size of the files to be copied
long selectedFilesSize = 0;
foreach (var pfi in queue)
{
selectedFilesSize += pfi.Size;
}
uint startTicks = (uint)Environment.TickCount;
long bytesCopied = 0;
int n = 0;
foreach (var pfi in queue)
{
if (bytesCopied == 0)
{
mediaQueue.ReportStatus($"{verb} file {n + 1} of {queue.Count}");
}
else
{
uint ticksElapsed;
unchecked
{
ticksElapsed = (uint)Environment.TickCount - startTicks;
}
double bps = ((double)bytesCopied * 1000.0) / (double)ticksElapsed;
double remaining = (selectedFilesSize - bytesCopied) / bps;
TimeSpan remain = new TimeSpan(((long)((selectedFilesSize - bytesCopied) / bps)) * 10000000L);
mediaQueue.ReportStatus($"{verb} file {n + 1} of {queue.Count}. Time remaining: {remain.FmtCustom()} MBps: {(bps / (1024 * 1024)):#,###.###}");
}
string dstFilepath = Path.Combine(sourceConfig.DestinationDirectory, Path.GetFileName(pfi.OriginalFilepath));
MediaFile.MakeFilepathUnique(ref dstFilepath);
if (sourceConfig.MoveFiles)
{
File.Move(pfi.Filepath, dstFilepath);
}
else
{
File.Copy(pfi.Filepath, dstFilepath);
}
pfi.Filepath = dstFilepath;
bytesCopied += pfi.Size;
++n;
// Add to the destination queue
mediaQueue.Add(pfi);
}
TimeSpan elapsed;
unchecked
{
uint ticksElapsed = (uint)Environment.TickCount - startTicks;
elapsed = new TimeSpan(ticksElapsed * 10000L);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($"{verb} complete. {queue.Count} files, {bytesCopied / (1024.0 * 1024.0): #,##0.0} MB, {elapsed.FmtCustom()} elapsed");
}
static void ParseCommandLine(string[] args)
{
s_sourceConfiguration = new SourceConfiguration();
Program.s_photoFinisher = new PhotoFinisher();
Program.s_photoFinisher.ProgressReported += ReportProgress;
Program.s_photoFinisher.StatusReported += ReportStatus;
try
{
if (args.Length == 0)
{
s_operation = Operation.ShowHelp;
return;
}
for (int i = 0; i < args.Length; ++i)
{
switch (args[i].ToLowerInvariant())
{
case "-h":
case "-?":
s_operation = Operation.ShowHelp;
return;
case "-s":
if (s_mediaSource != null)
{
throw new Exception("CommandLine -s: Can only specify one source.");
}
s_mediaSource = new FileSource(NextArgument(args, ref i), false);
s_operation = Operation.ProcessMediaFiles;
break;
case "-st":
if (s_mediaSource != null)
{
throw new Exception("CommandLine -st: Can only specify one source.");
}
s_mediaSource = new FileSource(NextArgument(args, ref i), true);
s_operation = Operation.ProcessMediaFiles;
break;
case "-sdcim":
if (s_mediaSource != null)
{
throw new Exception("CommandLine -sdcim: Can only specify one source.");
}
s_mediaSource = new DcimSource();
s_operation = Operation.ProcessMediaFiles;
break;
case "-sname":
if (s_mediaSource != null)
{
throw new Exception("CommandLine -sname: Can only specify one source.");
}
s_mediaSource = NamedSource.GetNamedSource(NextArgument(args, ref i));
s_operation = Operation.ProcessMediaFiles;
break;
#if DEBUG
case "-copydcim":
{
Console.WriteLine("Copying test files.");
int n = DcimSource.CopyDcimTestFiles();
Console.WriteLine($"Copied {n} files from DCIM_Test to DCIM.");
}
break;
#endif
case "-selectafter":
s_sourceConfiguration.SelectAfter = NextArgumentAsDate(args, ref i)
.ResolveTimeZone(TimeZoneInfo.Local).Date;
break;
case "-selectincremental":
s_sourceConfiguration.SelectIncremental = true;
break;
case "-d":
{
string dst = NextArgument(args, ref i);
if (!Directory.Exists(dst))
{
throw new ArgumentException($"Destination folder '{dst}' does not exist.");
}
s_sourceConfiguration.DestinationDirectory = Path.GetFullPath(dst);
s_photoFinisher.DestinationDirectory = s_sourceConfiguration.DestinationDirectory;
}
break;
case "-w":
s_waitBeforeExit = true;
break;
case "-sortby":
switch (NextArgument(args, ref i).ToLowerInvariant())
{
case "y":
s_photoFinisher.SortBy = DatePathType.Y;
break;
case "ym":
s_photoFinisher.SortBy = DatePathType.YM;
break;
case "ymd":
s_photoFinisher.SortBy = DatePathType.YMD;
break;
case "ymds":
s_photoFinisher.SortBy = DatePathType.YMDS;
break;
default:
throw new ArgumentException($"Unexpected value for -sortby: {args[i]}.");
}
break;
case "-sort":
s_photoFinisher.SortBy = DatePathType.YMD;
break;
case "-move":
s_sourceConfiguration.MoveFiles = true;
break;
case "-autorot":
s_photoFinisher.AutoRotate = true;
break;
case "-orderednames":
s_photoFinisher.SetOrderedNames = true;
s_photoFinisher.SetMetadataNames = false;
break;
case "-filenamefrommetadata":
s_photoFinisher.SetMetadataNames = true;
s_photoFinisher.SetOrderedNames = false;
break;
case "-metadatafromfilename":
s_photoFinisher.MetadataFromFilename = SetMode.SetIfEmpty;
break;
case "-metadatafromfilenameoverwrite":
s_photoFinisher.MetadataFromFilename = SetMode.SetAlways;
break;
case "-saveoriginalfn":
case "-saveoriginalfilename":
s_photoFinisher.SaveOriginalFilaname = true;
break;
case "-setuuid":
s_photoFinisher.SetUuid = true;
break;
case "-transcode":
s_photoFinisher.Transcode = true;
break;
case "-tag":
s_photoFinisher.AddKeywords.Add(NextArgument(args, ref i));
break;
case "-determinedate":
s_photoFinisher.AlwaysSetDate = true;
break;
case "-alltheway":
s_photoFinisher.AutoRotate = true;
if (!s_photoFinisher.SetMetadataNames)
{
s_photoFinisher.SetOrderedNames = true;
}
s_photoFinisher.SaveOriginalFilaname = true;
s_photoFinisher.SetUuid = true;
s_photoFinisher.Transcode = true;
s_photoFinisher.AlwaysSetDate = true;
break;
case "-deduplicate":
s_photoFinisher.DeDuplicate = true;
break;
case "-setdate":
s_photoFinisher.SetDateTo = NextArgumentAsDate(args, ref i)
.ResolveTimeZone(TimeZoneInfo.Local);
break;
case "-shiftdate":
{
string timeShift = NextArgument(args, ref i);
// If the next argument starts with a sign, then the shift amount is a simple timespan.
if (timeShift[0] == '+' || timeShift[0] == '-')
{
string s = (timeShift[0] == '+') ? timeShift.Substring(1) : timeShift;
TimeSpan ts;
if (!TimeSpan.TryParse(s, System.Globalization.CultureInfo.InvariantCulture, out ts))
{
throw new ArgumentException($"Invalid value for -shiftDate '{timeShift}'.");
}
s_photoFinisher.ShiftDateBy = ts;
}
// Else, shift amount is the difference of two times
else
{
if (i + 1 >= args.Length)
{
throw new ArgumentException("-shiftDate requires two dates or one offset.");
}
string secondDate = NextArgument(args, ref i);
FileMeta.DateTag dtTarget;
if (!FileMeta.DateTag.TryParse(timeShift, out dtTarget))
{
throw new ArgumentException($"Invalid value for -shiftDate '{args[i]}'.");
}
FileMeta.DateTag dtSource;
if (!FileMeta.DateTag.TryParse(secondDate, out dtSource))
{
throw new ArgumentException($"Invalid value for -shiftDate '{args[i]}'.");
}
// Resolve timezone if it was ambiguous.
dtTarget.ResolveTimeZone(TimeZoneInfo.Local);
dtSource.ResolveTimeZone(TimeZoneInfo.Local);
// For whatever reason, they might have used different timezones. Take the difference between the UTC versions.
s_photoFinisher.ShiftDateBy = dtTarget.DateUtc.Subtract(dtSource.DateUtc);
}
}
break;
case "-settimezone":
{
string tz = NextArgument(args, ref i);
var tzi = TimeZoneParser.ParseTimeZoneId(tz);
if (tzi == null)
{
throw new ArgumentException($"Invalid value for -setTimezone '{tz}'. Use '-listTimezones' option to find valid values.");
}
s_photoFinisher.SetTimezoneTo = tzi;
}
break;
case "-changetimezone":
{
string tz = NextArgument(args, ref i);
var tzi = TimeZoneParser.ParseTimeZoneId(tz);
if (tzi == null)
{
throw new ArgumentException($"Invalid value for -changeTimezone '{tz}'. Use '-listTimezones' option to find valid values.");
}
s_photoFinisher.ChangeTimezoneTo = tzi;
}
break;
case "-updatefscreate":
s_photoFinisher.UpdateFileSystemDateCreated = true;
break;
case "-updatefsmod":
s_photoFinisher.UpdateFileSystemDateModified = true;
break;
case "-setwidth":
s_photoFinisher.SetWidth = NextArgumentAsInt(args, ref i);
break;
case "-setheight":
s_photoFinisher.SetHeight = NextArgumentAsInt(args, ref i);
break;
case "-log":
s_log = true;
break;
case "-listtimezones":
case "-listtimezone":
s_operation = Operation.ListTimeZones;
break;
case "-listnamedsources":
s_operation = Operation.ListNamedSources;
break;
case "-authonedrive":
s_sourceName = NextArgument(args, ref i);
s_operation = Operation.AuthOneDrive;
break;
#if DEBUG
case "-testmetadatafromfilename":
s_testAction = MediaFile.TestMetadataFromFilename;
s_testArgument = NextArgument(args, ref i);
s_operation = Operation.TestAction;
break;
#endif
default:
throw new ArgumentException($"Unexpected command-line argument: {args[i]}");
}
}
if (s_photoFinisher.SortBy != DatePathType.None && string.IsNullOrEmpty(s_photoFinisher.DestinationDirectory))
{
throw new ArgumentException("'-sort' option requires '-d' destination option.");
}
}
catch (Exception err)
{
Console.WriteLine(err.Message);
Console.WriteLine("Use '-h' for syntax help");
s_operation = Operation.CommandLineError;
}
} // ParseCommandLine
void DownloadMediaFiles(List <OdFileInfo> queue, SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
mediaQueue.ReportProgress($"Downloading media files from OneDrive to working folder: {sourceConfig.DestinationDirectory}.");
DateTime newestSelection = DateTime.MinValue;
// Sum up the size of the files to be downloaded
long selectedFilesSize = 0;
foreach (var fi in queue)
{
selectedFilesSize += fi.Size;
}
uint startTicks = (uint)Environment.TickCount;
long bytesDownloaded = 0;
int n = 0;
foreach (var fi in queue)
{
if (bytesDownloaded == 0)
{
mediaQueue.ReportStatus($"Downloading file {n + 1} of {queue.Count}");
}
else
{
uint ticksElapsed;
unchecked
{
ticksElapsed = (uint)Environment.TickCount - startTicks;
}
double bps = ((double)bytesDownloaded * 8000.0) / (double)ticksElapsed;
TimeSpan remain = new TimeSpan(((long)((selectedFilesSize - bytesDownloaded) / (bps / 8))) * 10000000L);
mediaQueue.ReportStatus($"Downloading file {n + 1} of {queue.Count}. Time remaining: {remain.FmtCustom()} Mbps: {(bps / (1024 * 1024)):#,###.###}");
}
string dstFilepath = Path.Combine(sourceConfig.DestinationDirectory, fi.OriginalFilename);
MediaFile.MakeFilepathUnique(ref dstFilepath);
FetchFile(fi.Url, dstFilepath);
bytesDownloaded += fi.Size;
++n;
// Add to the destination queue
mediaQueue.Add(new ProcessFileInfo(
dstFilepath,
fi.Size,
fi.OriginalFilename,
fi.OriginalDateCreated ?? DateTime.MinValue,
fi.OriginalDateModified ?? DateTime.MinValue));
if (fi.BookmarkDate.HasValue && newestSelection < fi.BookmarkDate)
{
newestSelection = fi.BookmarkDate.Value;
}
}
TimeSpan elapsed;
unchecked
{
uint ticksElapsed = (uint)Environment.TickCount - startTicks;
elapsed = new TimeSpan(ticksElapsed * 10000L);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($"Download complete. {queue.Count} files, {bytesDownloaded / (1024.0 * 1024.0): #,##0.0} MB, {elapsed.FmtCustom()} elapsed");
if (newestSelection > DateTime.MinValue)
{
if (sourceConfig.SetBookmark(m_bookmarkPath, newestSelection))
{
mediaQueue.ReportProgress($"Bookmark Set to {newestSelection}");
}
}
}
private List <ProcessFileInfo> SelectFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
mediaQueue.ReportProgress($"Selecting {(m_recursive ? "from" : "tree")}: {m_path}");
// Determine the "after" threshold from SelectAfter and SelectIncremental
var after = sourceConfig.GetBookmarkOrAfter(m_path);
m_newestSelection = after ?? DateTime.MinValue;
var queue = new List <ProcessFileInfo>();
int skippedFiles = 0;
try
{
DirectoryInfo di = new DirectoryInfo(m_directory);
foreach (var fi in di.EnumerateFiles(m_pattern, m_recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
if (((queue.Count + skippedFiles) % 100) == 0)
{
string message = (skippedFiles == 0)
? $"Selected: {queue.Count}"
: $"Selected: {queue.Count} Not Selected: {skippedFiles}";
mediaQueue.ReportStatus(message);
}
if (MediaFile.IsSupportedMediaType(fi.Extension))
{
// Limit date window of files to be selected
if (after.HasValue)
{
var date = MediaFile.GetBookmarkDate(fi.FullName);
if (!date.HasValue || date.Value <= after.Value)
{
++skippedFiles;
continue;
}
// For this operation, we do everything in localtime because photo
// DateTaken metadata is in localtime.
// This is after-the-fact but since it's a debugging test that's OK.
Debug.Assert(date.Value.Kind == DateTimeKind.Local);
if (m_newestSelection < date.Value)
{
m_newestSelection = date.Value;
}
}
queue.Add(new ProcessFileInfo(fi));
}
}
}
catch (Exception err)
{
throw new ArgumentException($"Source '{m_path}' not found. ({err.Message})", err);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress(skippedFiles == 0
? $" Selected: {queue.Count}"
: $" Selected: {queue.Count} Not Selected: {skippedFiles}");
// If SelectIncremental, report the new bookmark
if (sourceConfig.SelectIncremental && queue.Count > 0)
{
Debug.Assert(m_newestSelection > DateTime.MinValue);
mediaQueue.ReportProgress($" Newest: {m_newestSelection:yyyy'-'MM'-'dd' 'HH':'mm':'ss}");
}
return(queue);
}
