public static void EnqueueFiles(List <ProcessFileInfo> queue, IMediaQueue mediaQueue)
{
foreach (var pfi in queue)
{
mediaQueue.Add(pfi);
}
}
List <OdFileInfo> SelectFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
mediaQueue.ReportProgress("Selecting files");
// Determine the "after" threshold from SelectAfter and SelectIncremental
var after = sourceConfig.GetBookmarkOrAfter(m_bookmarkPath);
if (after.HasValue)
{
mediaQueue.ReportProgress($" Filter: Created after {after.Value}");
}
// Retrieve file info and enqueue
var queue = new List <OdFileInfo>();
int count = 0;
// This is a Kludge. Need a way to either detect the correct folder or to
// configure it when setting up a new named source. And, with the named
// source, we need to clean up the circumstance when credentials expire.
string nextUrl = m_sourceName.Equals("BrandtOneDrive")
? c_oneDriveSamsungCameraUrl : c_oneDriveCameraRollUrl;
do
{
var fileList = FetchJson(nextUrl);
var root = fileList.CreateNavigator();
nextUrl = root.XPVal("/root/a:item[@item='@odata.nextLink']");
foreach (XPathNavigator node in fileList.CreateNavigator().Select("/root/value/item"))
{
++count;
var odfi = new OdFileInfo(node);
if (!after.HasValue ||
(odfi.BookmarkDate.HasValue && odfi.BookmarkDate > after.Value))
{
queue.Add(odfi);
}
}
mediaQueue.ReportStatus($" Selected {queue.Count} Skipped {count - queue.Count}");
}while (!string.IsNullOrEmpty(nextUrl));
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($"Selected {queue.Count} Skipped {count - queue.Count}");
if (queue.Count > c_maxBatch)
{
// Sort so that we'll keep the oldest ones.
queue.Sort((a, b) => DateCompare(a.BookmarkDate, b.BookmarkDate));
queue.RemoveRange(c_maxBatch, queue.Count - c_maxBatch);
mediaQueue.RequestAnotherBatch = true;
mediaQueue.ReportProgress($"Batch limited to {queue.Count}");
}
return(queue);
}
public void RetrieveMediaFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
// DCIM requires destination directory
if (string.IsNullOrEmpty(sourceConfig.DestinationDirectory))
{
throw new Exception("DCIM source requires -d destination directory.");
}
var queue = SelectDcimFiles(sourceConfig, mediaQueue);
FileMover.CopyOrMoveFiles(queue, sourceConfig, mediaQueue);
CleanupDcfDirectories(mediaQueue);
}
public void RetrieveMediaFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
// if a destination directory not specified, error
if (string.IsNullOrEmpty(sourceConfig.DestinationDirectory))
{
throw new InvalidOperationException("OneDrive Source requires destination directory -d");
}
mediaQueue.ReportProgress($"Connecting to OneDrive: {m_sourceName}");
m_accessToken = NamedSource.GetOnedriveAccessToken(m_refreshToken);
var queue = SelectFiles(sourceConfig, mediaQueue);
DownloadMediaFiles(queue, sourceConfig, mediaQueue);
}
/// <summary>
/// Remove empty DCF directories from which files were moved.
/// </summary>
private void CleanupDcfDirectories(IMediaQueue mediaQueue)
{
foreach (string directoryName in m_dcfDirectories)
{
// Clean up the folders
try
{
DirectoryInfo di = new DirectoryInfo(directoryName);
// Get rid of thumbnails unless a matching file still exists
foreach (FileInfo fi in di.GetFiles("*.thm"))
{
bool hasMatch = false;
foreach (FileInfo fi2 in di.GetFiles(Path.GetFileNameWithoutExtension(fi.Name) + ".*"))
{
if (!string.Equals(fi2.Extension, ".thm", StringComparison.OrdinalIgnoreCase))
{
hasMatch = true;
break;
}
}
if (!hasMatch)
{
fi.Delete();
}
}
// Get rid of Windows thumbnails file (if it exists)
{
string thumbName = Path.Combine(di.FullName, "Thumbs.db");
if (File.Exists(thumbName))
{
File.Delete(thumbName);
}
}
// If the folder is empty, delete it
if (!di.EnumerateFileSystemInfos().Any())
{
di.Delete();
}
}
catch (Exception err)
{
// Report errors during cleanup but proceed with other files.
mediaQueue.ReportProgress($"Error cleaning up folders on removable drive '{Path.GetPathRoot(directoryName)}': {err.Message}");
}
}
}
public void RetrieveMediaFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
var queue = SelectFiles(sourceConfig, mediaQueue);
// if a destination directory was specified, copy or move the files
if (sourceConfig.DestinationDirectory != null)
{
FileMover.CopyOrMoveFiles(queue, sourceConfig, mediaQueue);
}
// Else, simply put them in the mediaQueue
else
{
FileMover.EnqueueFiles(queue, mediaQueue);
}
// Save the bookmark
if (m_newestSelection > DateTime.MinValue)
{
// Only sets a bookmark if incremental is on.
sourceConfig.SetBookmark(m_path, m_newestSelection);
}
}
private List <ProcessFileInfo> SelectDcimFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
if (sourceConfig.SelectIncremental)
{
throw new Exception("DCIM source is not compatible with -selectIncremental");
}
if (sourceConfig.SelectAfter.HasValue)
{
throw new Exception("DCIM source does not support -selectAfter");
}
var sourceFolders = new List <string>();
// Process each removable drive
foreach (DriveInfo drv in DriveInfo.GetDrives())
{
if (drv.IsReady && drv.DriveType == DriveType.Removable)
{
try
{
// File system structure is according to JEITA "Design rule for Camera File System (DCF) which is JEITA specification CP-3461
// See if the DCIM folder exists
DirectoryInfo dcim = new DirectoryInfo(Path.Combine(drv.RootDirectory.FullName, c_dcimDirectory));
if (dcim.Exists)
{
// Folders containing images must be named with three digits followed
// by five alphanumeric characters. First digit cannot be zero.
foreach (DirectoryInfo di in dcim.EnumerateDirectories())
{
if (di.Name.Length == 8)
{
int dirnum;
if (int.TryParse(di.Name.Substring(0, 3), out dirnum) && dirnum >= 100 && dirnum <= 999)
{
sourceFolders.Add(di.FullName);
}
}
}
}
}
catch (Exception)
{
// Suppress the error and move to the next drive.
}
} // If drive is ready and removable
} // for each drive
var queue = new List <ProcessFileInfo>();
// Add files from each source folder
foreach (var path in sourceFolders)
{
int bookmark = queue.Count;
mediaQueue.ReportProgress($"Selecting from: {path}");
DirectoryInfo di = new DirectoryInfo(path);
foreach (var fi in di.EnumerateFiles())
{
if ((queue.Count % 100) == 0)
{
mediaQueue.ReportStatus($"Selected: {queue.Count}");
}
if (MediaFile.IsSupportedMediaType(fi.Extension))
{
queue.Add(new ProcessFileInfo(fi));
}
}
if (queue.Count > bookmark)
{
m_dcfDirectories.Add(path);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($" Selected: {queue.Count - bookmark}");
}
return(queue);
}
public static void CopyOrMoveFiles(List <ProcessFileInfo> queue, SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
string verb = sourceConfig.MoveFiles ? "Moving" : "Copying";
mediaQueue.ReportProgress($"{verb} media files to working folder: {sourceConfig.DestinationDirectory}.");
// Sum up the size of the files to be copied
long selectedFilesSize = 0;
foreach (var pfi in queue)
{
selectedFilesSize += pfi.Size;
}
uint startTicks = (uint)Environment.TickCount;
long bytesCopied = 0;
int n = 0;
foreach (var pfi in queue)
{
if (bytesCopied == 0)
{
mediaQueue.ReportStatus($"{verb} file {n + 1} of {queue.Count}");
}
else
{
uint ticksElapsed;
unchecked
{
ticksElapsed = (uint)Environment.TickCount - startTicks;
}
double bps = ((double)bytesCopied * 1000.0) / (double)ticksElapsed;
double remaining = (selectedFilesSize - bytesCopied) / bps;
TimeSpan remain = new TimeSpan(((long)((selectedFilesSize - bytesCopied) / bps)) * 10000000L);
mediaQueue.ReportStatus($"{verb} file {n + 1} of {queue.Count}. Time remaining: {remain.FmtCustom()} MBps: {(bps / (1024 * 1024)):#,###.###}");
}
string dstFilepath = Path.Combine(sourceConfig.DestinationDirectory, Path.GetFileName(pfi.OriginalFilepath));
MediaFile.MakeFilepathUnique(ref dstFilepath);
if (sourceConfig.MoveFiles)
{
File.Move(pfi.Filepath, dstFilepath);
}
else
{
File.Copy(pfi.Filepath, dstFilepath);
}
pfi.Filepath = dstFilepath;
bytesCopied += pfi.Size;
++n;
// Add to the destination queue
mediaQueue.Add(pfi);
}
TimeSpan elapsed;
unchecked
{
uint ticksElapsed = (uint)Environment.TickCount - startTicks;
elapsed = new TimeSpan(ticksElapsed * 10000L);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($"{verb} complete. {queue.Count} files, {bytesCopied / (1024.0 * 1024.0): #,##0.0} MB, {elapsed.FmtCustom()} elapsed");
}
public static async Task <IMediaItem[]> FetchMetaData(this IMediaQueue mediaQueue)
{
var mediaItems = mediaQueue.Select(i => i.FetchMetaData());
return(await Task.WhenAll(mediaItems));
}
void DownloadMediaFiles(List <OdFileInfo> queue, SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
mediaQueue.ReportProgress($"Downloading media files from OneDrive to working folder: {sourceConfig.DestinationDirectory}.");
DateTime newestSelection = DateTime.MinValue;
// Sum up the size of the files to be downloaded
long selectedFilesSize = 0;
foreach (var fi in queue)
{
selectedFilesSize += fi.Size;
}
uint startTicks = (uint)Environment.TickCount;
long bytesDownloaded = 0;
int n = 0;
foreach (var fi in queue)
{
if (bytesDownloaded == 0)
{
mediaQueue.ReportStatus($"Downloading file {n + 1} of {queue.Count}");
}
else
{
uint ticksElapsed;
unchecked
{
ticksElapsed = (uint)Environment.TickCount - startTicks;
}
double bps = ((double)bytesDownloaded * 8000.0) / (double)ticksElapsed;
TimeSpan remain = new TimeSpan(((long)((selectedFilesSize - bytesDownloaded) / (bps / 8))) * 10000000L);
mediaQueue.ReportStatus($"Downloading file {n + 1} of {queue.Count}. Time remaining: {remain.FmtCustom()} Mbps: {(bps / (1024 * 1024)):#,###.###}");
}
string dstFilepath = Path.Combine(sourceConfig.DestinationDirectory, fi.OriginalFilename);
MediaFile.MakeFilepathUnique(ref dstFilepath);
FetchFile(fi.Url, dstFilepath);
bytesDownloaded += fi.Size;
++n;
// Add to the destination queue
mediaQueue.Add(new ProcessFileInfo(
dstFilepath,
fi.Size,
fi.OriginalFilename,
fi.OriginalDateCreated ?? DateTime.MinValue,
fi.OriginalDateModified ?? DateTime.MinValue));
if (fi.BookmarkDate.HasValue && newestSelection < fi.BookmarkDate)
{
newestSelection = fi.BookmarkDate.Value;
}
}
TimeSpan elapsed;
unchecked
{
uint ticksElapsed = (uint)Environment.TickCount - startTicks;
elapsed = new TimeSpan(ticksElapsed * 10000L);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress($"Download complete. {queue.Count} files, {bytesDownloaded / (1024.0 * 1024.0): #,##0.0} MB, {elapsed.FmtCustom()} elapsed");
if (newestSelection > DateTime.MinValue)
{
if (sourceConfig.SetBookmark(m_bookmarkPath, newestSelection))
{
mediaQueue.ReportProgress($"Bookmark Set to {newestSelection}");
}
}
}
private List <ProcessFileInfo> SelectFiles(SourceConfiguration sourceConfig, IMediaQueue mediaQueue)
{
mediaQueue.ReportProgress($"Selecting {(m_recursive ? "from" : "tree")}: {m_path}");
// Determine the "after" threshold from SelectAfter and SelectIncremental
var after = sourceConfig.GetBookmarkOrAfter(m_path);
m_newestSelection = after ?? DateTime.MinValue;
var queue = new List <ProcessFileInfo>();
int skippedFiles = 0;
try
{
DirectoryInfo di = new DirectoryInfo(m_directory);
foreach (var fi in di.EnumerateFiles(m_pattern, m_recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
if (((queue.Count + skippedFiles) % 100) == 0)
{
string message = (skippedFiles == 0)
? $"Selected: {queue.Count}"
: $"Selected: {queue.Count} Not Selected: {skippedFiles}";
mediaQueue.ReportStatus(message);
}
if (MediaFile.IsSupportedMediaType(fi.Extension))
{
// Limit date window of files to be selected
if (after.HasValue)
{
var date = MediaFile.GetBookmarkDate(fi.FullName);
if (!date.HasValue || date.Value <= after.Value)
{
++skippedFiles;
continue;
}
// For this operation, we do everything in localtime because photo
// DateTaken metadata is in localtime.
// This is after-the-fact but since it's a debugging test that's OK.
Debug.Assert(date.Value.Kind == DateTimeKind.Local);
if (m_newestSelection < date.Value)
{
m_newestSelection = date.Value;
}
}
queue.Add(new ProcessFileInfo(fi));
}
}
}
catch (Exception err)
{
throw new ArgumentException($"Source '{m_path}' not found. ({err.Message})", err);
}
mediaQueue.ReportStatus(null);
mediaQueue.ReportProgress(skippedFiles == 0
? $" Selected: {queue.Count}"
: $" Selected: {queue.Count} Not Selected: {skippedFiles}");
// If SelectIncremental, report the new bookmark
if (sourceConfig.SelectIncremental && queue.Count > 0)
{
Debug.Assert(m_newestSelection > DateTime.MinValue);
mediaQueue.ReportProgress($" Newest: {m_newestSelection:yyyy'-'MM'-'dd' 'HH':'mm':'ss}");
}
return(queue);
}
