public static void ChangesFileList(string savedStartPageToken, string pageToken, string user, string savelocation)
{
string bigfiles = ConfigurationManager.AppSettings["bigfiles"];
int intbigfiles = int.Parse(bigfiles);
intbigfiles = ((intbigfiles * 1024) * 1024);
Form1 sendtolog = new Form1();
sendtolog.TxtLog = "Noting files for download. Please wait.";
int x = 0;
if (File.Exists(savelocation + ".deltalog.tok"))
{
File.Delete(savelocation + ".deltalog.tok");
}
var start = CreateService.BuildService(user).Changes.GetStartPageToken().Execute();
using (StreamWriter deltalog = new StreamWriter(savelocation + ".deltalog.tok", true))
{
StreamWriter logFile = new StreamWriter(savelocation + ".recent.log");
while (pageToken != null)
{
var request = CreateService.BuildService(user).Changes.List(pageToken);
request.Fields = "changes,kind,newStartPageToken,nextPageToken";
request.RestrictToMyDrive = false;
request.PageSize = 1000;
request.IncludeRemoved = false;
request.Spaces = "drive";
var changes = request.Execute();
foreach (var change in changes.Changes)
{
try
{
x++;
string folderid = "";
string updatedfile = "";
updatedfile = change.File.Name;
string mimetype = change.File.MimeType;
long? filesize = change.File.Size;
if (change.File.Parents != null)
{
folderid = String.Join(",", change.File.Parents);
}
//Form1.proUserclass.Value = x;
// Record the changed file
// Let's...not backup every file on Earth. Check user options and determine if we want to consume our backup resources to downloading a file that -probably isn't- a normal document (ie: uploaded video/movie)
// Check the bigfiles appsetting. If the file is bigger than X (whatever is set), skip it.
if (bigfiles != "") // Is there input in the "skip files over X MB" textbox.
{
if (filesize >= intbigfiles) // For reference: 104857600 bytes == 100MB
{
//
}
else
{
updatedfile = updatedfile.Replace(",", "_").Replace("\"", "_").Replace("(", "-").Replace(")", "-").Replace("?", "_").Replace("<", "_").Replace(">", "_").Replace(";", "_").Replace("$", "_").Replace("@", "_").Replace("!", "_").Replace("|", "_").Replace(":", "_");
logFile.WriteLine(user + ": New or changed file found: " + change.FileId + " --- " + updatedfile);
updatedfile = string.Join("_", updatedfile.Split(Path.GetInvalidFileNameChars()));
logFile.Flush();
if (updatedfile.Length >= 35)
{
updatedfile = (updatedfile.Substring(0, 35) + "...");
}
if (mimetype == "application/vnd.google-apps.folder")
{
//
}
else
{
deltalog.WriteLine(change.FileId + "," + updatedfile + "," + mimetype + "," + folderid);
deltalog.Flush();
}
}
}
else
{
updatedfile = updatedfile.Replace(",", "_").Replace("\"", "_").Replace("(", "-").Replace(")", "-").Replace("?", "_").Replace("<", "_").Replace(">", "_").Replace(";", "_").Replace("$", "_").Replace("@", "_").Replace("!", "_").Replace("|", "_").Replace(":", "_");
logFile.WriteLine(user + ": New or changed file found: " + change.FileId + " --- " + updatedfile);
logFile.Flush();
if (mimetype == "application/vnd.google-apps.folder")
{
//
}
else
{
deltalog.WriteLine(change.FileId + "," + updatedfile + "," + mimetype + "," + folderid);
deltalog.Flush();
}
}
}
catch (Exception ex)
{
// Google seems to track every file ever created for the user
// it also keeps a record of the fileId. Even if a user
// deletes the file, it's still retained in Google's record
// although the file is no longer available.
// We'll throw an error to the user so the program will continue forth.
if (ex.Message.ToString().Contains("404"))
{
Console.WriteLine(user + ": 404 error. Selected record no longer exists.");
logFile.WriteLine(user + ": 404 error. Selected record no longer exists.");
logFile.Flush();
}
else
{
logFile.WriteLine(user + ": Verbatim error: @F(x)_ChangeFileList" + ex.Message.ToString());
logFile.Flush();
}
}
}
if (changes.NewStartPageToken != null)
{
// Last page, save this token for the next polling interval
savedStartPageToken = changes.NewStartPageToken;
}
// Bring our token up to date for next run
pageToken = changes.NextPageToken;
savedStartPageToken = changes.NextPageToken;
//File.WriteAllText(savelocation + ".currenttoken.tok", start.StartPageTokenValue + "," + pageToken);
File.WriteAllText(savelocation + ".currenttoken.tok", start.StartPageTokenValue);
}
deltalog.Close();
logFile.Close();
deltalog.Dispose();
logFile.Dispose();
//Form1.proUserclass.Value = 0;
}
}
private void bgW_DoWork(object sender, DoWorkEventArgs e)
{
{
try
{
chkLDAP.Enabled = false;
txtFile.ReadOnly = true;
btnStart.Text = "Cancel Backup";
CheckForIllegalCrossThreadCalls = false;
var appSettings = ConfigurationManager.AppSettings;
string checkreplace = ConfigurationManager.AppSettings["checkreplace"];
string userfile = txtFile.Text;
int counter = 0;
proBar1.Visible = true;
// Take a peek inside our users file!
// enumerate through each domain account and check for changed
// or new files.
if (userfile.Length >= 1)
{
if (bgW.CancellationPending)
{
e.Cancel = true;
stripLabel.Text = "Operation was canceled!";
return;
}
string[] users = File.ReadAllLines(userfile);
for (int z = 0; z >= counter; z++)
{
if (bgW.CancellationPending)
{
e.Cancel = true;
stripLabel.Text = "Operation was canceled!";
return;
}
string names = users[z];
if (users != null)
{
try
{
proBar1.Visible = true;
stripLabel.Text = "";
Console.WriteLine("Selecting user: "******"Selecting user: "******"savelocation"] + user + "\\";
FileInfo testdir = new FileInfo(savelocation);
//Console.WriteLine("New directory created at: " + testdir);
testdir.Directory.Create();
string savedStartPageToken = "";
var start = CreateService.BuildService(user).Changes.GetStartPageToken().Execute();
StreamWriter logFile = new StreamWriter(savelocation + ".recent.log");
// This token is set by Google, it defines changes made and
// increments the token value automatically.
// The following reads the current token file (if it exists)
if (File.Exists(savelocation + ".currenttoken.tok"))
{
StreamReader curtokenfile = new StreamReader(savelocation + ".currenttoken.tok");
savedStartPageToken = curtokenfile.ReadLine().ToString();
curtokenfile.Close();
curtokenfile.Dispose();
}
else
{
// Token record didn't exist. Create a generic file, start at "1st" token
// In reality, I have no idea what token to start at, but 1 seems to be safe.
Console.Write("Creating new token file.\n");
txtLog.Text += ("Creating new token file.\n" + Environment.NewLine);
StreamWriter sw = new StreamWriter(savelocation + ".currenttoken.tok");
sw.Write(1);
sw.Close();
sw.Dispose();
savedStartPageToken = "1";
}
proBar1.Value = 0;
Console.WriteLine("Previous user token: " + savedStartPageToken);
txtLog.Text += ("Previous user token: " + savedStartPageToken + Environment.NewLine);
Console.WriteLine("My brand new token --->: " + start.StartPageTokenValue);
txtLog.Text += ("Current user token: " + start.StartPageTokenValue + Environment.NewLine);
Console.Write("Logging into your Gapps domain as: " + user + "\nChecking for any existing files...\nPlease wait as this may take a while.\n");
txtLog.Text += ("Now logging into your Gapps domain as: " + user + Environment.NewLine + "Please wait as this may take a while." + Environment.NewLine);
proBar1.Value = 0;
string pageToken = savedStartPageToken;
int gtoken = int.Parse(start.StartPageTokenValue);
int mytoken = int.Parse(savedStartPageToken);
txtPrevToken.Text = pageToken.ToString();
txtCurrentToken.Text = gtoken.ToString();
if (gtoken <= 10)
{
Console.WriteLine("Nothing to save!\n");
txtLog.Text += ("User has nothing to save!" + Environment.NewLine);
}
else
{
if (pageToken == start.StartPageTokenValue)
{
Console.WriteLine("No file changes found for " + user + "\n");
txtLog.Text += ("No file changes found! Please wait while I tidy up." + Environment.NewLine);
}
else
{
// .deltalog.tok is where we will place our records for changed files
Console.WriteLine("Changes detected. Making notes while we go through these.");
if (File.Exists(savelocation + ".deltalog.tok"))
{
File.Delete(savelocation + ".deltalog.tok");
}
lblProgresslbl.Text = "Scanning Drive directory.";
using (StreamWriter deltalog = new StreamWriter(savelocation + ".deltalog.tok", true))
{
StreamWriter folderlog = new StreamWriter(savelocation + "folderlog.txt", true);
while (pageToken != null)
{
counter1++;
var request = CreateService.BuildService(user).Changes.List(pageToken);
request.Fields = "changes(file,fileId)";
request.Spaces = "drive";
//DateTime dt = DateTime.Now;
var changes = request.Execute();
foreach (var change in changes.Changes)
{
if (bgW.CancellationPending)
{
e.Cancel = true;
stripLabel.Text = "Operation was canceled!";
return;
}
try
{
string updatedfile = change.File.Name;
string mimetype = change.File.MimeType;
string folderid = String.Join(",", change.File.Parents);
string dirname = (savelocation + updatedfile + "\\");
FileInfo newdir = new FileInfo(dirname);
// If it's a folder, let's create it now so we can move files into it later.
if (mimetype == "application/vnd.google-apps.folder")
{
newdir.Directory.Create();
logFile.WriteLine("Directory " + dirname + " was successfully created!");
Console.WriteLine("Directory " + dirname + " was successfully created!\n");
folderlog.WriteLine(change.FileId + "," + dirname);
folderlog.Flush();
}
else
{
// Record the changed file
Console.WriteLine(user + ": New or changed file found: " + updatedfile + "\n");
logFile.WriteLine(user + ": New or changed file found: " + change.FileId + " --- " + updatedfile);
txtLog.Text += (user + ": New or changed file found: --- " + updatedfile + " Type: " + mimetype + Environment.NewLine);
deltalog.WriteLine(change.FileId + "," + updatedfile + "," + mimetype + "," + folderid);
deltalog.Flush();
}
}
catch (Exception ex)
{
// Google seems to track every file ever created for the user
// it also keeps a record of the fileId. Even if a user
// deletes the file, it's still retained in Google's record
// although the file is no longer available.
// We'll throw an error to the user so the program will continue forth.
if (ex.Message.ToString().Contains("404"))
{
Console.WriteLine(user + ": 404 error. Selected record no longer exists.");
logFile.WriteLine(user + ": 404 error. Selected record no longer exists.");
txtLog.Text += (user + ": 404 error. Selected record no longer exists. Possibly old file that was renamed or removed by the user." + Environment.NewLine);
}
else
{
//Console.WriteLine(user + ": Verbatim error: " + ex.Message.ToString() + "\n");
logFile.WriteLine(user + ": Verbatim error: " + ex.Message.ToString() + "\n");
//txtLog.Text += (user + ": Verbatim error: " + ex.Message.ToString() + "\n" + Environment.NewLine);
}
}
}
if (changes.NewStartPageToken != null)
{
// Last page, save this token for the next polling interval
savedStartPageToken = changes.NewStartPageToken;
}
// Bring our token up to date for next run
pageToken = changes.NextPageToken;
File.WriteAllText(savelocation + ".currenttoken.tok", start.StartPageTokenValue);
}
deltalog.Close();
folderlog.Close();
}
// Get all our files for the user. Max page size is 1k
// after that, we have to use Google's next page token
// to let us get more files.
FilesResource.ListRequest listRequest = CreateService.BuildService(user).Files.List();
listRequest.PageSize = 1000;
listRequest.Fields = "nextPageToken, files(id, name)";
string[] deltafiles = File.ReadAllLines(savelocation + ".deltalog.tok");
int totalfiles = deltafiles.Count();
int cnttototal = 0;
proBar1.Maximum = totalfiles;
IList <Google.Apis.Drive.v3.Data.File> files = listRequest.Execute()
.Files;
Console.WriteLine("\nFiles to backup:\n");
txtLog.Text += (Environment.NewLine + "Files to backup:" + Environment.NewLine);
if (deltafiles == null)
{
return;
}
else
{
double damn = ((gtoken - double.Parse(txtPrevToken.Text)));
damn = Math.Round((damn / totalfiles));
foreach (var file in deltafiles)
{
try
{
if (bgW.CancellationPending)
{
stripLabel.Text = "Backup canceled!";
e.Cancel = true;
break;
}
DateTime dt = DateTime.Now;
string[] foldervalues = File.ReadAllLines(savelocation + "folderlog.txt");
cnttototal++;
bgW.ReportProgress(cnttototal);
stripLabel.Text = "File " + cnttototal + " of " + totalfiles;
double mathisfun = ((100 * cnttototal) / totalfiles);
double mathToken = double.Parse(txtPrevToken.Text);
mathToken = Math.Round((damn + mathToken));
// Bring our token up to date for next run
txtPrevToken.Text = mathToken.ToString();
File.WriteAllText(savelocation + ".currenttoken.tok", mathToken.ToString());
lblProgresslbl.Text = ("Current progress: " + mathisfun.ToString() + "% completed.");
//if (cnttototal == totalfiles)
// stripLabel.Text = cnttototal + "/" + totalfiles + "!";
// Our file is a CSV. Column 1 = file ID, Column 2 = File name
var values = file.Split(',');
string fileId = values[0];
string fileName = values[1];
string mimetype = values[2];
string folder = values[3];
int foundmatch = 0;
int folderfilelen = foldervalues.Count();
fileName = fileName.Replace('\\', '_').Replace('/', '_').Replace(':', '_').Replace('!', '_').Replace('\'', '_').Replace('*', '_').Replace('#', '_').Replace('[', '_').Replace(']', '_');
Console.WriteLine("Filename: " + values[1]);
txtLog.Text += (Environment.NewLine + "Timestamp: " + dt.ToString() + " Filename: " + values[1]);
logFile.WriteLine("ID: " + values[0] + " - Filename: " + values[1]);
var requestfileid = CreateService.BuildService(user).Files.Get(fileId);
//requestfileid.Fields = "*";
//var folderidexec = requestfileid.Execute();
//var folderid = string.Join(",", folderidexec.Parents);
var request = CreateService.BuildService(user).Files.Export(fileId, mimetype);
//Default extensions for files. Not sure what this should be, so we'll null it for now.
string ext = null;
// Things get sloppy here. The reason we're checking MimeTypes
// is because we have to export the files from Google's format
// to a format that is readable by a desktop computer program
// So for example, the google-apps.spreadsheet will become an MS Excel file.
if (mimetype == "application/vnd.google-apps.spreadsheet" || mimetype == "application/vnd.google-apps.ritz" || mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
{
request = CreateService.BuildService(user).Files.Export(fileId, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
ext = ".xls";
}
if (mimetype == "application/vnd.google-apps.document" || mimetype == "application/vnd.google-apps.kix" || mimetype == "application/msword")
{
request = CreateService.BuildService(user).Files.Export(fileId, "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
ext = ".docx";
}
if (mimetype == "application/vnd.google-apps.presentation" || mimetype == "application/vnd.google-apps.punch")
{
request = CreateService.BuildService(user).Files.Export(fileId, "application/vnd.openxmlformats-officedocument.presentationml.presentation");
ext = ".ppt";
}
// Google's folders are really just mime file types. Check for the file type
// grab the name of the file, and create it as such on the server FS.
lblFile.Text = (savelocation + fileName + ext);
if (mimetype == "application/vnd.google-apps.folder")
{
throw new Exception("This is a folder which already exists. Skipping.");
}
else
{
// Again, things get a little sloppy. Let's find if these files match any of the types in our IF statement.
if (mimetype == "image/gif" || mimetype == "image/jpeg" || mimetype == "image/png" || mimetype == "text/plain" || mimetype == "application/pdf" || mimetype == "application/vnd.google-apps.drawing" || mimetype == "application/vnd.google-apps.form" || mimetype == "application/vnd.google-apps.audio" || mimetype == "audio/mpeg" || mimetype == "application/vnd.google-apps.unknown")
{
if (mimetype == "application/pdf")
{
throw new Exception("PDF files not supported right now as they tend to hang the backup.");
}
Console.Write("MIME Type update: " + mimetype);
txtLog.Text += (Environment.NewLine + "MIME Type update: " + mimetype + Environment.NewLine);
logFile.WriteLine(fileName + " MIME Type ---> " + mimetype + "\n");
string dest1 = Path.Combine(savelocation, fileName);
var stream1 = new System.IO.FileStream(dest1, FileMode.OpenOrCreate, FileAccess.ReadWrite);
lblFile.Text = (savelocation + fileName + ext);
// Add a handler which will be notified on progress changes.
// It will notify on each chunk download and when the
// download is completed or failed.
requestfileid.MediaDownloader.ProgressChanged +=
(IDownloadProgress progress) =>
{
switch (progress.Status)
{
case DownloadStatus.Downloading:
{
throw new Exception("File may be corrupted.");
break;
}
case DownloadStatus.Completed:
{
Console.WriteLine("Download complete.");
logFile.WriteLine(fileName + " downloaded successfully.\n");
break;
}
case DownloadStatus.Failed:
{
Console.WriteLine("Error: There is nothing I can do with this file\n Is this is a draw or form?");
txtLog.Text += (Environment.NewLine + "Error: There is nothing I can do with this file\n Is this is a draw or form?" + Environment.NewLine);
logFile.WriteLine(fileName + " could not be downloaded. Possible Google draw/form OR bad name.\n");
break;
}
}
};
requestfileid.Download(stream1);
stream1.Close();
stream1.Dispose();
}
else
{
// Any other file type, assume as know what it is (which in our case, will be a txt file)
// apply the mime type and carry on.
logFile.WriteLine(fileName + " MIME Type ---> " + mimetype + "\n");
string dest = Path.Combine(savelocation, fileName + ext);
var stream = new System.IO.FileStream(dest, FileMode.OpenOrCreate, FileAccess.ReadWrite);
int oops = 0;
// Add a handler which will be notified on progress changes.
// It will notify on each chunk download and when the
// download is completed or failed.
request.MediaDownloader.ProgressChanged +=
(IDownloadProgress progress) =>
{
switch (progress.Status)
{
case DownloadStatus.Downloading:
{
throw new Exception("File may be corrupted.");
break;
}
case DownloadStatus.Completed:
{
Console.WriteLine("Download complete.");
logFile.WriteLine(fileName + " downloaded successfully\n");
break;
}
case DownloadStatus.Failed:
{
oops = 1;
logFile.WriteLine(fileName + " could not be downloaded. Possible Google draw/form OR bad name.\n");
break;
}
}
};
request.Download(stream);
stream.Close();
stream.Dispose();
if (oops == 1)
{
txtLog.Text += (Environment.NewLine + fileName + " could not be downloaded. Possible Google draw/form OR bad name.\n" + Environment.NewLine);
stream.Close();
stream.Dispose();
}
else
{
// Move file to the FS directory.
while (foundmatch != folderfilelen)
{
foreach (string folders in foldervalues)
{
var split = folders.Split(',');
string folderid = split[0];
string foldername = split[1];
if (folder == folderid)
{
txtLog.Text += Environment.NewLine + "Info: Moving file to correct directory";
string filetomove = (savelocation + fileName + ext);
string dest1 = (foldername + fileName + ext);
var copyfrom = Path.Combine(savelocation, fileName + ext);
File.Move(lblFile.Text, dest1);
throw new Exception("File moved to: " + dest1);
}
else
{
foundmatch++;
}
}
}
txtLog.Text += (Environment.NewLine + "Download complete." + Environment.NewLine);
}
}
}
}
catch (Exception ex)
{
// Well, we found a file that isn't usable or convertable
// we typically see this with a google form or google draw file, too.
Console.Write("\nInfo: ---> " + ex.Message.ToString() + "\n");
txtLog.Text += (Environment.NewLine + "Info: " + ex.Message.ToString() + Environment.NewLine);
}
}
}
Console.WriteLine("\n\n\tBackup completed for selected user!");
lblFile.Text = "";
txtLog.Text += ("Backup completed for current selected user!" + Environment.NewLine);
proBar1.Value = 0;
btnStart.Text = "&Start Backup";
logFile.Close();
logFile.Dispose();
}
}
}
catch (Exception ex)
{
Console.WriteLine("Info: " + ex.Message.ToString());
txtLog.Text += (Environment.NewLine + "Info: " + ex.Message.ToString() + Environment.NewLine);
}
}
else
{
Console.WriteLine("No data found in " + users);
txtLog.Text += ("No data found in " + users);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("Info: " + ex.Message.ToString());
txtLog.Text += (Environment.NewLine);
//txtLog.Text += (Environment.NewLine + "Error: " + ex.Message.ToString());
}
}
}
public static void RecordFolderList(string savedStartPageToken, string pageToken, string user, string savelocation)
{
Console.WriteLine("RECORDFOLDERLIST(): " + savelocation);
if (File.Exists(savelocation + "folderlog.txt"))
{
File.Delete(savelocation + "folderlog.txt");
}
FilesResource.ListRequest request1 = null;
StreamWriter folderlog = new StreamWriter(savelocation + "folderlog.txt", true);
StreamWriter logFile = new StreamWriter(savelocation + ".recent.log");
string filename = (savelocation + "folderlog.txt");
request1 = CreateService.BuildService(user).Files.List();
request1.Q = @"mimeType='application/vnd.google-apps.folder'";
request1.Fields = "files";
request1.Spaces = "drive";
request1.PageSize = 500;
var changes = request1.Execute();
foreach (var change in changes.Files)
{
try
{
var subfolderreq = CreateService.BuildService(user).Files.Get(change.Id);
subfolderreq.Fields = "id,name,parents";
var sendreq = subfolderreq.Execute();
List <string> subfolder = new List <string>();
subfolder.Add(sendreq.Name);
string folderid = "";
string updatedfile = change.Name;
string mimetype = change.MimeType;
if (change.Parents != null)
{
foreach (string sub in subfolder)
{
folderid = String.Join(",", change.Parents);
folderid = ("," + folderid + "," + sub.ToString());
string subreplace = sub.Replace(":", "_");
subreplace = string.Join("_", subreplace.Split(Path.GetInvalidFileNameChars()));
Console.WriteLine("Creating directory: " + savelocation + subreplace.ToString());
updatedfile = subreplace.ToString();
Directory.CreateDirectory(savelocation + subreplace.ToString());
}
}
updatedfile = updatedfile.Replace(":", "_");
string dirname = (savelocation + updatedfile + "\\");
//If it's a folder, let's create it now so we can move files into it later.
if (mimetype.Contains("folder"))
{
if (change.Parents == null)
{
folderlog.WriteLine(change.Id + "," + dirname + folderid + "," + "0" + "," + "1");
folderlog.Flush();
Console.WriteLine("Creating directory: " + dirname);
Directory.CreateDirectory(dirname);
}
else
{
folderlog.WriteLine(change.Id + "," + dirname + folderid + "," + "0" + "," + "0");
folderlog.Flush();
Directory.CreateDirectory(dirname);
}
}
}
catch (AggregateException ex)
{
logFile.WriteLine(ex.Message.ToString());
logFile.Flush();
Console.WriteLine("Oops!: " + ex.Message.ToString());
}
}
logFile.Close();
logFile.Dispose();
folderlog.Close();
folderlog.Dispose();
CreateSubDir(savelocation);
}