Example #1
0
        public static int MoveFilesToFileSystem()
        {
            long moved_bytes = 0;

            Manager.log.Info("MoveFilesToFileSystem: [START]");

            using (DB db = new DB()) {
                using (DB download_db = new DB()) {
                    while (true)
                    {
                        using (IDbCommand cmd = db.CreateCommand()) {
                            // execute this in chunks to avoid huge data transfers and slowdowns.
                            cmd.CommandText = "SELECT * FROM File WHERE NOT file_id IS NULL LIMIT 100";
                            using (IDataReader reader = cmd.ExecuteReader()) {
                                if (!reader.Read())
                                {
                                    break;
                                }

                                do
                                {
                                    DBFile  file   = new DBFile(reader);
                                    byte [] buffer = new byte [1024];
                                    int     oid    = file.file_id.Value;
                                    int     read;
                                    string  fn = FileUtilities.CreateFilename(file.md5, file.compressed_mime == MimeTypes.GZ, true);
                                    using (FileStream writer = new FileStream(fn, FileMode.Create, FileAccess.Write, FileShare.Read)) {
                                        using (Stream str = download_db.Download(file)) {
                                            while ((read = str.Read(buffer, 0, buffer.Length)) != 0)
                                            {
                                                writer.Write(buffer, 0, read);
                                            }
                                        }
                                    }

                                    IDbTransaction transaction = download_db.BeginTransaction();
                                    download_db.Manager.Delete(oid);
                                    file.file_id = null;
                                    file.Save(download_db);
                                    transaction.Commit();

                                    moved_bytes += file.size;
                                    log.InfoFormat("MoveFilesToFileSystem: Moved oid {0} to {1} ({2} bytes, {3} total bytes moved)", oid, fn, file.size, moved_bytes);
                                } while (reader.Read());
                            }
                        }
                    }

                    while (true)
                    {
                        using (IDbCommand cmd = db.CreateCommand()) {
                            // execute this in chunks to avoid huge data transfers and slowdowns.
                            cmd.CommandText = "SELECT * FROM Revision WHERE (diff_file_id IS NULL AND NOT diff = '') OR (log_file_id IS NULL AND NOT log = '') LIMIT 100";
                            using (IDataReader reader = cmd.ExecuteReader()) {
                                if (!reader.Read())
                                {
                                    break;
                                }

                                do
                                {
                                    DBRevision revision = new DBRevision(reader);
                                    string     tmpfile  = null;

                                    if (!string.IsNullOrEmpty(revision.diff))
                                    {
                                        int length = 0;
                                        if (revision.diff_file_id == null)
                                        {
                                            try {
                                                length  = revision.diff.Length;
                                                tmpfile = Path.GetTempFileName();
                                                File.WriteAllText(tmpfile, revision.diff);
                                                DBFile diff = download_db.Upload(tmpfile, ".log", false, null);
                                                revision.diff_file_id = diff.id;
                                                revision.diff         = null;
                                            } finally {
                                                try {
                                                    if (File.Exists(tmpfile))
                                                    {
                                                        File.Delete(tmpfile);
                                                    }
                                                } catch (Exception ex) {
                                                    log.ErrorFormat("error deleting temp file: {0}", ex);
                                                }
                                            }
                                            moved_bytes += length;
                                            log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s diff to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
                                        }
                                    }

                                    if (!string.IsNullOrEmpty(revision.log))
                                    {
                                        int length = 0;
                                        if (revision.log_file_id == null)
                                        {
                                            try {
                                                length  = revision.log.Length;
                                                tmpfile = Path.GetTempFileName();
                                                File.WriteAllText(tmpfile, revision.log);
                                                DBFile log = download_db.Upload(tmpfile, ".log", false, null);
                                                revision.log_file_id = log.id;
                                                revision.log         = null;
                                            } finally {
                                                try {
                                                    if (File.Exists(tmpfile))
                                                    {
                                                        File.Delete(tmpfile);
                                                    }
                                                } catch (Exception ex) {
                                                    log.ErrorFormat("error deleting temp file: {0}", ex);
                                                }
                                            }
                                            moved_bytes += length;
                                            Manager.log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s log to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
                                        }
                                        revision.log = null;
                                    }

                                    revision.Save(download_db);
                                } while (reader.Read());
                            }
                        }
                    }
                }
            }

            Manager.log.Info("MoveFilesToFileSystem: [Done]");

            return(0);
        }