public static int MoveFilesToFileSystem()
{
long moved_bytes = 0;
Manager.log.Info("MoveFilesToFileSystem: [START]");
using (DB db = new DB()) {
using (DB download_db = new DB()) {
while (true)
{
using (IDbCommand cmd = db.CreateCommand()) {
// execute this in chunks to avoid huge data transfers and slowdowns.
cmd.CommandText = "SELECT * FROM File WHERE NOT file_id IS NULL LIMIT 100";
using (IDataReader reader = cmd.ExecuteReader()) {
if (!reader.Read())
{
break;
}
do
{
DBFile file = new DBFile(reader);
byte [] buffer = new byte [1024];
int oid = file.file_id.Value;
int read;
string fn = FileUtilities.CreateFilename(file.md5, file.compressed_mime == MimeTypes.GZ, true);
using (FileStream writer = new FileStream(fn, FileMode.Create, FileAccess.Write, FileShare.Read)) {
using (Stream str = download_db.Download(file)) {
while ((read = str.Read(buffer, 0, buffer.Length)) != 0)
{
writer.Write(buffer, 0, read);
}
}
}
IDbTransaction transaction = download_db.BeginTransaction();
download_db.Manager.Delete(oid);
file.file_id = null;
file.Save(download_db);
transaction.Commit();
moved_bytes += file.size;
log.InfoFormat("MoveFilesToFileSystem: Moved oid {0} to {1} ({2} bytes, {3} total bytes moved)", oid, fn, file.size, moved_bytes);
} while (reader.Read());
}
}
}
while (true)
{
using (IDbCommand cmd = db.CreateCommand()) {
// execute this in chunks to avoid huge data transfers and slowdowns.
cmd.CommandText = "SELECT * FROM Revision WHERE (diff_file_id IS NULL AND NOT diff = '') OR (log_file_id IS NULL AND NOT log = '') LIMIT 100";
using (IDataReader reader = cmd.ExecuteReader()) {
if (!reader.Read())
{
break;
}
do
{
DBRevision revision = new DBRevision(reader);
string tmpfile = null;
if (!string.IsNullOrEmpty(revision.diff))
{
int length = 0;
if (revision.diff_file_id == null)
{
try {
length = revision.diff.Length;
tmpfile = Path.GetTempFileName();
File.WriteAllText(tmpfile, revision.diff);
DBFile diff = download_db.Upload(tmpfile, ".log", false, null);
revision.diff_file_id = diff.id;
revision.diff = null;
} finally {
try {
if (File.Exists(tmpfile))
{
File.Delete(tmpfile);
}
} catch (Exception ex) {
log.ErrorFormat("error deleting temp file: {0}", ex);
}
}
moved_bytes += length;
log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s diff to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
}
}
if (!string.IsNullOrEmpty(revision.log))
{
int length = 0;
if (revision.log_file_id == null)
{
try {
length = revision.log.Length;
tmpfile = Path.GetTempFileName();
File.WriteAllText(tmpfile, revision.log);
DBFile log = download_db.Upload(tmpfile, ".log", false, null);
revision.log_file_id = log.id;
revision.log = null;
} finally {
try {
if (File.Exists(tmpfile))
{
File.Delete(tmpfile);
}
} catch (Exception ex) {
log.ErrorFormat("error deleting temp file: {0}", ex);
}
}
moved_bytes += length;
Manager.log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s log to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
}
revision.log = null;
}
revision.Save(download_db);
} while (reader.Read());
}
}
}
}
}
Manager.log.Info("MoveFilesToFileSystem: [Done]");
return(0);
}