public static DFile Download(Connection share, SyncFile file, long start, int size)
{
try
{
var s = GetServerIP(share.MachineName);
if (s != null)
{
Packet p = new Packet();
p.Command = "" + COMMANDS.Download;
p.Data = new object[]
{
share.Name,
share.Token,
file,
start,
size
};
var ret = (ReturnPacket)send(s.IP.ToString(), p);
if (ret.OK)
{
return((DFile)ret.Data);
}
return(null);
}
}
catch { }
return(null);
}
private static SyncFile internalcreatezip(object[] files, string sharepath)
{
//Console.WriteLine
_log.Info($"creating zip for share : {sharepath} ,file count = {files.Length}");
string fn = Guid.NewGuid().ToString() + ".zip";
string path = sharepath + ".ts" + _S + "Temp" + _S;
Directory.CreateDirectory(path);
var z = RaptorDB.Common.ZipStorer.Create(path + fn, "");
foreach (var ff in files)
{
SyncFile f = (SyncFile)ff;
try
{
//Console.WriteLine
_log.Info("ziping : " + f.F);
//_log.Debug("ziping : " + f.F);
using (var fs = new FileStream(sharepath + f.F, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
z.AddStream(RaptorDB.Common.ZipStorer.Compression.Deflate,
f.F,
fs, f.D, f.D.ToString("yyyy-MM-dd HH:mm:ss"));
}
}
catch (Exception ex) { _log.Error(ex); }
}
z.Close();
return(new SyncFile {
F = fn, D = DateTime.Now, S = new FileInfo(path + fn).Length
});
}
public static void DeleteZip(string sharename, string token, SyncFile file)
{
var share = Global.Shares.Find(x => x.Name == sharename);
DFile ret = new DFile();
ret.err = DownloadError.NOTFOUND;
string path = share.Path + ".ts" + _S + "Temp" + _S;
if (File.Exists(path + file.F))
{
File.Delete(path + file.F);
}
}
private void DoWork()
{
while (!_shutdown)
{
DateTime _lastSave = DateTime.MinValue;
while (_conn.isPaused == false && Global.PauseAll == false)
{
if (_que.Count == 0)
{
SaveQueue();
break;
}
if (_que.Count > 0 && ClientCommands.isConnected(_conn.MachineName))
{
SyncFile file = null;
DownloadAsZipFile();
try
{
_que.TryDequeue(out file);
if (file != null)
{
ProcessFile(file);
}
}
catch (Exception ex)
{
_log.Error(ex);
if (file != null)
{
_ErrorQue.Enqueue(file);
}
}
if (FastDateTime.Now.Subtract(_lastSave).TotalMinutes > Global.SaveQueueEvery_X_Minute)
{
SaveQueue();
_lastSave = FastDateTime.Now;
}
}
else
{
Thread.Sleep(3000);
}
}
Thread.Sleep(1000);
}
}
private void ProcessFile(SyncFile file)
{
string fn = _conn.Path + file.F;
//_log.Info("create dir : " + LongDirectory.GetDirectoryName(fn));
LongDirectory.CreateDirectory(LongDirectory.GetDirectoryName(fn));
if (LongFile.Exists(fn))
{
// kludge : for seconds resolution
var fi = new LongFileInfo(fn);
var d1 = fi.LastWriteTime.Ticks / _tickfilter;
var d2 = file.D.Ticks / _tickfilter;
if (d1 >= d2) // skip if already copied or newer
{
return;
}
}
_connInfo.LastFileNameDownloaded = fn;
fn += ".!torpedosync";
// FEATURE : resume semi downloaded file
// remove old semi downloaded file
LongFile.Delete(fn);
if (TorpedoSync.Global.isWindows == false)
{
fn = fn.Replace("\\", "/");
}
_downloading = true;
if (downloadfile(file, fn, ClientCommands.Download))
{
MoveExistingFileToArchive(file.F);
// rename downloaded file
LongFile.Move(fn, _conn.Path + file.F);
LongFile.SetLastWriteTime(_conn.Path + file.F, file.D);
}
_downloading = false;
}
public static DFile Download(string sharename, string token, SyncFile file, long start, int size)
{
var share = Global.Shares.Find(x => x.Name == sharename);
DFile ret = new DFile();
ret.err = DownloadError.NOTFOUND;
if (share == null)
{
var cs = Global.ConnectionList.Find(x => x.Name == sharename);
if (cs != null && cs.Token == token)
{
sendfile(file, start, size, cs.Path, ret);
}
}
else if (share.ReadOnlyToken == token || share.ReadWriteToken == token)
{
sendfile(file, start, size, share.Path, ret);
}
return(ret);
}
private static void sendfile(SyncFile file, long start, int size, string path, DFile ret)
{
if (LongFile.Exists(path + file.F) == false)
{
ret.err = DownloadError.NOTFOUND;
return;
}
var fi = new LongFileInfo(path + file.F);
// kludge : for seconds resolution
var d1 = fi.LastWriteTime.Ticks / Global.tickfilter;
var d2 = file.D.Ticks / Global.tickfilter;
if (d1 == d2)
{
FileStream fs = null;
try
{
fs = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
byte[] b = new byte[size];
fs.Seek(start, SeekOrigin.Begin);
fs.Read(b, 0, size);
ret.err = DownloadError.OK;
ret.data = b;
}
catch
{
ret.err = DownloadError.LOCKED;
}
finally
{
if (fs != null)
{
fs.Close();
}
}
}
else
{
ret.err = DownloadError.OLDER;
}
}
public static void DeleteZip(Connection share, SyncFile file)
{
try
{
var s = GetServerIP(share.MachineName);
if (s != null)
{
Packet p = new Packet();
p.Command = "" + COMMANDS.DeleteZip;
p.Data = new object[]
{
share.Name,
share.Token,
file
};
var ret = (ReturnPacket)send(s.IP.ToString(), p);
}
}
catch { }
}
public static DFile DownloadZip(string sharename, string token, SyncFile file, long start, int size)
{
var share = Global.Shares.Find(x => x.Name == sharename);
DFile ret = new DFile();
ret.err = DownloadError.NOTFOUND;
if (share == null)
{
var cs = Global.ConnectionList.Find(x => x.Name == sharename && x.Token == token);
if (cs != null)
{
string path = cs.Path + ".ts" + _S + "Temp" + _S;
sendzip(file.F, start, size, path, ret);
}
}
else if (share.ReadOnlyToken == token || share.ReadWriteToken == token)
{
string path = share.Path + ".ts" + _S + "Temp" + _S;
sendzip(file.F, start, size, path, ret);
}
return(ret);
}
public static Delta ComputeDelta(State masterstate, State clientstate)
{
DateTime dt = FastDateTime.Now;
Delta delta = new Delta();
SortedList <string, SyncFile> sfiles = new SortedList <string, SyncFile>(clientstate.Files.Count);
List <string> mfiles = new List <string>(masterstate.Files.Count);
foreach (var i in clientstate.Files)
{
sfiles.Add(i.F, i);
}
foreach (var i in masterstate.Files)
{
mfiles.Add(i.F);
}
mfiles.Sort();
masterstate.Folders.Sort();
// if client folder not on master -> delete
foreach (var i in clientstate.Folders)
{
int k = masterstate.Folders.BinarySearch(i);
if (k < 0)
{
delta.FoldersDeleted.Add(i);
}
}
// if client file not on master -> delete
foreach (var i in clientstate.Files)
{
int k = mfiles.BinarySearch(i.F);
if (k < 0)
{
delta.FilesDeleted.Add(i.F);
}
}
foreach (var i in masterstate.Files)
{
SyncFile o = null;
if (sfiles.TryGetValue(i.F, out o) == false)
{
delta.FilesAdded.Add(i);
}
else
{
// kludge : for seconds resolution
var d1 = i.D.Ticks / Global.tickfilter;
var d2 = o.D.Ticks / Global.tickfilter;
if (d1 > d2)
{
delta.FilesChanged.Add(i);
}
}
}
var s = "Sync R time secs = " + FastDateTime.Now.Subtract(dt).TotalSeconds;
//Console.WriteLine(s);
_log.Debug(s);
return(delta);
}
public static Delta SynchronizeSendRecieve(QueueProcessor quep, Connection share, State clientstate, Delta clientdelta)
{
DateTime dt = FastDateTime.Now;
Stopwatch sw = new Stopwatch();
sw.Start();
State masterLastState = GetLastState(share);
State masterstate = GetCurrentState(share.Path, share);
quep.SetInfo(masterstate);
_log.Info(" Reading state ms : " + sw.ElapsedMilliseconds);
sw.Reset();
var masterdelta = ComputeDelta(masterstate, masterLastState);
_log.Info(" Master delta ms : " + sw.ElapsedMilliseconds);
sw.Reset();
Delta clientsend = new Delta();
Delta masterque = new Delta();
SortedList <string, SyncFile> sfiles = new SortedList <string, SyncFile>(clientstate.Files.Count);
SortedList <string, SyncFile> mfiles = new SortedList <string, SyncFile>(masterstate.Files.Count);
foreach (var i in clientstate.Files)
{
sfiles.Add(i.F.ToLower(), i);
}
foreach (var i in masterstate.Files)
{
mfiles.Add(i.F.ToLower(), i);
}
_log.Info(" Creating dictionaries ms : " + sw.ElapsedMilliseconds);
sw.Reset();
masterque.FilesDeleted = clientdelta.FilesDeleted;
masterque.FoldersDeleted = clientdelta.FoldersDeleted;
if (masterLastState.Files.Count > 0 || masterLastState.Folders.Count > 0)
{
clientsend.FilesDeleted = masterdelta.FilesDeleted;
clientsend.FoldersDeleted = masterdelta.FoldersDeleted;
}
// check master files not exist or older on client -> que for client
foreach (var i in masterstate.Files)
{
SyncFile o = null;
if (sfiles.TryGetValue(i.F.ToLower(), out o) == false)
{
clientsend.FilesAdded.Add(i);
}
else
{
// kludge : for seconds resolution
var d1 = i.D.Ticks / Global.tickfilter;
var d2 = o.D.Ticks / Global.tickfilter;
if (d1 > d2)
{
clientsend.FilesChanged.Add(i);
}
}
}
_log.Info(" Generate client delta ms : " + sw.ElapsedMilliseconds);
sw.Reset();
// check client files not exist or older on master -> que for master
foreach (var i in clientstate.Files)
{
SyncFile o = null;
if (mfiles.TryGetValue(i.F.ToLower(), out o) == false)
{
masterque.FilesAdded.Add(i);
}
else
{
// kludge : for seconds resolution
var d1 = i.D.Ticks / Global.tickfilter;
var d2 = o.D.Ticks / Global.tickfilter;
if (d1 > d2)
{
masterque.FilesChanged.Add(i);
}
}
}
_log.Info(" Generating master que ms : " + sw.ElapsedMilliseconds);
sw.Reset();
var s = "Sync RW time secs = " + FastDateTime.Now.Subtract(dt).TotalSeconds;
_log.Debug(s);
quep.QueueDelta(masterque);
SaveState(share, masterstate);
return(clientsend);
}
private bool downloadfile(SyncFile file, string saveto, Func <Connection, SyncFile, long, int, DFile> func)
{
long left = file.S;
int retry = 0;
int mb = Global.DownloadBlockSizeMB * Global.MB;
//Stopwatch sw = new Stopwatch();
//sw.Start();
while (left > 0)
{
long len = left;
long start = file.S - left;
if (len > mb)
{
len = mb;
}
//sw.Reset();
DFile df = func(_conn, file, start, (int)len);
//_connInfo.Mbs = (len / sw.ElapsedTicks) / (1000*TimeSpan.TicksPerSecond) ;
if (df == null)
{
retry++;
Thread.Sleep(new Random((int)FastDateTime.Now.Ticks).Next(2000) + 500);
if (retry > 10)
{
_log.Info(" null ");
_que.Enqueue(file);
//if (LongFile.Exists(saveto))
LongFile.Delete(saveto);
return(false);
}
}
else if (df.err == DownloadError.OK)
{
retry = 0;
string ifn = saveto;
if (TorpedoSync.Global.isWindows == false)
{
ifn = saveto.Replace("\\", "/");
}
_log.Info("len = " + len + " , " + saveto.Replace(".!torpedosync", ""));
left -= len;
LongDirectory.CreateDirectory(LongDirectory.GetDirectoryName(ifn));
// save to disk
var fs = new FileStream(ifn, FileMode.OpenOrCreate);
fs.Seek(0L, SeekOrigin.End);
fs.Write(df.data, 0, df.data.Length);
fs.Close();
if (left == 0)
{
var fi = new LongFileInfo(saveto);
if (fi.Length != file.S)
{
// FEATURE : file length don't match
_log.Info("file length don't match.");
return(false);
}
}
}
else if (df.err == DownloadError.OLDER || df.err == DownloadError.LOCKED)
{
_ErrorQue.Enqueue(file);
_connInfo.LastFileNameDownloaded = "";
_log.Info("Locked/Older : " + saveto);
return(false);
}
else if (df.err == DownloadError.NOTFOUND)
{
_log.Info("Not Found : " + saveto);
_connInfo.LastFileNameDownloaded = "";
_ErrorQue.Enqueue(file);
LongFile.Delete(saveto);
return(false);
}
}
return(true);
}
private void DownloadAsZipFile()
{
SyncFile file = null;
if (Global.BatchZip)
{
int size = 0;
int mb = Global.BatchZipFilesUnderMB * Global.MB;
List <SyncFile> ziplist = new List <SyncFile>();
while (_que.Count > 10)
{
if (_que.TryPeek(out file))
{
if (file.S + size < mb &&
file.S < Global.SmallFileSize) // small files
{
_que.TryDequeue(out file);
ziplist.Add(file);
}
else
{
break;
}
}
}
if (ziplist.Count > 0)
{
try
{
_downloading = true;
var sf = ClientCommands.CreateZip(_conn, ziplist);
if (sf != null)
{
string path = _conn.Path + ".ts" + _S + "Temp" + _S;
LongDirectory.CreateDirectory(path);
if (downloadfile(sf, path + sf.F, ClientCommands.DownloadZip))
{
string zfn = path + sf.F;
string fn = sf.F;
if (TorpedoSync.Global.isWindows == false)
{
zfn = zfn.Replace("\\", "/");
}
var zf = ZipStorer.Open(zfn, FileAccess.Read);
foreach (var z in zf.ReadCentralDir())
{
if (TorpedoSync.Global.isWindows)
{
fn = z.FilenameInZip.Replace("/", "\\");
}
else
{
fn = z.FilenameInZip;
}
MoveExistingFileToArchive(fn);
LongDirectory.CreateDirectory(LongDirectory.GetDirectoryName(_conn.Path + fn));
try
{
_log.Info("unzip : " + fn);
//_log.Debug("unzip : " + fn);
using (var fs = new FileStream(_conn.Path + fn, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite))
{
_connInfo.LastFileNameDownloaded = fn;
zf.ExtractFile(z, fs);
fs.Close();
}
var dt = z.ModifyTime;
if (z.Comment != "")
{
//_log.Info($"date {z.Comment}");
if (DateTime.TryParse(z.Comment, out dt) == false)
{
dt = z.ModifyTime;
}
}
var dtt = dt;
if (TorpedoSync.Global.isWindows)
{
dtt = dt.ToUniversalTime();
}
LongFile.SetLastWriteTime(_conn.Path + fn, dtt);
}
catch (Exception ex) { _log.Error(ex); }
}
zf.Close();
ClientCommands.DeleteZip(_conn, sf);
_log.Info("Decompress zip done : " + sf.F);
LongFile.Delete(zfn);
}
}
_downloading = false;
}
catch (Exception ex) { _log.Error(ex); }
}
}
}