private string IndexFile(FileQueue filequeue)
{
try
{
using (var client = new AutodeskFileServiceClient(Binding, Endpoint))
{
var indexedRevitFamily = Indexer.GetFile(filequeue.FilePath);
indexedRevitFamily.MC_OwnerId = Constants.OwnerId;
var request = new IndexAutodeskFileRequest(indexedRevitFamily, true);
var items = indexedRevitFamily.Items;
indexedRevitFamily.Items = null;
var result = client.IndexAutodeskFile(request);
foreach (var itemRequest in
items.Select(item => new AddTypeToFileRequest(item, indexedRevitFamily.Name, Constants.OwnerId)))
{
var typeResponse = client.AddTypeToFile(itemRequest);
}
return result.IndexAutodeskFileResult;
}
}
catch (Exception)
{
return Enums.IndexStatus.DataSyncFailed.ToString();
}
}
public void TestAddDirectoryCreateFileChangeFileRenameFileRemoveFileFast()
{
var directoryName = Guid.NewGuid().ToString();
var directory = Directory.CreateDirectory(directoryName);
var provider = new TokenProvider(new LexerMock());
var storage = new StorageMock();
using (var objectUnderTest = new FileQueue(storage, provider))
{
objectUnderTest.Add(directory.FullName);
var fileName = directory.FullName + "\\" + Guid.NewGuid().ToString();
using (var fs = File.Create(fileName))
{
fs.Write(new byte[] { 1 }, 0, 1);
}
using (StreamWriter sw = new StreamWriter(fileName))
{
sw.Write('a');
sw.Flush();
sw.Close();
}
var newFileName = directory.FullName + "\\" + Guid.NewGuid().ToString();
File.Move(fileName, newFileName);
File.Delete(newFileName);
Thread.Sleep((int)(0.1 * FileQueue.ProcessPeriodMS));
Assert.AreEqual(1, storage.Actions.Count);
Thread.Sleep((int)(1.25 * FileQueue.ProcessPeriodMS));
Assert.AreEqual(1, storage.Actions.Count);
var tuple = storage.Actions.Dequeue();
Assert.AreEqual(newFileName, tuple.Item1);
Assert.AreEqual(StorageMock._Delete, tuple.Item2);
}
Directory.Delete(directory.FullName);
}
public void SimpleTest()
{
QueueConfig config = new QueueConfig()
{
QueueDirectory = @"d:\workspace\data\test-queue",
DataConverter = new ObjectConverter(),
QueueName = "test01"
};
TestObject t1 = new TestObject()
{
Str = "t1-str",
Num = 123,
Flo = 1.3234f,
To = new TestObject()
{
Str = "t2-str",
Num = 35654,
Flo = 4935.2394f,
}
};
IFileQueue <TestObject> fq = FileQueue <TestObject> .Create(config);
fq.Enqueue(t1);
TestObject t2 = fq.Dequeue();
Assert.AreEqual(t1.Str, t2.Str);
Assert.AreEqual(t1.Num, t2.Num);
Assert.AreEqual(t1.Flo, t2.Flo);
Assert.AreEqual(t1.To.Str, t2.To.Str);
Assert.AreEqual(t1.To.Num, t2.To.Num);
Assert.AreEqual(t1.To.Flo, t2.To.Flo);
}
public void DequeueTimeoutTest()
{
int timeout = 5000;
QueueConfig config = new QueueConfig()
{
QueueDirectory = @"d:\workspace\data\test-queue",
DataConverter = new ObjectConverter(),
QueueName = "test04",
DequeueTimeoutMilliseconds = timeout
};
IFileQueue <string> fq = FileQueue <string> .Create(config);
DateTime s = DateTime.Now;
try
{
fq.Dequeue();
Assert.Fail();
}
catch (DequeueTimeoutException e)
{
if (DateTime.Now - s < TimeSpan.FromMilliseconds(timeout))
{
Assert.Fail();
}
else
{
Assert.IsFalse(e.IsBroken);
}
}
}
public ActionResult UploadFotoQueueFilial(FileQueue file)
{
if ((String)Session["Ativa"] == null)
{
return(RedirectToAction("Login", "ControleAcesso"));
}
if (file == null)
{
ModelState.AddModelError("", OdontoWeb_Resources.ResourceManager.GetString("M0019", CultureInfo.CurrentCulture));
return(RedirectToAction("VoltarAnexoEquipe"));
}
Int32 idAss = (Int32)Session["IdAssinante"];
Int32 idVolta = (Int32)Session["IdVolta"];
FILIAL item = baseApp.GetById(idVolta);
USUARIO usu = (USUARIO)Session["UserCredentials"];
var fileName = file.Name;
String caminho = "/Imagens/" + ((Int32)Session["IdAssinante"]).ToString() + "/Filial/" + item.FILI_CD_ID.ToString() + "/Logos/";
String path = Path.Combine(Server.MapPath(caminho), fileName);
System.IO.File.WriteAllBytes(path, file.Contents);
//Recupera tipo de arquivo
extensao = Path.GetExtension(fileName);
String a = extensao;
// Gravar registro
item.FILI_AQ_LOGOTIPO = "~" + caminho + fileName;
objetoAntes = item;
Int32 volta = baseApp.ValidateEdit(item, objetoAntes, usu);
return(RedirectToAction("EditarFilial", new { id = idVolta }));
}
void Trace(string strLog)
{
if (strLog.Length == 0)
{
return;
}
lock (ObjLock)
{
try
{
if (GenerateLogName())
{
UpdateFileQueue();
FileQueue.Enqueue(StrLogPath + "\\" + StrLogName);
}
StreamWriter streamWriter = File.AppendText(StrLogPath + "\\" + StrLogName);
streamWriter.WriteLine(strLog);
streamWriter.Flush();
streamWriter.Close();
}
catch (Exception e)
{
Console.Error.WriteLine(e.Message);
}
}
}
public void TestQueueStartAddDirectoryWithFiles()
{
var directoryName = Guid.NewGuid().ToString();
var directory = Directory.CreateDirectory(directoryName);
var fileName1 = directory.FullName + "\\" + Guid.NewGuid().ToString();
var fileName2 = directory.FullName + "\\" + Guid.NewGuid().ToString();
using (var fs = File.Create(fileName1))
{
}
using (var fs = File.Create(fileName2))
{
}
var provider = new TokenProvider(new LexerMock());
var storage = new StorageMock();
using (var objectUnderTest = new FileQueue(storage, provider))
{
objectUnderTest.Add(directory.FullName);
Thread.Sleep((int)(FileQueue.ProcessPeriodMS * 1.25));
Assert.AreEqual(2, storage.Actions.Count);
var file1 = storage.Actions.Dequeue();
var file2 = storage.Actions.Dequeue();
Assert.IsTrue((file1.Item1 == fileName1 && file2.Item1 == fileName2) ||
(file1.Item1 == fileName2 && file2.Item1 == fileName1));
Assert.AreEqual(file1.Item2, StorageMock._Add);
Assert.AreEqual(file2.Item2, StorageMock._Add);
}
File.Delete(fileName1);
File.Delete(fileName2);
Directory.Delete(directory.FullName);
}
private void ThreadProc(object state)
{
while (!cancellation.IsCancellationRequested)
{
MultiThreadedFileOperationState targets;
while (!FileQueue.TryDequeue(out targets))
{
Thread.Sleep(500);
}
var source = targets.SourceFile;
var target = targets.TargetFile;
try
{
OnOperationStarted(this, new FileOperationStartedEventArgs(source.FullName));
PreOperationHandlers(source, target);
ProcessFile(source, target);
target.Refresh();
PostOperationHandlers(source, target);
OnOperationCompleted(this, new FileOperationCompletedEventArgs(target));
}
catch (OperationCanceledException)
{
}
catch (Exception e)
{
OnError(this, new FileOperationErrorEventArgs(e));
}
}
}
static void CacheWSFilesImpl2()
{
try {
Log.Called();
var timer = Stopwatch.StartNew();
var wsPath = Entry.GamePaths.WorkshopModsPath;
var files1 = Directory.GetFiles(wsPath, "*.dll", searchOption: SearchOption.AllDirectories)
.Where(path => !path.Contains(Path.PathSeparator + "_"));
var files2 = Directory.GetFiles(wsPath, "*.crp", searchOption: SearchOption.AllDirectories)
.Where(path => !Packages.IsPathExcluded(path));
var files = files1.Concat(files2).ToArray();
var fileQueue = new FileQueue(files);
int nThreads = Math.Min(100, files.Length / 5);
Thread[] threads = new Thread[nThreads];
for (int i = 0; i < nThreads; ++i)
{
threads[i] = new Thread(CacheFilesThread);
threads[i].Start(fileQueue);
}
while (!fileQueue.Finished)
{
Thread.Sleep(10);
}
Log.Info($"caching access to {files.Length} files took {timer.ElapsedMilliseconds}ms");
} catch (Exception ex) {
Log.Exception(ex);
}
public void WriteReport(string FileName, Dictionary <string, string> dicData)
{
string content = File.ReadAllText(".\\config\\Page.htm", Encoding.GetEncoding(936));
File.WriteAllText(this.StrTempPath + FileName + ".htm", ReplaceData(content, dicData), Encoding.GetEncoding(936));
UpdateFileQueue();
FileQueue.Enqueue(StrTempPath + FileName);
}
public void TestQueueSimpleStartStop()
{
var provider = new TokenProvider(new LexerMock());
var storage = new StorageMock();
using (var objectUnderTest = new FileQueue(storage, provider))
{
}
}
protected override void SearchForFiles(CancellationToken cancellationToken = default)
{
InitializeThreads();
base.SearchForFiles(cancellationToken);
while (FileQueue.Any() && !cancellation.IsCancellationRequested)
{
Thread.Sleep(500);
}
}
void UpdateFileQueue()
{
if (MaxFileQty > 0)
{
int qty = FileQueue.Count - MaxFileQty + 1;
if (qty > 0)
{
for (int i = 0; i < qty; i++)
{
File.Delete(FileQueue.Dequeue());
}
}
}
}
void GenFileQueue()
{
DirectoryInfo dirinfo = new DirectoryInfo(StrLogPath);
FileInfo[] Files = dirinfo.GetFiles();
// 递增排序
Array.Sort <FileInfo>(Files, (FileInfo x, FileInfo y) => { return(x.LastWriteTime.CompareTo(y.LastWriteTime)); });
// 递减排序
//Array.Sort<FileInfo>(Files, (FileInfo x, FileInfo y) => { return y.LastWriteTime.CompareTo(x.LastWriteTime); });
foreach (var item in Files)
{
FileQueue.Enqueue(StrLogPath + "\\" + item.Name);
}
}
/// <summary>
/// Поиск файлов в выбранной папке (рекурсивно) и заполнение списка файлов
/// </summary>
/// <param name="path"></param>
public static void Search(FileQueue fileQueue, string path)
{
string[] files = Directory.GetFiles(path);
string[] directories = Directory.GetDirectories(path);
foreach (var file in files)
{
fileQueue.Enqueue(new FileInfo(file));
}
foreach (var dir in directories)
{
Search(fileQueue, dir);
}
}
protected override void OnFileFound(object sender, FileFoundEventArgs args)
{
var file = args.Item;
var target = FileUtils.GetDestinationFile(Source, file, Destination);
// Block so we don't queue too many files at once.
while (FileQueue.Count > MaxThreads * 5 && !cancellation.IsCancellationRequested)
{
Thread.Sleep(500);
}
if (cancellation.IsCancellationRequested)
{
return;
}
FileQueue.Enqueue(new MultiThreadedFileOperationState(file, target));
}
public void UploadFileToSession(IEnumerable <HttpPostedFileBase> files)
{
List <FileQueue> queue = new List <FileQueue>();
foreach (var file in files)
{
FileQueue f = new FileQueue();
f.Name = Path.GetFileName(file.FileName);
f.ContentType = Path.GetExtension(file.FileName);
MemoryStream ms = new MemoryStream();
file.InputStream.CopyTo(ms);
f.Contents = ms.ToArray();
queue.Add(f);
}
Session["FileQueueAgenda"] = queue;
}
public void DequeueTimeoutExceptionBrokenTest()
{
QueueConfig config = new QueueConfig()
{
QueueDirectory = @"d:\workspace\data\test-queue",
DataConverter = new Utf8Converter(),
QueueName = "test06",
DequeueTimeoutMilliseconds = 5000
};
IFileQueue <string> fq = FileQueue <string> .Create(config);
StringBuilder data = new StringBuilder();
for (int j = 0; j < 1000; ++j)
{
data.Append(Guid.NewGuid().ToString());
}
fq.Enqueue(data.ToString());
try
{
string result = fq.Dequeue();
Assert.Fail();
}
catch (DequeueTimeoutException e)
{
byte[] remain = fq.DequeueRawData();
byte[] whole = new byte[e.QueueData.Length + remain.Length];
using (MemoryStream wholeStream = new MemoryStream(whole))
{
wholeStream.Write(e.QueueData, 0, e.QueueData.Length);
wholeStream.Write(remain, 0, remain.Length);
}
string result = fq.DeserializeQueueData(whole);
Assert.AreEqual(data.ToString(), result);
}
}
public void ScanFolderAndFill()
{
if (_pathToSrc.Length == 0)
{
return;
}
// Флаги используются в тасках
_flagFileSearchEnd = false;
_flagParsingEnd = false;
// Очереди обработки, используются в тасках
_fileQueue = new FileQueue();
_fileTypeQueue = new FileTypesQueue();
_finderList = new FinderList();
//
// Ищем все файлы в отдельном потоке
//
var taskFileSearch = Task.Factory.StartNew(FindFiles);
//
// Запускаем задачу заполнения списка для поиска
//
Task.Factory.StartNew(FillFinderList);
//
// Парсим файлы в несколько потоков
//
// Определяем сколько ядер в система и создадим столько же потоков по парсингу.
var numberCores = Environment.ProcessorCount;
var tasks = new Task[numberCores];
for (int i = 0; i < numberCores; i++)
{
tasks[i] = Task.Factory.StartNew(ParseFiles);
}
// После выполнения всез задач будет сбрасываться флаг
Task.Factory.ContinueWhenAll(tasks, (taskResult) => { _flagParsingEnd = true; });
}
public void RawDataTest()
{
QueueConfig config = new QueueConfig()
{
QueueDirectory = @"d:\workspace\data\test-queue",
DataConverter = new Utf8Converter(),
QueueName = "test05",
DequeueTimeoutMilliseconds = 5000
};
IFileQueue <string> fq = FileQueue <string> .Create(config);
StringBuilder data = new StringBuilder();
for (int j = 0; j < 20; ++j)
{
data.Append(Guid.NewGuid().ToString());
}
fq.Enqueue(data.ToString());
byte[] raw = fq.DequeueRawData();
string result = fq.DeserializeQueueData(raw);
Assert.AreEqual(data.ToString(), result);
}
public void TestCreateNestedDirectoryCopyFileCreateFileCMoveFileUpDeleteFileSlow()
{
var directoryName = Guid.NewGuid().ToString();
var directory = Directory.CreateDirectory(directoryName);
var helperDirectory = Directory.CreateDirectory(Guid.NewGuid().ToString());
var fileNameWOPath = Guid.NewGuid().ToString();
var fileName = helperDirectory.FullName + "\\" + fileNameWOPath;
using (var fs = File.Create(fileName))
{
fs.Write(new byte[] { 1 }, 0, 1);
}
var provider = new TokenProvider(new LexerMock());
var storage = new StorageMock();
using (var objectUnderTest = new FileQueue(storage, provider))
{
objectUnderTest.Add(directory.FullName);
var subdirectory = directory.CreateSubdirectory(Guid.NewGuid().ToString());
var newFileName = subdirectory.FullName + "\\" + fileNameWOPath;
File.Move(fileName, newFileName);
Thread.Sleep((int)(1.25 * FileQueue.ProcessPeriodMS));
Assert.AreEqual(1, storage.Actions.Count);
var createdFileName = subdirectory.FullName + "\\" + Guid.NewGuid().ToString();
using (var fs = File.Create(createdFileName))
{
fs.Write(new byte[] { 1 }, 0, 1);
}
Thread.Sleep((int)(1.25 * FileQueue.ProcessPeriodMS));
Assert.AreEqual(2, storage.Actions.Count);
var newDestination = directory.FullName + "\\" + Guid.NewGuid().ToString();
File.Move(createdFileName, newDestination);
Thread.Sleep((int)(1.25 * FileQueue.ProcessPeriodMS));
Assert.AreEqual(4, storage.Actions.Count);
File.Delete(newDestination);
Assert.AreEqual(5, storage.Actions.Count);
File.Delete(newFileName);
Assert.AreEqual(6, storage.Actions.Count);
var tuple = storage.Actions.Dequeue();
Assert.AreEqual(newFileName, tuple.Item1);
Assert.AreEqual(StorageMock._Add, tuple.Item2);
tuple = storage.Actions.Dequeue();
Assert.AreEqual(createdFileName, tuple.Item1);
Assert.AreEqual(StorageMock._Add, tuple.Item2);
tuple = storage.Actions.Dequeue();
Assert.AreEqual(createdFileName, tuple.Item1);
Assert.AreEqual(StorageMock._Delete, tuple.Item2);
tuple = storage.Actions.Dequeue();
Assert.AreEqual(newDestination, tuple.Item1);
Assert.AreEqual(StorageMock._Add, tuple.Item2);
tuple = storage.Actions.Dequeue();
Assert.AreEqual(newDestination, tuple.Item1);
Assert.AreEqual(StorageMock._Delete, tuple.Item2);
tuple = storage.Actions.Dequeue();
Assert.AreEqual(newFileName, tuple.Item1);
Assert.AreEqual(StorageMock._Delete, tuple.Item2);
}
Directory.Delete(helperDirectory.FullName);
Directory.Delete(directory.FullName, true);
}
public ActionResult UploadFileQueueAgenda(FileQueue file)
{
if ((String)Session["Ativa"] == null)
{
return(RedirectToAction("Login", "ControleAcesso"));
}
if (file == null)
{
ModelState.AddModelError("", OdontoWeb_Resources.ResourceManager.GetString("M0019", CultureInfo.CurrentCulture));
return(RedirectToAction("VoltarAnexoAgenda"));
}
AGENDA item = baseApp.GetById((Int32)Session["IdVolta"]);
USUARIO usu = (USUARIO)Session["UserCredentials"];
var fileName = file.Name;
if (fileName.Length > 250)
{
ModelState.AddModelError("", OdontoWeb_Resources.ResourceManager.GetString("M0024", CultureInfo.CurrentCulture));
return(RedirectToAction("VoltarAnexoAgenda"));
}
String caminho = "/Imagens/" + ((Int32)Session["IdVolta"]).ToString() + "/Agenda/" + item.AGEN_CD_ID.ToString() + "/Anexos/";
String path = Path.Combine(Server.MapPath(caminho), fileName);
System.IO.Directory.CreateDirectory(Server.MapPath(caminho));
System.IO.File.WriteAllBytes(path, file.Contents);
//Recupera tipo de arquivo
extensao = Path.GetExtension(fileName);
String a = extensao;
// Gravar registro
AGENDA_ANEXO foto = new AGENDA_ANEXO();
foto.AGAN_AQ_ARQUIVO = "~" + caminho + fileName;
foto.AGAN_DT_ANEXO = DateTime.Today;
foto.AGAN_IN_ATIVO = 1;
Int32 tipo = 3;
if (extensao.ToUpper() == ".JPG" || extensao.ToUpper() == ".GIF" || extensao.ToUpper() == ".PNG" || extensao.ToUpper() == ".JPEG")
{
tipo = 1;
}
if (extensao.ToUpper() == ".MP4" || extensao.ToUpper() == ".AVI" || extensao.ToUpper() == ".MPEG")
{
tipo = 2;
}
if (extensao.ToUpper() == ".PDF")
{
tipo = 3;
}
foto.AGAN_IN_TIPO = tipo;
foto.AGAN_NM_TITULO = fileName;
foto.AGEN_CD_ID = item.AGEN_CD_ID;
item.AGENDA_ANEXO.Add(foto);
objetoAntes = item;
Int32 volta = baseApp.ValidateEdit(item, objetoAntes, usu);
return(RedirectToAction("VoltarAnexoAgenda"));
}
public FileQueueTests()
{
_directory = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetTempFileName()));
Directory.CreateDirectory(_directory);
_sut = new FileQueue(_directory, "Mepp");
}
public void DoSomethingWithRef(ref string[] firstRow, ref string[] secondRow)
{
var fq = new FileQueue();
fileParser.ParseFirstTwoRows(fq, ref firstRow, ref secondRow);
}
public void Initialize()
{
var path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
_testQueue = new FileQueue<string>(path);
}
public void MultiThreadTest()
{
QueueConfig config = new QueueConfig()
{
QueueDirectory = @"d:\workspace\data\test-queue",
DataConverter = new Utf8Converter(),
QueueName = "test02",
ReadBufferSize = 1024 * 1024
};
IFileQueue <string> fq = FileQueue <string> .Create(config);
HashSet <string> writeData = new HashSet <string>();
int successCount = 0;
int failCount = 0;
Thread[] ta = new Thread[400];
for (int i = 0; i < ta.Length; ++i)
{
if (i % 2 == 0)
{
ta[i] = new Thread(() =>
{
StringBuilder data = new StringBuilder();
for (int j = 0; j < 20; ++j)
{
data.Append(Guid.NewGuid().ToString());
}
lock (writeData)
{
writeData.Add(data.ToString());
}
fq.Enqueue(data.ToString());
});
}
else if (i % 2 == 1)
{
ta[i] = new Thread(() =>
{
while (true)
{
string data = fq.Dequeue();
if (data != null)
{
lock (writeData)
{
if (writeData.Contains(data))
{
writeData.Remove(data);
Interlocked.Increment(ref successCount);
}
else
{
Interlocked.Increment(ref failCount);
}
}
break;
}
}
});
}
}
for (int i = 0; i < ta.Length; ++i)
{
ta[i].Start();
}
while (true)
{
if (successCount + failCount == ta.Length / 2)
{
break;
}
Thread.Sleep(1000);
}
Assert.AreEqual(successCount, ta.Length / 2);
}
public MessagingClient(MessagingClientConfiguration configuration)
{
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
_fileQueue = new FileQueue(_configuration.QueueDirectory, _configuration.EndpointName);
_tcpClient = new TcpMessagingClient();
}
public LogController(FileQueue q)
{
_q = q;
}
public void MoveFileTest()
{
QueueConfig config = new QueueConfig()
{
QueueDirectory = @"d:\workspace\data\test-queue",
DataConverter = new Utf8Converter(),
MaxQueueSize = 1024 * 500,
QueueName = "test03",
ReadBufferSize = 1024 * 1024 * 3
};
IFileQueue <string> fq = FileQueue <string> .Create(config);
HashSet <string> writeData = new HashSet <string>();
int successCount = 0;
int failCount = 0;
int dataCount = 20000;
Thread t1 = new Thread(() =>
{
for (int i = 0; i < dataCount; ++i)
{
StringBuilder data = new StringBuilder();
for (int j = 0; j < 20; ++j)
{
data.Append(Guid.NewGuid().ToString());
}
lock (writeData)
{
writeData.Add(data.ToString());
}
fq.Enqueue(data.ToString());
}
});
Thread t2 = new Thread(() =>
{
for (int i = 0; i < dataCount; ++i)
{
while (true)
{
string data = fq.Dequeue();
if (data != null)
{
lock (writeData)
{
if (writeData.Contains(data))
{
writeData.Remove(data);
Interlocked.Increment(ref successCount);
}
else
{
Interlocked.Increment(ref failCount);
}
}
break;
}
}
}
});
t1.Start();
t2.Start();
while (true)
{
if (successCount + failCount == dataCount)
{
break;
}
Thread.Sleep(1000);
}
Assert.AreEqual(successCount, dataCount);
}
