private static void Main()
{
using (
var client = new HttpClient {BaseAddress = new Uri(ConfigurationManager.AppSettings["VSOnlineBaseUrl"])}
)
{
var history = new Historian(
new WorkItemRepository(new TfsConnection(ConfigurationManager.AppSettings["username"],
ConfigurationManager.AppSettings["password"], client)));
var burnup = history.GetBurnUpDataSince(new DateTime(2014, 7, 7, 23, 59, 59), @"BPS.Scrum\Dev -SEP Project");
using (var writer = new StreamWriter(@".\completed" + DateTime.Now.ToString("yyyymmmmdd") + ".xls"))
{
var csvWriter = new CsvWriter(writer, new CsvConfiguration { Delimiter = "\t" });
csvWriter.WriteRecords(burnup.First(s => s.Title == "Completed").Data);
}
using (var writer = new StreamWriter(@".\requested" + DateTime.Now.ToString("yyyymmmmdd") + ".xls"))
{
var csvWriter = new CsvWriter(writer, new CsvConfiguration { Delimiter = "\t" });
csvWriter.WriteRecords(burnup.First(s => s.Title == "Requested").Data);
}
}
}
public TrendingDataSet getTrendsforChannelIDDate(string ChannelID, string targetDate)
{
//DateTime epoch = new DateTime(1970, 1, 1);
//string theSproc = "dbo.selectTrendingDataByChannelIDDate2";
//DataSet dataSet = new DataSet();
//TrendingDataSet trendingDataSet = new TrendingDataSet();
//using (SqlConnection connection = new SqlConnection(connectionstring))
//using (SqlCommand command = connection.CreateCommand())
//using (SqlDataAdapter adapter = new SqlDataAdapter(command))
//{
// command.CommandText = theSproc;
// command.CommandType = CommandType.StoredProcedure;
// command.Parameters.AddWithValue("@EventDate", targetDate);
// command.Parameters.AddWithValue("@ChannelID", ChannelID);
// command.CommandTimeout = 300;
// connection.Open();
// adapter.Fill(dataSet);
// trendingDataSet.ChannelData = dataSet.Tables[0].Rows
// .Cast<DataRow>()
// .Select(row => new TrendingDataPoint()
// {
// Time = row.Field<DateTime>("thedate").Subtract(epoch).TotalMilliseconds,
// Maximum = row.Field<double>("themaximum"),
// Minimum = row.Field<double>("theminimum"),
// Average = row.Field<double>("theaverage")
// })
// .ToArray();
// trendingDataSet.AlarmLimits = dataSet.Tables[1].Rows
// .Cast<DataRow>()
// .Select(row => new TrendingAlarmLimit()
// {
// TimeStart = row.Field<DateTime>("thedatefrom").Subtract(epoch).TotalMilliseconds,
// TimeEnd = row.Field<DateTime>("thedateto").Subtract(epoch).TotalMilliseconds,
// High = row.Field<double?>("alarmlimithigh"),
// Low = row.Field<double?>("alarmlimitlow")
// })
// .ToArray();
// trendingDataSet.OffNormalLimits = dataSet.Tables[2].Rows
// .Cast<DataRow>()
// .Select(row => new TrendingAlarmLimit()
// {
// TimeStart = row.Field<DateTime>("thedatefrom").Subtract(epoch).TotalMilliseconds,
// TimeEnd = row.Field<DateTime>("thedateto").Subtract(epoch).TotalMilliseconds,
// High = row.Field<double?>("offlimithigh"),
// Low = row.Field<double?>("offlimitlow")
// })
// .ToArray();
//}
string historianServer;
string historianInstance;
IEnumerable<int> channelIDs = new List<int>() { Convert.ToInt32(ChannelID) };
DateTime startDate = Convert.ToDateTime(targetDate);
DateTime endDate = startDate.AddDays(1);
TrendingDataSet trendingDataSet = new TrendingDataSet();
DateTime epoch = new DateTime(1970, 1, 1);
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
historianServer = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'Historian.Server'") ?? "127.0.0.1";
historianInstance = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'Historian.Instance'") ?? "XDA";
using (Historian historian = new Historian(historianServer, historianInstance))
{
foreach (openHistorian.XDALink.TrendingDataPoint point in historian.Read(channelIDs, startDate, endDate))
{
if (!trendingDataSet.ChannelData.Exists(x => x.Time == point.Timestamp.Subtract(epoch).TotalMilliseconds))
{
trendingDataSet.ChannelData.Add(new TrendingDataDatum());
trendingDataSet.ChannelData[trendingDataSet.ChannelData.Count - 1].Time = point.Timestamp.Subtract(epoch).TotalMilliseconds;
}
if (point.SeriesID.ToString() == "Average")
trendingDataSet.ChannelData[trendingDataSet.ChannelData.IndexOf(x => x.Time == point.Timestamp.Subtract(epoch).TotalMilliseconds)].Average = point.Value;
else if (point.SeriesID.ToString() == "Minimum")
trendingDataSet.ChannelData[trendingDataSet.ChannelData.IndexOf(x => x.Time == point.Timestamp.Subtract(epoch).TotalMilliseconds)].Minimum = point.Value;
else if (point.SeriesID.ToString() == "Maximum")
trendingDataSet.ChannelData[trendingDataSet.ChannelData.IndexOf(x => x.Time == point.Timestamp.Subtract(epoch).TotalMilliseconds)].Maximum = point.Value;
}
}
IEnumerable<DataRow> table = Enumerable.Empty<DataRow>();
table = connection.RetrieveData(" Select {0} AS thedatefrom, " +
" DATEADD(DAY, 1, {0}) AS thedateto, " +
" CASE WHEN AlarmRangeLimit.PerUnit <> 0 AND Channel.PerUnitValue IS NOT NULL THEN AlarmRangeLimit.High * PerUnitValue ELSE AlarmRangeLimit.High END AS alarmlimithigh," +
" CASE WHEN AlarmRangeLimit.PerUnit <> 0 AND Channel.PerUnitValue IS NOT NULL THEN AlarmRangeLimit.Low * PerUnitValue ELSE AlarmRangeLimit.Low END AS alarmlimitlow " +
" FROM AlarmRangeLimit JOIN " +
" Channel ON AlarmRangeLimit.ChannelID = Channel.ID " +
"WHERE AlarmRangeLimit.AlarmTypeID = (SELECT ID FROM AlarmType where Name = 'Alarm') AND " +
" AlarmRangeLimit.ChannelID = {1}", startDate, Convert.ToInt32(ChannelID)).Select();
foreach (DataRow row in table)
{
trendingDataSet.AlarmLimits.Add(new TrendingAlarmLimit() { High = row.Field<double?>("alarmlimithigh"), Low = row.Field<double?>("alarmlimitlow"), TimeEnd = row.Field<DateTime>("thedateto").Subtract(epoch).TotalMilliseconds, TimeStart = row.Field<DateTime>("thedatefrom").Subtract(epoch).TotalMilliseconds });
}
table = Enumerable.Empty<DataRow>();
table = connection.RetrieveData(" DECLARE @dayOfWeek INT = DATEPART(DW, {0}) - 1 " +
" DECLARE @hourOfWeek INT = @dayOfWeek * 24 " +
" ; WITH HourlyIndex AS" +
" ( " +
" SELECT @hourOfWeek AS HourOfWeek " +
" UNION ALL " +
" SELECT HourOfWeek + 1 " +
" FROM HourlyIndex" +
" WHERE (HourOfWeek + 1) < @hourOfWeek + 24" +
" ) " +
" SELECT " +
" DATEADD(HOUR, HourlyIndex.HourOfWeek - @hourOfWeek, {0}) AS thedatefrom, " +
" DATEADD(HOUR, HourlyIndex.HourOfWeek - @hourOfWeek + 1, {0}) AS thedateto, " +
" HourOfWeekLimit.High AS offlimithigh, " +
" HourOfWeekLimit.Low AS offlimitlow " +
" FROM " +
" HourlyIndex LEFT OUTER JOIN " +
" HourOfWeekLimit ON HourOfWeekLimit.HourOfWeek = HourlyIndex.HourOfWeek " +
" WHERE " +
" HourOfWeekLimit.ChannelID IS NULL OR " +
" HourOfWeekLimit.ChannelID = {1} ", startDate, Convert.ToInt32(ChannelID)).Select();
foreach (DataRow row in table)
{
trendingDataSet.OffNormalLimits.Add(new TrendingAlarmLimit() { High = row.Field<double?>("offlimithigh"), Low = row.Field<double?>("offlimitlow"), TimeEnd = row.Field<DateTime>("thedateto").Subtract(epoch).TotalMilliseconds, TimeStart = row.Field<DateTime>("thedatefrom").Subtract(epoch).TotalMilliseconds });
}
}
return trendingDataSet;
}
private List<List<TrendingDataLocation>> GetFramesFromHistorian(ContourQuery contourQuery)
{
DataTable idTable;
string historianServer;
string historianInstance;
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
string query =
"SELECT " +
" Channel.ID AS ChannelID, " +
" Meter.ID AS MeterID, " +
" Meter.Name AS MeterName, " +
" MeterLocation.Latitude, " +
" MeterLocation.Longitude, " +
" Channel.PerUnitValue " +
"FROM " +
" Meter JOIN " +
" MeterLocation ON Meter.MeterLocationID = MeterLocation.ID LEFT OUTER JOIN " +
" Channel ON " +
" Channel.MeterID = Meter.ID AND " +
" Channel.ID IN (SELECT ChannelID FROM ContourChannel WHERE ContourColorScaleName = {1}) " +
"WHERE Meter.ID IN (SELECT * FROM authMeters({0}))";
idTable = connection.RetrieveData(query, contourQuery.UserName, contourQuery.ColorScaleName);
historianServer = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'Historian.Server'") ?? "127.0.0.1";
historianInstance = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'Historian.Instance'") ?? "XDA";
}
if (!string.IsNullOrEmpty(contourQuery.Meters))
{
const int byteSize = 8;
// Meter selections are stored as a base-64 string without padding, using '-' instead of '+' and '_' instead of '/'
string padding = "A==".Remove(3 - (contourQuery.Meters.Length + 3) % 4);
string base64 = contourQuery.Meters.Replace('-', '+').Replace('_', '/') + padding;
byte[] meterSelections = Convert.FromBase64String(base64);
// The resulting byte array is a simple set of bitflags ordered by meter ID and packed into the most significant bits.
// In order to properly interpret the bytes, we must first group and order the data by meter ID to determine the location
// of each meter's bitflag. Then we can filter out the unwanted data from the original table of IDs
idTable.Select()
.Select((Row, Index) => new { Row, Index })
.GroupBy(obj => obj.Row.ConvertField<int>("MeterID"))
.OrderBy(grouping => grouping.Key)
.Where((grouping, index) => (meterSelections[index / byteSize] & (0x80 >> (index % byteSize))) == 0)
.SelectMany(grouping => grouping)
.OrderByDescending(obj => obj.Index)
.ToList()
.ForEach(obj => idTable.Rows.RemoveAt(obj.Index));
}
List<DataRow> meterRows = idTable
.Select()
.DistinctBy(row => row.ConvertField<int>("MeterID"))
.ToList();
DateTime startDate = contourQuery.GetStartDate();
DateTime endDate = contourQuery.GetEndDate();
int stepSize = contourQuery.StepSize;
// The frames to be included are those whose timestamps fall
// within the range which is specified by startDate and
// endDate. We start by aligning startDate and endDate with
// the nearest frame timestamps which fall within that range
int startTimeOffset = (int)Math.Ceiling((startDate - startDate.Date).TotalMinutes / stepSize);
startDate = startDate.Date.AddMinutes(startTimeOffset * stepSize);
int endTimeOffset = (int)Math.Floor((endDate - endDate.Date).TotalMinutes / stepSize);
endDate = endDate.Date.AddMinutes(endTimeOffset * stepSize);
// Since each frame includes data from all timestamps between
// the previous frame's timestamp and its own timestamp, we
// must include one additional frame of data before startDate
startDate = startDate.AddMinutes(-stepSize);
int frameCount = (int)((endDate - startDate).TotalMinutes / stepSize);
List<Dictionary<int, TrendingDataLocation>> frames = Enumerable.Repeat(meterRows, frameCount)
.Select(rows => rows.Select(row => new TrendingDataLocation()
{
id = row.ConvertField<int>("MeterID"),
name = row.ConvertField<string>("MeterName"),
Latitude = row.ConvertField<double>("Latitude"),
Longitude = row.ConvertField<double>("Longitude")
}))
.Select(locations => locations.ToDictionary(location => location.id))
.ToList();
Dictionary<int, double?> nominalLookup = idTable
.Select("ChannelID IS NOT NULL")
.ToDictionary(row => row.ConvertField<int>("ChannelID"), row => row.ConvertField<double?>("PerUnitValue"));
Dictionary<int, List<TrendingDataLocation>> lookup = idTable
.Select("ChannelID IS NOT NULL")
.Select(row =>
{
int meterID = row.ConvertField<int>("MeterID");
return new
{
ChannelID = row.ConvertField<int>("ChannelID"),
Frames = frames.Select(locationLookup => locationLookup[meterID]).ToList()
};
})
.ToDictionary(obj => obj.ChannelID, obj => obj.Frames);
using (Historian historian = new Historian(historianServer, historianInstance))
{
foreach (TrendingDataPoint point in historian.Read(lookup.Keys, startDate, endDate))
{
List<TrendingDataLocation> locations = lookup[point.ChannelID];
// Use ceiling to sort data into the next nearest frame.
// Subtract 1 because startDate was shifted to include one additional frame of data
int frameIndex = (int)Math.Ceiling((point.Timestamp - startDate).TotalMinutes / stepSize) - 1;
if (frameIndex < 0 || frameIndex >= locations.Count)
continue;
TrendingDataLocation frame = locations[frameIndex];
double nominal = nominalLookup[point.ChannelID] ?? 1.0D;
double value = point.Value / nominal;
switch (point.SeriesID)
{
case SeriesID.Minimum:
frame.Minimum = Math.Min(value, frame.Minimum ?? value);
break;
case SeriesID.Maximum:
frame.Maximum = Math.Max(value, frame.Maximum ?? value);
break;
case SeriesID.Average:
frame.Aggregate(value);
frame.Average = frame.GetAverage();
break;
}
}
}
return frames
.Select(frame => frame.Values.ToList())
.ToList();
}
void SaveHistorian(Historian historian, bool isNew)
{
bool continueSave = true;
if (!isNew && (historian.TypeName != "HistorianAdapters.LocalOutputAdapter" || !historian.IsLocal))
{
SystemMessages sm = new SystemMessages(new Message() { UserMessage = "You are changing your historian type.", SystemMessage = "You are changing your historian type from an in-process local historian to another historian provider. Please note that once the changes are applied, any customizations you may have made to the in-process local historian in the openPDC configuration file will be lost." + Environment.NewLine + "Do you want to continue?", UserMessageType = MessageType.Confirmation }, ButtonType.YesNo);
sm.Closed += new EventHandler(delegate(object popupWindow, EventArgs eargs)
{
if ((bool)sm.DialogResult)
continueSave = true;
else
continueSave = false;
});
sm.ShowPopup();
}
if (continueSave)
m_client.SaveHistorianAsync(historian, isNew);
}
public void AddNewHistorian(Historian historian)
{
DataContext.Table <Historian>().AddNewRecord(historian);
}
private List<List<TrendingDataLocation>> GetFramesFromHistorian(ContourQuery contourQuery)
{
DataTable idTable;
string historianServer;
string historianInstance;
using (AdoDataConnection connection = new AdoDataConnection(connectionstring, typeof(SqlConnection), typeof(SqlDataAdapter)))
{
string query =
"SELECT " +
" Channel.ID AS ChannelID, " +
" Meter.ID AS MeterID, " +
" Meter.Name AS MeterName, " +
" MeterLocation.Latitude, " +
" MeterLocation.Longitude, " +
" Channel.PerUnitValue " +
"FROM " +
" Meter JOIN " +
" MeterLocation ON Meter.MeterLocationID = MeterLocation.ID LEFT OUTER JOIN " +
" Channel ON " +
" Channel.MeterID = Meter.ID AND " +
" Channel.ID IN (SELECT ChannelID FROM ContourChannel WHERE ContourColorScaleName = {1}) " +
"WHERE " +
" Meter.ID IN (SELECT * FROM authMeters({0}))";
idTable = connection.RetrieveData(query, contourQuery.UserName, contourQuery.ColorScaleName);
historianServer = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'Historian.Server'") ?? "127.0.0.1";
historianInstance = connection.ExecuteScalar<string>("SELECT Value FROM Setting WHERE Name = 'Historian.Instance'") ?? "XDA";
}
List<DataRow> meterRows = idTable
.Select()
.DistinctBy(row => row.ConvertField<int>("MeterID"))
.ToList();
DateTime startDate = contourQuery.GetStartDate();
DateTime endDate = contourQuery.GetEndDate();
int stepSize = contourQuery.StepSize;
// The frames to be included are those whose timestamps fall
// within the range which is specified by startDate and
// endDate. We start by aligning startDate and endDate with
// the nearest frame timestamps which fall within that range
int startTimeOffset = (int)Math.Ceiling((startDate - startDate.Date).TotalMinutes / stepSize);
startDate = startDate.Date.AddMinutes(startTimeOffset * stepSize);
int endTimeOffset = (int)Math.Floor((endDate - endDate.Date).TotalMinutes / stepSize);
endDate = endDate.Date.AddMinutes(endTimeOffset * stepSize);
// Since each frame includes data from all timestamps between
// the previous frame's timestamp and its own timestamp, we
// must include one additional frame of data before startDate
startDate = startDate.AddMinutes(-stepSize);
int frameCount = (int)((endDate - startDate).TotalMinutes / stepSize);
List<Dictionary<int, TrendingDataLocation>> frames = Enumerable.Repeat(meterRows, frameCount)
.Select(rows => rows.Select(row => new TrendingDataLocation()
{
id = row.ConvertField<int>("MeterID"),
name = row.ConvertField<string>("MeterName"),
Latitude = row.ConvertField<double>("Latitude"),
Longitude = row.ConvertField<double>("Longitude")
}))
.Select(locations => locations.ToDictionary(location => location.id))
.ToList();
Dictionary<int, double?> nominalLookup = idTable
.Select("ChannelID IS NOT NULL")
.ToDictionary(row => row.ConvertField<int>("ChannelID"), row => row.ConvertField<double?>("PerUnitValue"));
Dictionary<int, List<TrendingDataLocation>> lookup = idTable
.Select("ChannelID IS NOT NULL")
.Select(row =>
{
int meterID = row.ConvertField<int>("MeterID");
return new
{
ChannelID = row.ConvertField<int>("ChannelID"),
Frames = frames.Select(locationLookup => locationLookup[meterID]).ToList()
};
})
.ToDictionary(obj => obj.ChannelID, obj => obj.Frames);
using (Historian historian = new Historian(historianServer, historianInstance))
{
foreach (TrendingDataPoint point in historian.Read(lookup.Keys, startDate, endDate))
{
List<TrendingDataLocation> locations = lookup[point.ChannelID];
// Use ceiling to sort data into the next nearest frame.
// Subtract 1 because startDate was shifted to include one additional frame of data
int frameIndex = (int)Math.Ceiling((point.Timestamp - startDate).TotalMinutes / stepSize) - 1;
if (frameIndex < 0 || frameIndex >= locations.Count)
continue;
TrendingDataLocation frame = locations[frameIndex];
double nominal = nominalLookup[point.ChannelID] ?? 1.0D;
double value = point.Value / nominal;
switch (point.SeriesID)
{
case SeriesID.Minimum:
frame.Minimum = Math.Min(value, frame.Minimum ?? value);
break;
case SeriesID.Maximum:
frame.Maximum = Math.Max(value, frame.Maximum ?? value);
break;
case SeriesID.Average:
frame.Aggregate(value);
frame.Average = frame.GetAverage();
break;
}
}
}
return frames
.Select(frame => frame.Values.ToList())
.ToList();
}
private void StrategyAsGetCurrent(string fileName)
{
Storage.Batch(accessor =>
{
var conflict = accessor.GetConfigurationValue <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile(fileName));
var localMetadata = accessor.GetFile(fileName, 0, 0).Metadata;
var localHistory = Historian.DeserializeHistory(localMetadata);
// incorporate remote version history into local
foreach (var remoteHistoryItem in conflict.RemoteHistory.Where(remoteHistoryItem => !localHistory.Contains(remoteHistoryItem)))
{
localHistory.Add(remoteHistoryItem);
}
localMetadata[SynchronizationConstants.RavenSynchronizationHistory] = Historian.SerializeHistory(localHistory);
accessor.UpdateFileMetadata(fileName, localMetadata);
ConflictArtifactManager.Delete(fileName, accessor);
Publisher.Publish(new ConflictResolvedNotification {
FileName = fileName
});
});
}
public async Task <HttpResponseMessage> MultipartProceed()
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var tempFileName = RavenFileNameHelper.DownloadingFileName(fileName);
var sourceServerInfo = InnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues("ETag").First().Trim('\"'));
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders); // InnerHeaders.FilterHeadersToObject();
var localMetadata = GetLocalMetadata(fileName);
if (localMetadata != null)
{
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, fileName);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWritting(Storage, Search, StorageOperationsTask,
tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(fileName);
accessor.RenameFile(tempFileName, fileName);
Search.Delete(tempFileName);
Search.Index(fileName, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes was updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes was updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(fileName);
Publisher.Publish(new ConflictResolvedNotification {
FileName = fileName
});
}
}
catch (Exception ex)
{
report.Exception = ex;
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
public HttpResponseMessage Delete(string fileName)
{
var sourceServerInfo = InnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues("ETag").First().Trim('\"'));
Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.Delete);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
var localMetadata = GetLocalMetadata(fileName);
if (localMetadata != null)
{
// REVIEW: Use InnerHeaders for consistency?
var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject();
bool isConflictResolved;
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(fileName);
var tombstoneMetadata = new RavenJObject
{
{
SynchronizationConstants.RavenSynchronizationHistory,
localMetadata[SynchronizationConstants.RavenSynchronizationHistory]
},
{
SynchronizationConstants.RavenSynchronizationVersion,
localMetadata[SynchronizationConstants.RavenSynchronizationVersion]
},
{
SynchronizationConstants.RavenSynchronizationSource,
localMetadata[SynchronizationConstants.RavenSynchronizationSource]
}
}.WithDeleteMarker();
Historian.UpdateLastModified(tombstoneMetadata);
accessor.PutFile(fileName, 0, tombstoneMetadata, true);
});
PublishFileNotification(fileName, FileChangeAction.Delete);
}
}
catch (Exception ex)
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
finally
{
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
}
if (report.Exception == null)
{
Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo);
}
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
public HttpResponseMessage UpdateMetadata(string fileName)
{
var sourceServerInfo = InnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
// REVIEW: (Oren) It works, but it seems to me it is not an scalable solution.
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues("ETag").First().Trim('\"'));
Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName,
sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.MetadataUpdate);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
var localMetadata = GetLocalMetadata(fileName);
var sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders);
bool isConflictResolved;
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Historian.UpdateLastModified(sourceMetadata);
Storage.Batch(accessor => accessor.UpdateFileMetadata(fileName, sourceMetadata));
Search.Index(fileName, sourceMetadata);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(fileName);
Publisher.Publish(new ConflictResolvedNotification {
FileName = fileName
});
}
PublishFileNotification(fileName, FileChangeAction.Update);
}
catch (Exception ex)
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occured during metadata synchronization of file '{0}' from {1}", fileName,
sourceServerInfo), ex);
}
finally
{
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
}
if (report.Exception == null)
{
Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo);
}
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
public async Task <HttpResponseMessage> Put(string name, string uploadId = null, bool preserveTimestamps = false)
{
try
{
FileSystem.MetricsCounters.FilesPerSecond.Mark();
name = FileHeader.Canonize(name);
var headers = this.GetFilteredMetadataFromHeaders(ReadInnerHeaders);
if (preserveTimestamps)
{
if (!headers.ContainsKey(Constants.RavenCreationDate))
{
if (headers.ContainsKey(Constants.CreationDate))
{
headers[Constants.RavenCreationDate] = headers[Constants.CreationDate];
}
else
{
throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header.");
}
}
var lastModified = GetHeader(Constants.RavenLastModified);
if (lastModified != null)
{
DateTimeOffset when;
if (!DateTimeOffset.TryParse(lastModified, out when))
{
when = DateTimeOffset.UtcNow;
}
Historian.UpdateLastModified(headers, when);
}
else
{
Historian.UpdateLastModified(headers);
}
}
else
{
headers[Constants.RavenCreationDate] = DateTimeOffset.UtcNow;
Historian.UpdateLastModified(headers);
}
// TODO: To keep current filesystems working. We should remove when adding a new migration.
headers[Constants.CreationDate] = headers[Constants.RavenCreationDate].Value <DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture);
Historian.Update(name, headers);
SynchronizationTask.Cancel(name);
long?size = -1;
ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor =>
{
AssertPutOperationNotVetoed(name, headers);
AssertFileIsNotBeingSynced(name, accessor, true);
var contentLength = Request.Content.Headers.ContentLength;
var sizeHeader = GetHeader("RavenFS-size");
long sizeForParse;
if (contentLength == 0 || long.TryParse(sizeHeader, out sizeForParse) == false)
{
size = contentLength;
if (Request.Headers.TransferEncodingChunked ?? false)
{
size = null;
}
}
else
{
size = sizeForParse;
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, headers));
using (FileSystem.DisableAllTriggersForCurrentThread())
{
StorageOperationsTask.IndicateFileToDelete(name);
}
accessor.PutFile(name, size, headers);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, headers));
Search.Index(name, headers);
}));
log.Debug("Inserted a new file '{0}' with ETag {1}", name, headers.Value <Guid>(Constants.MetadataEtagField));
using (var contentStream = await Request.Content.ReadAsStreamAsync())
using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, headers))
{
await readFileToDatabase.Execute();
if (readFileToDatabase.TotalSizeRead != size)
{
StorageOperationsTask.IndicateFileToDelete(name);
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
if (!preserveTimestamps)
{
Historian.UpdateLastModified(headers); // update with the final file size.
}
log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name);
headers["Content-MD5"] = readFileToDatabase.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(name, headers));
int totalSizeRead = readFileToDatabase.TotalSizeRead;
headers["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture);
Search.Index(name, headers);
Publisher.Publish(new FileChangeNotification {
Action = FileChangeAction.Add, File = FilePathTools.Cannoicalise(name)
});
log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, headers.Value <Guid>(Constants.MetadataEtagField));
StartSynchronizeDestinationsInBackground();
}
}
catch (Exception ex)
{
if (uploadId != null)
{
Guid uploadIdentifier;
if (Guid.TryParse(uploadId, out uploadIdentifier))
{
Publisher.Publish(new CancellationNotification {
UploadId = uploadIdentifier, File = name
});
}
}
log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex);
var concurrencyException = ex as ConcurrencyException;
if (concurrencyException != null)
{
throw ConcurrencyResponseException(concurrencyException);
}
throw;
}
return(GetEmptyMessage(HttpStatusCode.Created));
}
