feat: TestAdapter constraints

src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs

@@ -0,0 +1,18 @@
+using BenchmarkDotNet.Reports;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+    /// <summary>
+    /// Base <see cref="System.Attribute"/> for TestAdapter <see cref="Running.BenchmarkCase"/> constraints
+    /// </summary>
+    [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)]
+    public abstract class BenchmarkCaseConstraintAttribute : System.Attribute
+    {
+        /// <summary>
+        /// Validate the <see cref="Running.BenchmarkCase"/> constraint
+        /// </summary>
+        /// <param name="report"></param>
+        /// <param name="builder"></param>
+        protected internal abstract void Validate(BenchmarkReport report, System.Text.StringBuilder builder);
+    }
+}

src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs

@@ -0,0 +1,37 @@
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+    /// <summary>
+    /// Comparison Operators
+    /// </summary>
+    public enum ComparisonOperator
+    {
+        /// <summary>
+        /// Equal comparison operator
+        /// </summary>
+        Equal,
+        /// <summary>
+        /// Not Equal comparison operator
+        /// </summary>
+        NotEqual,
+        /// <summary>
+        ///  comparison operator
+        /// </summary>
+        Greater,
+        /// <summary>
+        ///  Greater than comparison operator
+        /// </summary>
+        GreaterOrEqual,
+        /// <summary>
+        ///  Greater or Equal than comparison operator
+        /// </summary>
+        Less,
+        /// <summary>
+        /// Less than comparison operator
+        /// </summary>
+        LessOrEqual,
+        /// <summary>
+        /// Less or equal than comparison operator
+        /// </summary>
+        Between
+    }
+}

src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs

@@ -0,0 +1,95 @@
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+    /// <summary>
+    /// <see cref="Running.BenchmarkCase"/> GC Gen Collections constraints
+    /// </summary>
+    public sealed class GCGenCollectionsConstraintAttribute : BenchmarkCaseConstraintAttribute
+    {
+        /// <summary>
+        /// Instance new <see cref="GCGenCollectionsConstraintAttribute"/>
+        /// </summary>
+        /// <param name="generation"></param>
+        /// <param name="operator"></param>
+        /// <param name="from"></param>
+        /// <param name="to"></param>
+        public GCGenCollectionsConstraintAttribute(GCGeneration generation, ComparisonOperator @operator, int from, int? to)
+        {
+            Operator = @operator;
+            From = from;
+            To = to;
+            Generation = generation;
+        }
+
+        /// <summary>
+        /// Instance new <see cref="GCGenCollectionsConstraintAttribute"/>
+        /// </summary>
+        /// <param name="generation"></param>
+        /// <param name="operator"></param>
+        /// <param name="from"></param>
+        public GCGenCollectionsConstraintAttribute(GCGeneration generation, ComparisonOperator @operator, int from)
+            : this(generation, @operator, from, null)
+        {
+        }
+
+        /// <summary>
+        /// GC Generation
+        /// </summary>
+        public GCGeneration Generation { get; }
+
+        /// <summary>
+        /// Comparison Operator
+        /// </summary>
+        public ComparisonOperator Operator { get; }
+        /// <summary>
+        /// From mean value
+        /// </summary>
+        public int From { get; }
+        /// <summary>
+        /// To mean value
+        /// </summary>
+        public int? To { get; }
+
+        /// inheritdoc
+        protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+        {
+            var resultRuns = report.GetResultRuns();
+            var gcStats = report.GcStats;
+            var genCollections = Generation switch
+            {
+                GCGeneration.Gen0 => gcStats.Gen0Collections,
+                GCGeneration.Gen1 => gcStats.Gen1Collections,
+                GCGeneration.Gen2 => gcStats.Gen2Collections,
+                _ => throw new System.NotSupportedException(),
+            };
+            switch (Operator)
+            {
+                case ComparisonOperator.Equal when genCollections != From:
+                    builder.AppendLine($"{Generation} is not equal to expected value {From}");
+                    break;
+                case ComparisonOperator.NotEqual when genCollections == From:
+                    builder.AppendLine($"{Generation} is equal to expected value {From}");
+                    break;
+                case ComparisonOperator.Less when genCollections >= From:
+                    builder.AppendLine($"{Generation} is greater or equal that expected value {From}");
+                    break;
+                case ComparisonOperator.LessOrEqual when genCollections > From:
+                    builder.AppendLine($"{Generation} is greater  that expected value {From}");
+                    break;
+                case ComparisonOperator.Greater when genCollections <= From:
+                    builder.AppendLine($"{Generation} is less or equal that expected value {From}");
+                    break;
+                case ComparisonOperator.GreaterOrEqual when genCollections < From:
+                    builder.AppendLine($"{Generation} is lest that expected value {From}");
+                    break;
+                case ComparisonOperator.Between when genCollections < From && genCollections > To:
+                    builder.AppendLine($"{Generation} is not betwenn  expected value [{From}-{To}]");
+                    break;
+                default:
+                    break;
+            }
+        }
+    }
+}

src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs

@@ -0,0 +1,21 @@
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+    /// <summary>
+    /// GC Generation
+    /// </summary>
+    public enum GCGeneration: int
+    {
+        /// <summary>
+        /// GC Generation 0
+        /// </summary>
+        Gen0,
+        /// <summary>
+        /// GC Generation 1
+        /// </summary>
+        Gen1,
+        /// <summary>
+        /// GC Generation 2
+        /// </summary>
+        Gen2
+    }
+}

src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs

@@ -0,0 +1,83 @@
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+    /// <summary>
+    /// /// <see cref="Running.BenchmarkCase"/> mean constraints
+    /// </summary>
+    [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)]
+    public sealed class MeanConstraintAttribute : BenchmarkCaseConstraintAttribute
+    {
+        /// <summary>
+        /// Instance new <see cref="MeanConstraintAttribute"/>
+        /// </summary>
+        /// <param name="operator"></param>
+        /// <param name="from"></param>
+        /// <param name="to"></param>
+        public MeanConstraintAttribute(ComparisonOperator @operator, double from, double? to)
+        {
+            Operator = @operator;
+            From = from;
+            To = to;
+        }
+
+        /// <summary>
+        /// Instance new <see cref="MeanConstraintAttribute"/>
+        /// </summary>
+        /// <param name="operator"></param>
+        /// <param name="from"></param>
+        public MeanConstraintAttribute(ComparisonOperator @operator, double from) :
+            this(@operator, from, null)
+        {
+        }
+
+        /// <summary>
+        /// Comparison Operator
+        /// </summary>
+        public ComparisonOperator Operator { get; }
+        /// <summary>
+        /// From mean value
+        /// </summary>
+        public double From { get; }
+        /// <summary>
+        /// To mean value
+        /// </summary>
+        public double? To { get; }
+
+        /// inheritdoc
+        protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+        {
+            var resultRuns = report.GetResultRuns();
+            var statistics = resultRuns.GetStatistics();
+            switch (Operator)
+            {
+                case ComparisonOperator.Equal when statistics.Mean != From:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is not equal to expected value {From}");
+                    break;
+                case ComparisonOperator.NotEqual when statistics.Mean == From:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is equal to expected value {From}");
+                    break;
+                case ComparisonOperator.Less when statistics.Mean >= From:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is greater or equal that expected value {From}");
+                    break;
+                case ComparisonOperator.LessOrEqual when statistics.Mean > From:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is greater  that expected value {From}");
+                    break;
+                case ComparisonOperator.Greater when statistics.Mean <= From:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is less or equal that expected value {From}");
+                    break;
+                case ComparisonOperator.GreaterOrEqual when statistics.Mean < From:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is lest that expected value {From}");
+                    break;
+                case ComparisonOperator.Between when statistics.Mean < From && statistics.Mean > To:
+                    builder.AppendLine($"{nameof(statistics.Mean)} is not between expected value [{From}-{To}]");
+                    break;
+                default:
+                    break;
+            }
+
+        }
+    }
+}

src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs

@@ -0,0 +1,82 @@
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+    /// <summary>
+    /// <see cref="Running.BenchmarkCase"/> standard deviation constraints
+    /// </summary>
+    public sealed class StdDevConstraintAttribute : BenchmarkCaseConstraintAttribute
+    {
+        /// <summary>
+        /// Instance new <see cref="StdDevConstraintAttribute"/>
+        /// </summary>
+        /// <param name="operator"></param>
+        /// <param name="from"></param>
+        /// <param name="to"></param>
+        public StdDevConstraintAttribute(ComparisonOperator @operator, double from, double? to)
+        {
+            Operator = @operator;
+            From = from;
+            To = to;
+        }
+
+        /// <summary>
+        /// Instance new <see cref="StdDevConstraintAttribute"/>
+        /// </summary>
+        /// <param name="operator"></param>
+        /// <param name="from"></param>
+        public StdDevConstraintAttribute(ComparisonOperator @operator, double from) :
+            this(@operator, from, null)
+        {
+        }
+
+        /// <summary>
+        /// Comparison Operator
+        /// </summary>
+        public ComparisonOperator Operator { get; }
+        /// <summary>
+        /// From mean value
+        /// </summary>
+        public double From { get; }
+        /// <summary>
+        /// To mean value
+        /// </summary>
+        public double? To { get; }
+
+        /// inheritdoc
+        protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+        {
+            var resultRuns = report.GetResultRuns();
+            var statistics = resultRuns.GetStatistics();
+            var stdDev = statistics.StandardDeviation;
+            switch (Operator)
+            {
+                case ComparisonOperator.Equal when stdDev != From:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is not equal to expected value {From}");
+                    break;
+                case ComparisonOperator.NotEqual when stdDev == From:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is equal to expected value {From}");
+                    break;
+                case ComparisonOperator.Less when stdDev >= From:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is greater or equal that expected value {From}");
+                    break;
+                case ComparisonOperator.LessOrEqual when stdDev > From:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is greater  that expected value {From}");
+                    break;
+                case ComparisonOperator.Greater when stdDev <= From:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is less or equal that expected value {From}");
+                    break;
+                case ComparisonOperator.GreaterOrEqual when stdDev < From:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is lest that expected value {From}");
+                    break;
+                case ComparisonOperator.Between when stdDev < From && stdDev > To:
+                    builder.AppendLine($"{nameof(statistics.StandardDeviation)} is not between expected value [{From}-{To}]");
+                    break;
+                default:
+                    break;
+            }
+        }
+    }
+}