Update BDN tests base to only run framework that is current instead of both 8.0 and 9.0 each time (#1159)

* Trying to scrub out framework (net8.0 or .net9.0) from the results file that is part of that specific test run. DEbug info in there now as it will only run one test.

* Remove debug code so ready to try full test run.

* Will just try this out in the pipeline

* Updated BDN Base Config to only run one Framework (current framework) instead of both net8.0 and net.9.0 each time. Also removed the code from the test script that was scrubbing out the data from the log file as that is no longer needed.

* Add ability for user to pass framework as an argument  ... but if not passed, then it does both net80 and net90

* Got BDN work with ablity to send framework want to test on and if not specified then it will run on both .net80 and .net90

* Changed it to use a BDNRUNPARAM env var to pass what framework want to run BDN tests under. If don't specify the env var, then it will run both net8.0 and net9.0

* Formatting check failure ... looks like from extra spaces had after a line

* minor reorg of code

---------

Co-authored-by: Badrish Chandramouli <badrishc@microsoft.com>

Author: darrenge

.github/workflows/ci-bdnbenchmark.yml

@@ -56,7 +56,7 @@ jobs:
         run: dotnet restore
 
       - name: Run BDN.benchmark Perf Test
-        run: ./test/BDNPerfTests/run_bdnperftest.ps1 ${{ matrix.test }}
+        run: ./test/BDNPerfTests/run_bdnperftest.ps1 ${{ matrix.test }} ${{ matrix.framework }}
         shell: pwsh
         continue-on-error: false
 

benchmark/BDN.benchmark/Program.cs

@@ -25,21 +25,37 @@ public class BaseConfig : ManualConfig
 
     public BaseConfig()
     {
-        AddLogger(ConsoleLogger.Default);
-        AddExporter(DefaultExporters.Markdown);
-        AddColumnProvider(DefaultColumnProviders.Instance);
-        WithSummaryStyle(SummaryStyle.Default.WithSizeUnit(SizeUnit.B));
+        _ = AddLogger(ConsoleLogger.Default);
+        _ = AddExporter(DefaultExporters.Markdown);
+        _ = AddColumnProvider(DefaultColumnProviders.Instance);
+        _ = WithSummaryStyle(SummaryStyle.Default.WithSizeUnit(SizeUnit.B));
 
         var baseJob = Job.Default.WithGcServer(true);
 
-        Net8BaseJob = baseJob.WithRuntime(CoreRuntime.Core80)
+        Net8BaseJob = baseJob
+            .WithRuntime(CoreRuntime.Core80)
             .WithEnvironmentVariables(new EnvironmentVariable("DOTNET_TieredPGO", "0"));
-        Net9BaseJob = baseJob.WithRuntime(CoreRuntime.Core90)
+        Net9BaseJob = baseJob
+            .WithRuntime(CoreRuntime.Core90)
             .WithEnvironmentVariables(new EnvironmentVariable("DOTNET_TieredPGO", "0"));
 
-        AddJob(
-            Net8BaseJob.WithId(".NET 8"),
-            Net9BaseJob.WithId(".NET 9")
-            );
+        // Get value of environment variable BDNRUNPARAM - determines if running net8.0, net9.0 or both (if env var is not set or invalid)
+        var bdnRunParam = Environment.GetEnvironmentVariable("BDNRUNPARAM");
+
+        switch (bdnRunParam)
+        {
+            case "net8.0":
+                _ = AddJob(Net8BaseJob.WithId(".NET 8"));
+                break;
+            case "net9.0":
+                _ = AddJob(Net9BaseJob.WithId(".NET 9"));
+                break;
+            default:
+                _ = AddJob(
+                    Net8BaseJob.WithId(".NET 8"),
+                    Net9BaseJob.WithId(".NET 9")
+                    );
+                break;
+        }
     }
 }

test/BDNPerfTests/run_bdnperftest.ps1

@@ -15,10 +15,12 @@
     ./run_bdnperftest.ps1
     ./run_bdnperftest.ps1 BDN.benchmark.Operations.BasicOperations.*
     ./run_bdnperftest.ps1 Operations.BasicOperations    <-- can run this way but this is how specify in ci-bdnbenchmark.yml
+    ./run_bdnperftest.ps1 Operations.BasicOperations net9.0
 #>
 
 param (
-  [string]$currentTest = "BDN.benchmark.Operations.BasicOperations.*"
+  [string]$currentTest = "BDN.benchmark.Operations.BasicOperations.*",
+  [string]$framework = "net8.0"
 )
 
 $OFS = "`r`n"
@@ -57,7 +59,6 @@ function AnalyzeResult {
         return $false # the values are too different
     }
   }
- 
 
 ######### ParseValueFromResults ###########
 #
@@ -95,7 +96,6 @@ param ($ResultsLine, $columnNum)
 # Set all the config options
 $configFile = "BDN_Benchmark_Config.json"
 $configuration = "Release"
-$framework = "net8.0"
 $allocatedColumn = "-1"   # last one is allocated, just to ensure in case other column gets added
 $acceptableAllocatedRange = "10"   # percent allowed variance when comparing expected vs actual found value - same for linux and windows.
 
@@ -184,16 +184,16 @@ $resultsFileName = $currentTestStripped + "_" + $CurrentOS + ".results"
 $resultsFile = "$resultsDir/$resultsFileName"
 $BDNbenchmarkErrorFile = "$errorLogDir/$currentTestStripped" + "_StandardError_" +$CurrentOS+".log"
 $filter = $currentTest
+$exporter = "json" 
 
 Write-Output " "
-Write-Output "** Start:  dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath --exporters json > $resultsFile 2> $BDNbenchmarkErrorFile"
-dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath --exporters json  > $resultsFile 2> $BDNbenchmarkErrorFile
+Write-Output "** Start:  dotnet run -c $configuration -f $framework --project $BDNbenchmarkPath --filter $filter --exporters $exporter -e BDNRUNPARAM=$framework > $resultsFile 2> $BDNbenchmarkErrorFile"
+dotnet run -c $configuration -f $framework --project $BDNbenchmarkPath --filter $filter --exporters $exporter -e BDNRUNPARAM=$framework > $resultsFile 2> $BDNbenchmarkErrorFile
 
 Write-Output "** BDN Benchmark for $filter finished"
 Write-Output " "
 
 Write-Output "**** ANALYZE THE RESULTS FILE $resultsFile ****"
-
 # First check if results file is there and if not, error out gracefully
 if (-not (Test-Path -Path $resultsFile)) {
     Write-Error -Message "The test results file $resultsFile does not exist. Check to make sure the test was ran." -Category ObjectNotFound