feat(baseline): fix out directory for the disk baseline provider

The disk baseline provider always wrote the baseline to a hard-coded
"StrykerOutput" folder under the project path. Storing the baseline
inside the test project's directory is problematic: without a
non-obvious workaround, Stryker flags it as a change to the test
project and ignores the baseline.

This PR changes disk baseline provider so --output is honored if specified.
In this case, both a 'baseline' folder and .gitignore file will be put inside the specified output directory as siblings to the 'reports' folder.

If --output is not specified, StrykerOutput will be created as before, but .gitignore will be generated inside StrykerOutput instead of inside the 'reports' folder,
  which fixes the problem of the baseline being incorrectly flagged as a change to the test project.

Author: matt W10

src/Stryker.Abstractions/Options/IStrykerOptions.cs

@@ -15,6 +15,7 @@ public interface IStrykerOptions
     string S3Endpoint { get; init; }
     string S3Region { get; init; }
     BaselineProvider BaselineProvider { get; init; }
+    string BaselineOutputPath { get; init; }
     bool BreakOnInitialTestFailure { get; set; }
     int Concurrency { get; init; }
     string Configuration { get; init; }

src/Stryker.CLI/Stryker.CLI.UnitTest/Logging/InputBuilderTests.cs

@@ -27,7 +27,8 @@ public void ShouldAddGitIgnore()
         var gitIgnoreFile =
             fileSystemMock.AllFiles.Single(x => x.EndsWith(Path.Combine(".gitignore")));
         gitIgnoreFile.ShouldNotBeNull();
-        DateTime.TryParse(Directory.GetParent(gitIgnoreFile)!.Name.Split(".")[0], out _).ShouldBeTrue();
+        // the gitignore lives at the stable output root, not the per-run timestamped folder
+        Directory.GetParent(gitIgnoreFile)!.Name.ShouldBe("StrykerOutput");
         var fileContents = fileSystemMock.GetFile(gitIgnoreFile).Contents;
         Encoding.Default.GetString(fileContents).ShouldBe("*");
     }
@@ -67,4 +68,36 @@ public void ShouldAddGitIgnoreWithRelativePath()
         var fileContents = fileSystemMock.GetFile(gitIgnoreFile).Contents;
         Encoding.Default.GetString(fileContents).ShouldBe("*");
     }
+
+    [TestMethod]
+    public void ShouldSetBaselineOutputToStableRoot()
+    {
+        var fileSystemMock = new MockFileSystem();
+        var basePath = Directory.GetCurrentDirectory();
+        var target = new LoggingInitializer();
+
+        var inputs = new StrykerInputs();
+        inputs.BasePathInput.SuppliedInput = basePath;
+        target.SetupLogOptions(inputs, fileSystemMock);
+
+        // the baseline follows the stable output root, not the per-run timestamped output path
+        inputs.BaselineOutputInput.SuppliedInput.ShouldBe(Path.Combine(basePath, "StrykerOutput"));
+        inputs.OutputPathInput.SuppliedInput.ShouldStartWith(Path.Combine(basePath, "StrykerOutput") + Path.DirectorySeparatorChar);
+    }
+
+    [TestMethod]
+    public void ShouldSetBaselineOutputToSuppliedOutputPath()
+    {
+        var fileSystemMock = new MockFileSystem();
+        var basePath = Directory.GetCurrentDirectory();
+        var target = new LoggingInitializer();
+
+        var inputs = new StrykerInputs();
+        inputs.BasePathInput.SuppliedInput = basePath;
+        inputs.OutputPathInput.SuppliedInput = "output";
+        target.SetupLogOptions(inputs, fileSystemMock);
+
+        // an explicit output path has no timestamp subfolder, so it is the root itself
+        inputs.BaselineOutputInput.SuppliedInput.ShouldBe(Path.Combine(basePath, "output"));
+    }
 }

src/Stryker.CLI/Stryker.CLI/Logging/LoggingInitializer.cs

@@ -36,7 +36,17 @@ public void SetupLogOptions(IStrykerInputs inputs, IFileSystem fileSystem = null
 
     private string CreateOutputPath(IStrykerInputs inputs, IFileSystem fileSystem)
     {
-        var outputPath = inputs.OutputPathInput.SuppliedInput ?? Path.Combine("StrykerOutput", DateTime.Now.ToString("yyyy-MM-dd.HH-mm-ss"));
+        // The stable output root. When no output path is supplied the per-run output lives in a
+        // timestamped subfolder of this root; an explicitly supplied output path is the root itself.
+        // The root is where the disk baseline is stored (so it can be found on the next run) and
+        // where the gitignore is placed (so the baseline and all run outputs are excluded from git).
+        var outputRoot = inputs.OutputPathInput.SuppliedInput ?? "StrykerOutput";
+        var outputPath = inputs.OutputPathInput.SuppliedInput ?? Path.Combine(outputRoot, DateTime.Now.ToString("yyyy-MM-dd.HH-mm-ss"));
+
+        if (!Path.IsPathRooted(outputRoot))
+        {
+            outputRoot = Path.Combine(inputs.BasePathInput.SuppliedInput, outputRoot);
+        }
 
         if (!Path.IsPathRooted(outputPath))
         {
@@ -46,8 +56,11 @@ private string CreateOutputPath(IStrykerInputs inputs, IFileSystem fileSystem)
         // outputpath should always be created
         fileSystem.Directory.CreateDirectory(FilePathUtils.NormalizePathSeparators(outputPath));
 
-        // add gitignore if it didn't exist yet
-        var gitignorePath = FilePathUtils.NormalizePathSeparators(Path.Combine(outputPath, ".gitignore"));
+        // store the baseline under the stable output root so it follows --output and persists across runs
+        inputs.BaselineOutputInput.SuppliedInput = outputRoot;
+
+        // add gitignore to the output root if it didn't exist yet
+        var gitignorePath = FilePathUtils.NormalizePathSeparators(Path.Combine(outputRoot, ".gitignore"));
         if (!fileSystem.File.Exists(gitignorePath))
         {
             try

src/Stryker.Configuration/Options/Inputs/BaselineOutputInput.cs

@@ -0,0 +1,17 @@
+namespace Stryker.Configuration.Options.Inputs;
+
+public class BaselineOutputInput : Input<string>
+{
+    protected override string Description => "The directory the disk baseline provider stores and loads the baseline report from. This is derived from the output path (the stable output root) rather than supplied directly, so the baseline follows --output, is covered by the gitignore, and persists across runs since it lives outside the per-run timestamped folder.";
+
+    public override string Default => "StrykerOutput";
+
+    public string Validate()
+    {
+        if (string.IsNullOrWhiteSpace(SuppliedInput))
+        {
+            return Default;
+        }
+        return SuppliedInput;
+    }
+}

src/Stryker.Configuration/Options/StrykerInputs.cs

@@ -14,6 +14,7 @@ public interface IStrykerInputs
     S3RegionInput S3RegionInput { get; init; }
     AzureFileStorageUrlInput AzureFileStorageUrlInput { get; init; }
     BaselineProviderInput BaselineProviderInput { get; init; }
+    BaselineOutputInput BaselineOutputInput { get; init; }
     BasePathInput BasePathInput { get; init; }
     ConcurrencyInput ConcurrencyInput { get; init; }
     ConfigurationInput ConfigurationInput { get; init; }
@@ -90,6 +91,7 @@ public StrykerInputs(IFileSystem fileSystem = null)
     public WithBaselineInput WithBaselineInput { get; init; } = new();
     public ReportersInput ReportersInput { get; init; } = new();
     public BaselineProviderInput BaselineProviderInput { get; init; } = new();
+    public BaselineOutputInput BaselineOutputInput { get; init; } = new();
     public AzureFileStorageUrlInput AzureFileStorageUrlInput { get; init; } = new();
     public AzureFileStorageSasInput AzureFileStorageSasInput { get; init; } = new();
     public S3BucketNameInput S3BucketNameInput { get; init; } = new();
@@ -175,6 +177,7 @@ public IStrykerOptions ValidateAll()
             S3Region = S3RegionInput.Validate(baselineProvider, withBaseline),
             WithBaseline = withBaseline,
             BaselineProvider = baselineProvider,
+            BaselineOutputPath = BaselineOutputInput.Validate(),
             FallbackVersion = FallbackVersionInput.Validate(withBaseline, projectVersion, sinceTarget),
             Since = sinceEnabled,
             SinceTarget = sinceTarget,

src/Stryker.Configuration/Options/StrykerOptions.cs

@@ -158,6 +158,13 @@ public string Configuration
     /// </summary>
     public BaselineProvider BaselineProvider { get; init; }
 
+    /// <summary>
+    /// The directory the disk baseline provider stores and loads the baseline report from.
+    /// A relative path is resolved against <see cref="ProjectPath"/>. Defaults to the stable
+    /// StrykerOutput folder so baselines persist across runs.
+    /// </summary>
+    public string BaselineOutputPath { get; init; }
+
     /// <summary>
     /// The url to connect to the Azure File Storage API
     /// </summary>

src/Stryker.Core/Stryker.Core.UnitTest/Baseline/Providers/DiskBaselineProviderTests.cs

@@ -38,6 +38,65 @@ public async Task ShouldWriteToDiskAsync()
         file.ShouldNotBeNull();
     }
 
+    [TestMethod]
+    public async Task ShouldWriteToConfiguredBaselineOutputPathAsync()
+    {
+        var fileSystemMock = new MockFileSystem();
+        var options = new StrykerOptions()
+        {
+            ProjectPath = @"C:/Users/JohnDoe/Project/TestFolder",
+            BaselineOutputPath = "custom-baseline"
+        };
+        var sut = new DiskBaselineProvider(options, fileSystemMock);
+
+        await sut.Save(JsonReport.Build(options, ReportTestHelper.CreateProjectWith(), It.IsAny<TestProjectsInfo>()), "baseline/version");
+
+        var path = FilePathUtils.NormalizePathSeparators(@"C:/Users/JohnDoe/Project/TestFolder/custom-baseline/baseline/version/stryker-report.json");
+
+        var file = fileSystemMock.GetFile(path);
+        file.ShouldNotBeNull();
+    }
+
+    [TestMethod]
+    public async Task ShouldWriteToAbsoluteBaselineOutputPathAsync()
+    {
+        var fileSystemMock = new MockFileSystem();
+        var options = new StrykerOptions()
+        {
+            ProjectPath = @"C:/Users/JohnDoe/Project/TestFolder",
+            BaselineOutputPath = @"D:/shared/baselines"
+        };
+        var sut = new DiskBaselineProvider(options, fileSystemMock);
+
+        await sut.Save(JsonReport.Build(options, ReportTestHelper.CreateProjectWith(), It.IsAny<TestProjectsInfo>()), "baseline/version");
+
+        var path = FilePathUtils.NormalizePathSeparators(@"D:/shared/baselines/baseline/version/stryker-report.json");
+
+        var file = fileSystemMock.GetFile(path);
+        file.ShouldNotBeNull();
+    }
+
+    [TestMethod]
+    public async Task ShouldLoadReportFromConfiguredBaselineOutputPathAsync()
+    {
+        var fileSystemMock = new MockFileSystem();
+        var options = new StrykerOptions()
+        {
+            ProjectPath = @"C:/Users/JohnDoe/Project/TestFolder",
+            BaselineOutputPath = "custom-baseline"
+        };
+        var report = JsonReport.Build(options, ReportTestHelper.CreateProjectWith(), It.IsAny<ITestProjectsInfo>());
+
+        fileSystemMock.AddFile("C:/Users/JohnDoe/Project/TestFolder/custom-baseline/baseline/version/stryker-report.json", report.ToJson());
+
+        var target = new DiskBaselineProvider(options, fileSystemMock);
+
+        var result = await target.Load("baseline/version");
+
+        result.ShouldNotBeNull();
+        result.ToJson().ShouldBe(report.ToJson());
+    }
+
     [TestMethod]
     public async Task ShouldHandleFileNotFoundExceptionOnLoadAsync()
     {

src/Stryker.Core/Stryker.Core.UnitTest/Options/Inputs/BaselineOutputInputTests.cs

@@ -0,0 +1,39 @@
+using Shouldly;
+using Stryker.Configuration.Options.Inputs;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Stryker.Core.UnitTest.Options.Inputs;
+
+[TestClass]
+public class BaselineOutputInputTests : TestBase
+{
+    [TestMethod]
+    public void ShouldHaveHelpText()
+    {
+        var target = new BaselineOutputInput();
+        target.HelpText.ShouldNotBeNullOrEmpty();
+    }
+
+    [TestMethod]
+    [DataRow(null)]
+    [DataRow("")]
+    [DataRow("  ")]
+    public void ShouldReturnDefault_WhenNotSupplied(string input)
+    {
+        var target = new BaselineOutputInput { SuppliedInput = input };
+
+        var result = target.Validate();
+
+        result.ShouldBe("StrykerOutput");
+    }
+
+    [TestMethod]
+    public void ShouldReturnSuppliedValue_WhenSupplied()
+    {
+        var target = new BaselineOutputInput { SuppliedInput = "custom-baseline" };
+
+        var result = target.Validate();
+
+        result.ShouldBe("custom-baseline");
+    }
+}

src/Stryker.Core/Stryker.Core.UnitTest/Options/StrykerInputsTests.cs

@@ -19,6 +19,7 @@ public class StrykerInputsTests : TestBase
         AzureFileStorageSasInput = new AzureFileStorageSasInput(),
         AzureFileStorageUrlInput = new AzureFileStorageUrlInput(),
         BaselineProviderInput = new BaselineProviderInput(),
+        BaselineOutputInput = new BaselineOutputInput(),
         BasePathInput = new BasePathInput() { SuppliedInput = Directory.GetCurrentDirectory() },
         ConcurrencyInput = new ConcurrencyInput(),
         DashboardApiKeyInput = new DashboardApiKeyInput(),
@@ -200,6 +201,24 @@ public void WithBaselineShouldNotThrow_2743() // https://github.com/stryker-muta
         Should.NotThrow(() => _target.ValidateAll());
     }
 
+    [TestMethod]
+    public void BaselineOutputPathShouldDefaultToStrykerOutput()
+    {
+        var result = _target.ValidateAll();
+
+        result.BaselineOutputPath.ShouldBe("StrykerOutput");
+    }
+
+    [TestMethod]
+    public void ShouldSetBaselineOutputPathWhenSupplied()
+    {
+        _target.BaselineOutputInput.SuppliedInput = "custom-baseline";
+
+        var result = _target.ValidateAll();
+
+        result.BaselineOutputPath.ShouldBe("custom-baseline");
+    }
+
     [TestMethod]
     public void BaseLineOptionsShouldBeSetToDefaultWhenBaselineIsDisabled()
     {

src/Stryker.Core/Stryker.Core/Baseline/Providers/DiskBaselineProvider.cs

@@ -16,7 +16,16 @@ public class DiskBaselineProvider : IBaselineProvider
     private readonly IStrykerOptions _options;
     private readonly IFileSystem _fileSystem;
     private readonly ILogger<DiskBaselineProvider> _logger;
-    private const string _outputPath = "StrykerOutput";
+    private const string _defaultOutputPath = "StrykerOutput";
+
+    /// <summary>
+    /// The directory (relative to the project path, or absolute) the baseline is stored under.
+    /// Honors the configured baseline output path, falling back to the stable StrykerOutput
+    /// folder so baselines persist across runs when no output path was supplied.
+    /// </summary>
+    private string OutputPath => string.IsNullOrWhiteSpace(_options.BaselineOutputPath)
+        ? _defaultOutputPath
+        : _options.BaselineOutputPath;
 
     public DiskBaselineProvider(IStrykerOptions options, IFileSystem fileSystem = null)
     {
@@ -29,7 +38,7 @@ public DiskBaselineProvider(IStrykerOptions options, IFileSystem fileSystem = nu
     public async Task<IJsonReport> Load(string version)
     {
         var reportPath = FilePathUtils.NormalizePathSeparators(
-            Path.Combine(_options.ProjectPath, _outputPath, version, "stryker-report.json"));
+            Path.Combine(_options.ProjectPath, OutputPath, version, "stryker-report.json"));
 
         if (_fileSystem.File.Exists(reportPath))
         {
@@ -45,7 +54,7 @@ public async Task<IJsonReport> Load(string version)
     public async Task Save(IJsonReport report, string version)
     {
         var reportDirectory = FilePathUtils.NormalizePathSeparators(
-            Path.Combine(_options.ProjectPath, _outputPath, version));
+            Path.Combine(_options.ProjectPath, OutputPath, version));
 
         _fileSystem.Directory.CreateDirectory(reportDirectory);