From bf318863813832c093af040fe797775f7a4e46f2 Mon Sep 17 00:00:00 2001 From: LeeDayday Date: Thu, 18 Jun 2026 18:42:25 +0900 Subject: [PATCH 1/2] Add verification testing lab package --- VerificationTestingLab/README.md | 0 .../testing-lab-verification/.gitignore | 8 ++ .../testing-lab-verification/Package.swift | 23 ++++++ .../LoginValidator.swift | 32 ++++++++ .../RandomIDGenerator.swift | 42 +++++++++++ .../SharedCounter.swift | 23 ++++++ .../ThreeSixNineGame.swift | 18 +++++ .../01_WeakAssertionTests.swift | 29 ++++++++ .../02_FalsePositiveTests.swift | 74 +++++++++++++++++++ .../03_FlakyTests.swift | 43 +++++++++++ .../04_SharedMutableStateTests.swift | 51 +++++++++++++ .../05_ParameterizedTests.swift | 46 ++++++++++++ .../06_TestValidationChecklistTests.swift | 57 ++++++++++++++ 13 files changed, 446 insertions(+) create mode 100644 VerificationTestingLab/README.md create mode 100644 VerificationTestingLab/testing-lab-verification/.gitignore create mode 100644 VerificationTestingLab/testing-lab-verification/Package.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/LoginValidator.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/RandomIDGenerator.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/SharedCounter.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/ThreeSixNineGame.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/01_WeakAssertionTests.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/02_FalsePositiveTests.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/03_FlakyTests.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/04_SharedMutableStateTests.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/05_ParameterizedTests.swift create mode 100644 VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/06_TestValidationChecklistTests.swift diff --git a/VerificationTestingLab/README.md b/VerificationTestingLab/README.md new file mode 100644 index 0000000..e69de29 diff --git a/VerificationTestingLab/testing-lab-verification/.gitignore b/VerificationTestingLab/testing-lab-verification/.gitignore new file mode 100644 index 0000000..0023a53 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/VerificationTestingLab/testing-lab-verification/Package.swift b/VerificationTestingLab/testing-lab-verification/Package.swift new file mode 100644 index 0000000..fa6520a --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Package.swift @@ -0,0 +1,23 @@ +// swift-tools-version: 6.0 + +import PackageDescription + +let package = Package( + name: "VerificationTestingLab", + platforms: [ + .macOS(.v13) + ], + products: [ + .library( + name: "VerificationTestingLab", + targets: ["VerificationTestingLab"] + ) + ], + targets: [ + .target(name: "VerificationTestingLab"), + .testTarget( + name: "VerificationTestingLabTests", + dependencies: ["VerificationTestingLab"] + ) + ] +) diff --git a/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/LoginValidator.swift b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/LoginValidator.swift new file mode 100644 index 0000000..44cf3ab --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/LoginValidator.swift @@ -0,0 +1,32 @@ +import Foundation + +public struct Credentials: Equatable, Sendable { + public let username: String + public let password: String + + public init(username: String, password: String) { + self.username = username + self.password = password + } +} + +public struct LoginValidator { + public init() {} + + public func isValid(username: String, password: String) -> Bool { + username.trimmingCharacters(in: .whitespacesAndNewlines).count >= 3 + && password.count >= 8 + } + + public func credentials(from input: [String: String]) -> Credentials? { + guard + let username = input["username"], + let password = input["password"], + isValid(username: username, password: password) + else { + return nil + } + + return Credentials(username: username, password: password) + } +} diff --git a/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/RandomIDGenerator.swift b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/RandomIDGenerator.swift new file mode 100644 index 0000000..d99f2cd --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/RandomIDGenerator.swift @@ -0,0 +1,42 @@ +public protocol IntegerRandomizing: Sendable { + func nextInt(in range: ClosedRange) -> Int +} + +public struct SystemRandomizer: IntegerRandomizing { + public init() {} + + public func nextInt(in range: ClosedRange) -> Int { + Int.random(in: range) + } +} + +public struct FixedRandomizer: IntegerRandomizing { + private let value: Int + + public init(_ value: Int) { + self.value = value + } + + public func nextInt(in range: ClosedRange) -> Int { + min(max(value, range.lowerBound), range.upperBound) + } +} + +public struct RandomIDGenerator { + private let randomizer: any IntegerRandomizing + + public init(randomizer: any IntegerRandomizing = SystemRandomizer()) { + // Production uses SystemRandomizer; tests can inject FixedRandomizer. + self.randomizer = randomizer + } + + public func makeID(prefix: String = "user") -> String { + // Build a readable ID from a prefix and a four-digit number. + "\(prefix)-\(randomizer.nextInt(in: 1000...9999))" + } + + public func parseNumber(from id: String) -> Int? { + // Pull out the part after the final "-" so tests can inspect the range. + Int(id.split(separator: "-").last ?? "") + } +} diff --git a/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/SharedCounter.swift b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/SharedCounter.swift new file mode 100644 index 0000000..f3ef1a6 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/SharedCounter.swift @@ -0,0 +1,23 @@ +public final class SharedCounter: @unchecked Sendable { + public static let shared = SharedCounter() + + private var value: Int + + public init(startingAt value: Int = 0) { + self.value = value + } + + @discardableResult + public func increment() -> Int { + value += 1 + return value + } + + public func currentValue() -> Int { + value + } + + public func reset() { + value = 0 + } +} diff --git a/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/ThreeSixNineGame.swift b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/ThreeSixNineGame.swift new file mode 100644 index 0000000..7e2ce6e --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Sources/VerificationTestingLab/ThreeSixNineGame.swift @@ -0,0 +1,18 @@ +public struct ThreeSixNineGame { + public init() {} + + public func play(_ number: Int) -> String { + // Count how many digits should become claps. + let clapCount = String(number).filter { digit in + digit == "3" || digit == "6" || digit == "9" + }.count + + // If no clap digits are found, return the original number. + guard clapCount > 0 else { + return String(number) + } + + // Convert each matching digit into one "clap". + return Array(repeating: "clap", count: clapCount).joined(separator: " ") + } +} diff --git a/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/01_WeakAssertionTests.swift b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/01_WeakAssertionTests.swift new file mode 100644 index 0000000..4b4b362 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/01_WeakAssertionTests.swift @@ -0,0 +1,29 @@ +import Testing +import VerificationTestingLab + +@Suite("Experiment 1 - Weak Assertions") +struct WeakAssertionTests { + @Test("A weak assertion can pass while missing the exact behavior") + func weakAssertionExample() { + // 1. Run the game with a number that should produce two claps. + let result = ThreeSixNineGame().play(33) + + // 2. Check only that the result contains "clap". + // This passes for the correct implementation, but it is weak: + // "clap", "clap clap", and "wrong clap text" could all satisfy it. + #expect(result.contains("clap")) + } + + @Test("A strong assertion verifies the full expected output") + func strongAssertionExample() { + // 1. Run the same behavior as the weak test. + let result = ThreeSixNineGame().play(33) + + // 2. Compare the entire result to the exact expected value. + #expect(result == "clap clap") + } + + // To validate this test, intentionally break ThreeSixNineGame.play(_:) so it + // always returns "clap" for any number containing a clap digit. The weak test + // above will still pass for 33, while this exact assertion will fail. +} diff --git a/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/02_FalsePositiveTests.swift b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/02_FalsePositiveTests.swift new file mode 100644 index 0000000..b98f515 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/02_FalsePositiveTests.swift @@ -0,0 +1,74 @@ +import Testing +import VerificationTestingLab + +@Suite("Experiment 2 - False Positives") +struct FalsePositiveTests { + @Test("False positive example: expected data alone proves nothing") + func expectedValueWithoutSystemUnderTest() { + // 1. Prepare an expected value. + let expected = "clap clap" + + // 2. Compare the expected value to itself. + // This assertion passes without calling ThreeSixNineGame at all. + // It proves only that this test file contains the string "clap clap". + #expect(expected == "clap clap") + } + + @Test("Stronger alternative: exercise the system under test") + func exerciseSystemUnderTest() { + // 1. Call the production code that we actually want to verify. + let result = ThreeSixNineGame().play(99) + + // 2. Compare the result to an independently chosen expected value. + #expect(result == "clap clap") + } + + @Test("False positive example: duplicating production logic in the test") + func expectedValueGeneratedWithSameLogic() { + // 1. Choose sample input. + let number = 38 + + // 2. Build the expected value by repeating the production algorithm. + // This is risky because the test can copy the same bug as the app code. + let expected = String(number) + .filter { $0 == "3" || $0 == "6" || $0 == "9" } + .map { _ in "clap" } + .joined(separator: " ") + + // 3. Compare production output to the duplicated algorithm. + // This passes, but it is misleading. If production and test code both + // make the same mistake, the test can agree with the bug. + #expect(ThreeSixNineGame().play(number) == expected) + } + + @Test("Stronger alternative: use independently chosen examples") + func independentlyChosenExpectedValue() { + // 1. Use examples written from the game rules, not copied code. + // 2. Check both clap and non-clap cases so an incorrect digit is caught. + #expect(ThreeSixNineGame().play(38) == "clap") + #expect(ThreeSixNineGame().play(89) == "clap") + #expect(ThreeSixNineGame().play(88) == "88") + } + + @Test("Use #require when a later assertion depends on an optional value") + func requireValidCredentialsBeforeInspectingThem() throws { + // 1. Create the validator that parses valid login input. + let validator = LoginValidator() + + // 2. Require a non-nil result before making detailed assertions. + // If this is nil, the test stops here with a useful failure. + let credentials = try #require(validator.credentials(from: [ + "username": "lee", + "password": "trustworthy" + ])) + + // 3. Verify the exact credentials returned from the validator. + #expect(credentials == Credentials(username: "lee", password: "trustworthy")) + } + + // Mutation validation: + // BUG: Treat digit 8 as a clap digit. + // The duplicated-logic test above would not catch that bug if copied in both + // places. The independent examples for 88 and 89 would catch it because they + // specify behavior from the rules, not from the implementation. +} diff --git a/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/03_FlakyTests.swift b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/03_FlakyTests.swift new file mode 100644 index 0000000..e244e14 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/03_FlakyTests.swift @@ -0,0 +1,43 @@ +import Testing +import VerificationTestingLab + +@Suite("Experiment 3 - Flaky Tests") +struct FlakyTests { + @Test( + "Disabled flaky example: exact random output is unreliable", + .disabled("Re-enable to see that this test only passes when the random value happens to be 1234.") + ) + func disabledFlakyExample() { + // 1. Generate an ID using real randomness. + let id = RandomIDGenerator().makeID() + + // 2. This is intentionally weak test design. + // It expects one exact random value, so it will usually fail even when + // the production code is correct. + #expect(id == "user-1234") + } + + @Test("Dependency injection makes random behavior deterministic") + func deterministicIDGeneration() { + // 1. Inject a fixed randomizer so the test controls the generated number. + let generator = RandomIDGenerator(randomizer: FixedRandomizer(1234)) + + // 2. Verify exact output because randomness has been removed. + #expect(generator.makeID() == "user-1234") + #expect(generator.makeID(prefix: "session") == "session-1234") + } + + @Test("A useful randomized test checks stable properties, not exact luck") + func randomizedTestChecksShapeOnly() throws { + // 1. Use the real random generator. + let generator = RandomIDGenerator() + + // 2. Generate an ID and parse the numeric suffix for later assertions. + let id = generator.makeID() + let number = try #require(generator.parseNumber(from: id)) + + // 3. Verify stable rules that should be true for every random value. + #expect(id.hasPrefix("user-")) + #expect((1000...9999).contains(number)) + } +} diff --git a/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/04_SharedMutableStateTests.swift b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/04_SharedMutableStateTests.swift new file mode 100644 index 0000000..fc49fb0 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/04_SharedMutableStateTests.swift @@ -0,0 +1,51 @@ +import Testing +import VerificationTestingLab + +@Suite("Experiment 4 - Shared Mutable State") +struct SharedMutableStateTests { + @Test( + "Disabled order-dependent setup: resets and increments shared state", + .disabled("Re-enable with the next disabled test to demonstrate hidden coupling through shared state.") + ) + func disabledOrderDependentSetup() { + // 1. Reset the shared singleton. + SharedCounter.shared.reset() + + // 2. This passes by itself, but it leaves shared state behind. + #expect(SharedCounter.shared.increment() == 1) + } + + @Test( + "Disabled order-dependent assertion: assumes another test already ran", + .disabled("Re-enable to see that this test depends on execution order and fails when run alone.") + ) + func disabledOrderDependentAssertion() { + // 1. This test assumes disabledOrderDependentSetup() ran first. + // Swift Testing does not guarantee that order. + #expect(SharedCounter.shared.increment() == 2) + } + + @Test("Resetting shared state avoids leaking behavior between tests") + func resetSharedCounterInsideTest() { + // 1. Reset the singleton before using it so earlier tests cannot leak in. + SharedCounter.shared.reset() + + // 2. Verify the behavior this test owns. + #expect(SharedCounter.shared.increment() == 1) + #expect(SharedCounter.shared.increment() == 2) + + // 3. Reset again so this test does not leak state into later tests. + SharedCounter.shared.reset() + } + + @Test("Best alternative: create isolated state per test") + func isolatedCounterPerTest() { + // 1. Create a fresh counter owned only by this test. + let counter = SharedCounter() + + // 2. Verify the counter without depending on any global state. + #expect(counter.currentValue() == 0) + #expect(counter.increment() == 1) + #expect(counter.increment() == 2) + } +} diff --git a/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/05_ParameterizedTests.swift b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/05_ParameterizedTests.swift new file mode 100644 index 0000000..64a77b8 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/05_ParameterizedTests.swift @@ -0,0 +1,46 @@ +import Testing +import VerificationTestingLab + +struct PlayCase: Sendable { + // One row in the parameterized test table. + let number: Int + let expected: String +} + +@Suite("Experiment 5 - Parameterized Tests") +struct ParameterizedTests { + @Test( + "Three-six-nine examples", + arguments: [ + PlayCase(number: 1, expected: "1"), + PlayCase(number: 3, expected: "clap"), + PlayCase(number: 33, expected: "clap clap"), + PlayCase(number: 28, expected: "28"), + PlayCase(number: 99, expected: "clap clap") + ] + ) + func playReturnsExpectedValue(sample: PlayCase) { + // 1. Swift Testing runs this function once for each PlayCase above. + // 2. Compare the game result to that row's expected value. + #expect(ThreeSixNineGame().play(sample.number) == sample.expected) + } + + @Test( + "Single-clap numbers", + arguments: [ + PlayCase(number: 6, expected: "clap"), + PlayCase(number: 9, expected: "clap"), + PlayCase(number: 13, expected: "clap"), + PlayCase(number: 16, expected: "clap") + ] + ) + func singleClapCases(sample: PlayCase) { + // 1. Reuse the same test shape for several single-clap examples. + // 2. Each row becomes its own reported test case. + #expect(ThreeSixNineGame().play(sample.number) == sample.expected) + } + + // Parameterized tests make it cheap to add examples. That matters for trust: + // a single example might accidentally pass, but a table of independent cases + // is more likely to catch a broken rule. +} diff --git a/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/06_TestValidationChecklistTests.swift b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/06_TestValidationChecklistTests.swift new file mode 100644 index 0000000..318600b --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Tests/VerificationTestingLabTests/06_TestValidationChecklistTests.swift @@ -0,0 +1,57 @@ +import Testing +import VerificationTestingLab + +@Suite("Experiment 6 - Test Validation Checklist") +struct TestValidationChecklistTests { + @Test("Mutation: replacing 9 with 8 would be caught by independent examples") + func mutationReplacingNineWithEightIsCaught() { + // 1. Create the system under test once for this group of examples. + let game = ThreeSixNineGame() + + // 2. Verify that 9 is a clap digit. + #expect(game.play(9) == "clap") + #expect(game.play(99) == "clap clap") + + // 3. Verify that 8 is not a clap digit. + #expect(game.play(8) == "8") + #expect(game.play(88) == "88") + } + + @Test("Mutation: returning one clap for every match would be caught") + func mutationReturningOnlyOneClapIsCaught() { + // 1. Create the system under test. + let game = ThreeSixNineGame() + + // 2. Check numbers with two clap digits so "clap" is not enough. + #expect(game.play(33) == "clap clap") + #expect(game.play(36) == "clap clap") + #expect(game.play(39) == "clap clap") + } + + @Test("Checklist example: exact, deterministic, isolated") + func checklistExample() { + // 1. Create fresh test-owned state and deterministic dependencies. + let counter = SharedCounter() + let generator = RandomIDGenerator(randomizer: FixedRandomizer(4321)) + + // 2. Verify exact behavior, deterministic output, and isolated state. + #expect(ThreeSixNineGame().play(28) == "28") + #expect(generator.makeID() == "user-4321") + #expect(counter.increment() == 1) + } + + // Mutation validation notes: + // + // BUG: + // Treat digit 8 as a clap digit. + // + // Tests that catch it: + // - independentlyChosenExpectedValue checks 88 stays "88" + // - mutationReplacingNineWithEightIsCaught checks 8 and 88 stay numeric + // - parameterized 28 case checks unrelated digits do not clap + // + // Tests that may fail to detect it: + // - weakAssertionExample only checks that 33 contains "clap" + // - expectedValueWithoutSystemUnderTest never calls production code + // - expectedValueGeneratedWithSameLogic can repeat the same mistake +} From 632c284d04469bcf01d72c92cc0f10ddbb8b2d8f Mon Sep 17 00:00:00 2001 From: LeeDayday Date: Thu, 18 Jun 2026 18:42:51 +0900 Subject: [PATCH 2/2] Document test verification methodology --- VerificationTestingLab/README.md | 137 +++++++ .../Docs/verification-tests.md | 367 ++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100644 VerificationTestingLab/testing-lab-verification/Docs/verification-tests.md diff --git a/VerificationTestingLab/README.md b/VerificationTestingLab/README.md index e69de29..cb99737 100644 --- a/VerificationTestingLab/README.md +++ b/VerificationTestingLab/README.md @@ -0,0 +1,137 @@ +--- +Author: Sammy +Research Idea: Swift Testing +--- + +# Test Verification with Swift Testing + +| Item | Content | +| --- | --- | +| Research Idea | Swift Testing | +| Essential Question | How can tests be validated to ensure they are actually working correctly when using Swift Testing? | +| Challenge Statement | Design and evaluate testing approaches using Swift Testing to ensure tests behave correctly by comparing success and failure cases | +| Challenge Response | Investigate how tests can be verified using Apple's Swift Testing framework. The goal is not only to write passing tests, but to ask whether those tests would fail when the production code is wrong. | + +## Research Background + +Passing tests can still give false confidence. A test may pass because its assertion is too weak, because it never exercises the production code, because it repeats the same logic as the implementation, or because it depends on randomness or shared mutable state. + +This project explores test verification: the practice of checking whether tests can actually detect realistic bugs. Instead of only asking "does the test pass?", this lab asks "would this test fail if the production code were intentionally broken?" + +The project uses Swift Testing rather than XCTest so the experiments can focus on modern Swift test syntax such as `@Suite`, `@Test`, `#expect`, `#require`, and `@Test(arguments:)`. + +## Objective + +The objective is to design a small Swift Package that demonstrates how test quality can be evaluated. + +The lab investigates: + +- Weak assertions that miss incorrect behavior +- False positives where tests pass without proving production behavior +- Flaky tests caused by randomness +- Hidden coupling caused by shared mutable state +- Parameterized tests for broader coverage +- Mutation validation as a way to check whether tests catch bugs + +Success is measured by whether the test suite passes by default and whether the documented mutations would be caught by stronger tests. + +## Methodology + +The methodology is organized by **test verification risk**, not by file location. A diagram grouped only by `Sources` and `Tests` shows where files live, but it does not clearly show why each experiment matters. Grouping by risk makes the testing lesson easier to see: + +- What can make a test misleading? +- Which production type demonstrates that risk? +- Which test suite shows the weak version? +- Which test suite shows the stronger verification strategy? + +```mermaid +classDiagram + namespace ProductionSubjects { + class ThreeSixNineGame { + +play(number: Int) String + } + + class LoginValidator { + +credentials(from: Dictionary) Credentials? + } + + class RandomIDGenerator { + +makeID(prefix: String) String + } + + class SharedCounter { + +increment() Int + +reset() + } + } + + namespace VerificationRisks { + class WeakAssertion { + "checks only part of the result" + } + + class FalsePositive { + "passes without proving behavior" + } + + class FlakyTest { + "depends on random luck" + } + + class SharedStateCoupling { + "depends on test order or global state" + } + } + + namespace VerificationStrategies { + class ExactExpectation { + "#expect(result == expected)" + } + + class IndependentExpectedValue { + "expected value comes from rules" + } + + class DeterministicDependency { + "inject FixedRandomizer" + } + + class IsolatedState { + "create fresh object per test" + } + + class ParameterizedCoverage { + "@Test(arguments:)" + } + + class MutationValidation { + "break code intentionally" + } + } + + ThreeSixNineGame --> WeakAssertion : can reveal + ThreeSixNineGame --> FalsePositive : can reveal + RandomIDGenerator --> FlakyTest : can reveal + SharedCounter --> SharedStateCoupling : can reveal + LoginValidator --> FalsePositive : uses #require example + + WeakAssertion --> ExactExpectation : improved by + FalsePositive --> IndependentExpectedValue : improved by + FlakyTest --> DeterministicDependency : improved by + SharedStateCoupling --> IsolatedState : improved by + + ExactExpectation --> MutationValidation : verified by + IndependentExpectedValue --> MutationValidation : verified by + ParameterizedCoverage --> MutationValidation : strengthens +``` + +The experiment process is: + +1. Implement simple production behavior in the Swift Package. +2. Write tests that pass by default. +3. Add examples of weak or misleading tests. +4. Add stronger alternatives that verify exact behavior. +5. Document intentional mutations, such as treating `8` as a clap digit. +6. Identify which tests catch the mutation and which tests fail to detect it. + +Detailed experiment notes are in [`testing-lab-verification/Docs/verification-tests.md`](testing-lab-verification/Docs/verification-tests.md). diff --git a/VerificationTestingLab/testing-lab-verification/Docs/verification-tests.md b/VerificationTestingLab/testing-lab-verification/Docs/verification-tests.md new file mode 100644 index 0000000..c15c4d2 --- /dev/null +++ b/VerificationTestingLab/testing-lab-verification/Docs/verification-tests.md @@ -0,0 +1,367 @@ +# Test Verification with Swift Testing + +## Project Purpose + +This package is a small testing lab. It is not an iOS app and it does not launch a screen. Instead, it separates simple Swift logic into a package so the behavior can be tested quickly with `swift test`. + +The package asks one question: + +> Passing tests are nice, but how do we know the tests would fail when the code is wrong? + +## How to Run + +From the package root: + +```bash +cd VerificationTestingLab/testing-lab-verification +swift test +``` + +The important files are: + +- `Sources/VerificationTestingLab`: production code being tested +- `Tests/VerificationTestingLabTests`: Swift Testing test suites +- `Docs/verification-tests.md`: notes about what each experiment demonstrates + +## Essential Question + +How can tests be validated to ensure they are actually working correctly when using Swift Testing? + +## Challenge + +Investigate how trustworthy tests can be designed using Apple's Swift Testing framework. The goal is not only to write passing tests, but to ask whether those tests would fail when the production code is wrong. + +## Methodology + +This lab separates the code into three conceptual layers: + +1. Production layer: small Swift types that contain behavior. +2. Test layer: Swift Testing suites that exercise the behavior. +3. Testing foundation: Swift Testing tools such as `@Suite`, `@Test`, `#expect`, `#require`, and `@Test(arguments:)`. + +The methodology is: + +1. Write simple production behavior. +2. Write tests that pass. +3. Question whether those tests are strong enough. +4. Introduce realistic mistakes as disabled tests, comments, or temporary mutations. +5. Check which tests would catch the mistake. +6. Improve tests by making them exact, deterministic, isolated, and independent from implementation logic. + +```mermaid +classDiagram + namespace Production { + class ThreeSixNineGame { + +play(number: Int) String + } + + class LoginValidator { + +isValid(username: String, password: String) Bool + +credentials(from: Dictionary) Credentials? + } + + class RandomIDGenerator { + -randomizer: IntegerRandomizing + +makeID(prefix: String) String + +parseNumber(from: String) Int? + } + + class IntegerRandomizing { + <> + +nextInt(in: ClosedRange~Int~) Int + } + + class SystemRandomizer { + +nextInt(in: ClosedRange~Int~) Int + } + + class FixedRandomizer { + -value: Int + +nextInt(in: ClosedRange~Int~) Int + } + + class SharedCounter { + +shared: SharedCounter + -value: Int + +increment() Int + +currentValue() Int + +reset() + } + } + + namespace TestLayer { + class WeakAssertionTests + class FalsePositiveTests + class FlakyTests + class SharedMutableStateTests + class ParameterizedTests + class TestValidationChecklistTests + } + + namespace SwiftTesting { + class Suite { + <> + } + class Test { + <> + } + class Expect { + <> + } + class Require { + <> + } + } + + SystemRandomizer ..|> IntegerRandomizing + FixedRandomizer ..|> IntegerRandomizing + RandomIDGenerator --> IntegerRandomizing : has + + WeakAssertionTests --> ThreeSixNineGame : tests + FalsePositiveTests --> ThreeSixNineGame : tests + FalsePositiveTests --> LoginValidator : tests + FlakyTests --> RandomIDGenerator : tests + FlakyTests --> FixedRandomizer : injects + SharedMutableStateTests --> SharedCounter : tests + ParameterizedTests --> ThreeSixNineGame : tests + TestValidationChecklistTests --> ThreeSixNineGame : validates mutations + TestValidationChecklistTests --> RandomIDGenerator : checks determinism + TestValidationChecklistTests --> SharedCounter : checks isolation + + WeakAssertionTests ..> Suite : uses + WeakAssertionTests ..> Test : uses + WeakAssertionTests ..> Expect : uses + FalsePositiveTests ..> Require : uses + FlakyTests ..> Require : uses + ParameterizedTests ..> Test : uses arguments +``` + +## Useful Swift Testing Terms + +- `@Suite`: groups related tests. +- `@Test`: marks a function as a test. +- `@Test(arguments:)`: runs the same test once for each argument row. +- `#expect`: checks that a condition is true, but allows the test function to continue. +- `#require`: unwraps a required value. If the value is `nil`, the test fails immediately. + +Tests should be independent. A suite groups tests by topic, but tests inside one suite should not depend on being run in the order they appear in the file. + +## Experiments + +### 1. Weak Assertions + +A weak assertion checks only part of the result: + +```swift +#expect(result.contains("clap")) +``` + +That can pass for both `"clap"` and `"clap clap"`, so it misses a bug where double-clap numbers produce only one clap. + +A stronger assertion checks the exact behavior: + +```swift +#expect(result == "clap clap") +``` + +### 2. False Positives + +False positives happen when a test passes without proving the behavior. Two common examples are: + +- The test never calls the system under test. +- The expected value is generated with the same logic as the production code. + +In this lab, false positive means: + +> The production code can be wrong, but the test still passes. + +Expected values should come from the rules, examples, or requirements, not from copying the implementation. + +Weak example: + +```swift +let expected = "clap clap" +#expect(expected == "clap clap") +``` + +This test never calls `ThreeSixNineGame.play(_:)`, so it cannot prove anything about the game. + +Stronger example: + +```swift +let result = ThreeSixNineGame().play(99) +#expect(result == "clap clap") +``` + +This test exercises the system under test and compares it to an independently chosen expected value. + +### 3. Flaky Tests + +Flaky tests pass or fail for reasons unrelated to code changes. A test that expects `RandomIDGenerator()` to produce one exact random value is unreliable. + +This lab keeps the flaky example as a disabled test: + +```swift +@Test( + "Disabled flaky example: exact random output is unreliable", + .disabled("Re-enable to see that this test only passes when the random value happens to be 1234.") +) +func disabledFlakyExample() { + let id = RandomIDGenerator().makeID() + #expect(id == "user-1234") +} +``` + +It is disabled so the normal verification run remains stable, while the failing test body is still visible. + +The improved design injects randomness: + +```swift +let generator = RandomIDGenerator(randomizer: FixedRandomizer(1234)) +#expect(generator.makeID() == "user-1234") +``` + +Deterministic tests improve reliability because a failure points to a real behavior change. + +Another valid approach is to keep randomness but check only stable properties: + +```swift +#expect(id.hasPrefix("user-")) +#expect((1000...9999).contains(number)) +``` + +This does not guess the exact random number. It checks rules that should be true for every generated ID. + +### 4. Shared Mutable State + +Shared state can create hidden coupling. If one test increments `SharedCounter.shared`, another test can pass or fail depending on test order. + +Tests should not depend on the order they are written in. Even when multiple `@Test` functions are inside the same `@Suite` or `struct`, each test must be able to pass on its own. A suite groups related tests, but it should not be treated like an ordered script. + +Avoid this pattern: + +```swift +@Test func firstTest() { + SharedCounter.shared.reset() + #expect(SharedCounter.shared.increment() == 1) +} + +@Test func secondTest() { + #expect(SharedCounter.shared.increment() == 2) +} +``` + +`secondTest` only passes if `firstTest` ran before it. That creates an order-dependent test. + +In the test suite, this risk is represented with disabled tests. They are real test functions, but skipped during normal verification so failure examples do not run accidentally. + +Prefer isolated state: + +```swift +let counter = SharedCounter() +#expect(counter.increment() == 1) +``` + +If shared state is unavoidable, reset it inside the test that uses it. This is still a weaker option than isolated state because reset calls can be forgotten, and parallel tests can still interfere with each other. + +### 5. Parameterized Tests + +Swift Testing supports tables of examples with `@Test(arguments:)`. This improves coverage while keeping tests readable: + +```swift +@Test(arguments: [ + PlayCase(number: 1, expected: "1"), + PlayCase(number: 3, expected: "clap"), + PlayCase(number: 33, expected: "clap clap") +]) +func playReturnsExpectedValue(sample: PlayCase) { + #expect(ThreeSixNineGame().play(sample.number) == sample.expected) +} +``` + +Parameterized tests make it easier to cover normal cases, edge cases, and regression cases. + +Each argument row is reported as its own test case. That makes it easier to see which input failed. + +### 6. Mutation Validation + +Mutation validation means intentionally breaking production code and checking whether tests fail. + +Example mutation: + +```swift +// BUG: +// Treat digit 8 as a clap digit. +``` + +Broken logic: + +```swift +digit == "3" || digit == "6" || digit == "8" +``` + +Correct logic: + +```swift +digit == "3" || digit == "6" || digit == "9" +``` + +Tests that catch this mutation: + +- Exact tests for `9 -> "clap"` and `99 -> "clap clap"` +- Exact tests for `8 -> "8"` and `88 -> "88"` +- Parameterized cases such as `(28, "28")` + +Tests that may not catch this mutation: + +- Tests that only check `.contains("clap")` +- Tests that never call `ThreeSixNineGame.play(_:)` +- Tests that calculate expected values by copying production logic + +To try this manually: + +1. Open `Sources/VerificationTestingLab/ThreeSixNineGame.swift`. +2. Temporarily change the clap digits from `3, 6, 9` to `3, 6, 8`. +3. Run `swift test`. +4. Confirm that the stronger tests fail. +5. Change the code back to `3, 6, 9`. + +This is how we verify that the tests are not just passing, but actually capable of catching bugs. + +## Findings + +Trustworthy tests are designed to fail for the right reasons. Passing is not enough. A good test should prove that the production behavior matches the rule, and it should become red when that rule is intentionally broken. + +The most useful patterns were: + +- Assert exact behavior instead of vague properties when exact behavior matters. +- Exercise the real system under test. +- Keep expected values independent from implementation logic. +- Inject nondeterministic dependencies such as randomness. +- Avoid shared mutable state between tests. +- Do not depend on test execution order, even inside one suite. +- Use parameterized tests to cover many independent examples. +- Validate tests by trying small intentional mutations. + +## Trustworthy Test Checklist + +- Does the test fail when production code is intentionally broken? +- Does it verify exact behavior? +- Does it call the real system under test? +- Is expected data independent from implementation? +- Is the test deterministic? +- Is it isolated from other tests? +- Would it still pass if run alone or in a different order? +- Does the test name clearly describe behavior? +- Are edge cases covered? +- Would failure messages help identify the bug? + +## Beginner Summary + +A trustworthy test should answer three questions: + +1. Did I call the real code I meant to test? +2. Did I compare the result to an expected value from the requirement, not from copied implementation logic? +3. Would this test fail if I intentionally introduced a realistic bug? + +If the answer to any of these is "no", the test may be giving false confidence.