[PATCH v2 9/9] bootlinux: add support for A/B kernel testing

[Luis Chamberlain <mcgrof@kernel.org>]

Right now we use the same kernel for all target nodes. We want to
compare and contrast different kenrels for different features. We
add support for A/B testing by leveraging the baseline and dev groups
provided to us by KDEVOPS_BASELINE_AND_DEV.

This extends the bootlinux playbook by enabling us to allow a different
kernel tree / ref to be used for the dev group. This just becomes a
configuration thing. The targets are intuitive:

  make linux                 # Handles A/B compilation transparently
  make linux-baseline        # Build and install baseline kernel only
  make linux-dev             # Build and install development kernel only

We also add a simple check to verify all different build types end
up respecting different kernels if we so choose:

  make check-linux-ab

This does not launch targets it just verifies we don't regress in the
future with the different ref tags.

Generated-by: Claude AI
Signed-off-by: Luis Chamberlain <mcgrof@kernel.org>
---
 .github/workflows/linux-ab-testing.yml       | 217 +++++++++++++++++++
 .github/workflows/linux-ab.yml               |  47 ++++
 Makefile                                     |   1 +
 PROMPTS.md                                   |  52 +++++
 defconfigs/linux-ab-testing                  |  14 ++
 defconfigs/linux-ab-testing-9p               |  15 ++
 defconfigs/linux-ab-testing-builder          |  15 ++
 defconfigs/linux-ab-testing-target           |  15 ++
 docs/kdevops-make-linux.md                   | 158 ++++++++++++++
 playbooks/roles/bootlinux/defaults/main.yml  |  14 ++
 playbooks/roles/bootlinux/tasks/build/9p.yml |  20 +-
 playbooks/roles/bootlinux/tasks/main.yml     | 112 ++++++++++
 scripts/infer_last_stable_kernel.sh          |  35 +++
 scripts/linux-ab-testing.Makefile            |  51 +++++
 scripts/test-linux-ab-config.py              | 182 ++++++++++++++++
 scripts/test-linux-ab.sh                     | 213 ++++++++++++++++++
 workflows/linux/Kconfig                      | 102 ++++++++-
 workflows/linux/Makefile                     |  39 ++++
 18 files changed, 1291 insertions(+), 11 deletions(-)
 create mode 100644 .github/workflows/linux-ab-testing.yml
 create mode 100644 .github/workflows/linux-ab.yml
 create mode 100644 defconfigs/linux-ab-testing
 create mode 100644 defconfigs/linux-ab-testing-9p
 create mode 100644 defconfigs/linux-ab-testing-builder
 create mode 100644 defconfigs/linux-ab-testing-target
 create mode 100755 scripts/infer_last_stable_kernel.sh
 create mode 100644 scripts/linux-ab-testing.Makefile
 create mode 100755 scripts/test-linux-ab-config.py
 create mode 100755 scripts/test-linux-ab.sh

diff --git a/.github/workflows/linux-ab-testing.yml b/.github/workflows/linux-ab-testing.yml
new file mode 100644
index 00000000..aa52abbb
--- /dev/null
+++ b/.github/workflows/linux-ab-testing.yml
@@ -0,0 +1,217 @@
+name: Linux A/B Testing Verification
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  linux-ab-testing-verification:
+    name: Verify Linux A/B Testing Variables
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro_container:
+          - debian:testing
+          - fedora:latest
+        build_method:
+          - target
+          - 9p
+          - builder
+
+    container: ${{ matrix.distro_container }}
+    steps:
+      - name: Document test environment
+        run: |
+          echo "Running Linux A/B testing verification on ${{ matrix.distro_container }}"
+          echo "Build method: ${{ matrix.build_method }}"
+          uname -a
+
+      - name: Install dependencies
+        run: |
+          if [ "${{ matrix.distro_container }}" = "debian:testing" ]; then
+            echo "Installing packages for Debian"
+            apt-get update
+            apt-get install -y ansible-core make gcc ncurses-dev bison flex git python3
+          elif [ "${{ matrix.distro_container }}" = "fedora:latest" ]; then
+            echo "Installing packages for Fedora"
+            dnf install -y ansible make gcc ncurses-devel bison flex git python3
+          else
+            echo "Unknown distribution: ${{ matrix.distro_container }}"
+            exit 1
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure git for kdevops
+        run: |
+          git config --global --add safe.directory '*'
+          git config --global user.name "kdevops-ci"
+          git config --global user.email "kdevops@lists.linux.dev"
+
+      - name: Apply A/B testing defconfig
+        run: |
+          echo "Applying linux-ab-testing-${{ matrix.build_method }} defconfig"
+          make defconfig-linux-ab-testing-${{ matrix.build_method }}
+
+          # Verify configuration was applied correctly
+          echo "=== Verifying A/B testing configuration ==="
+          grep -E "CONFIG_KDEVOPS_BASELINE_AND_DEV=y" .config || exit 1
+          grep -E "CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y" .config || exit 1
+
+          # Check build method specific configs
+          case "${{ matrix.build_method }}" in
+            target)
+              grep -E "CONFIG_BOOTLINUX_TARGET=y" .config || exit 1
+              ;;
+            9p)
+              grep -E "CONFIG_BOOTLINUX_9P=y" .config || exit 1
+              ;;
+            builder)
+              grep -E "CONFIG_BOOTLINUX_BUILDER=y" .config || exit 1
+              ;;
+          esac
+
+      - name: Run make to generate extra_vars.yaml
+        run: |
+          make
+
+      - name: Extract and verify kernel references
+        run: |
+          echo "=== Extracting kernel references from configuration ==="
+
+          # Get the baseline ref (should be master or main)
+          BASELINE_REF=$(grep "^bootlinux_tree_ref:" extra_vars.yaml | awk '{print $2}')
+          echo "Baseline ref: $BASELINE_REF"
+
+          # Get the dev ref using the inference script
+          DEV_REF=$(grep "^bootlinux_dev_tree_ref:" extra_vars.yaml | awk '{print $2}')
+          echo "Dev ref from config: $DEV_REF"
+
+          # Since we're in a container without /mirror/linux.git, the script will fallback
+          # For CI, we'll simulate getting the latest stable tag
+          if [ -f scripts/infer_last_stable_kernel.sh ]; then
+            INFERRED_STABLE=$(./scripts/infer_last_stable_kernel.sh 2>/dev/null || echo "v6.12")
+            echo "Inferred stable version: $INFERRED_STABLE"
+          fi
+
+          # Verify refs are different
+          if [ "$BASELINE_REF" = "$DEV_REF" ]; then
+            echo "ERROR: Baseline and dev refs should be different for A/B testing"
+            exit 1
+          fi
+
+          # Store refs for later verification
+          echo "BASELINE_REF=$BASELINE_REF" >> $GITHUB_ENV
+          echo "DEV_REF=$DEV_REF" >> $GITHUB_ENV
+
+      - name: Test debug functionality with Ansible
+        run: |
+          echo "=== Testing debug output with DEBUG_REF=1 ==="
+
+          # Create a minimal hosts file for container testing
+          cat > hosts << EOF
+          [all]
+          localhost ansible_connection=local
+
+          [baseline]
+          localhost ansible_connection=local
+
+          [dev]
+          EOF
+
+          # Run the bootlinux playbook with debug enabled and capture output
+          export DEBUG_REF=1
+          ansible-playbook -i hosts playbooks/bootlinux.yml --tags vars,debug -v > debug_output.txt 2>&1 || true
+
+          # Verify debug output based on build method
+          case "${{ matrix.build_method }}" in
+            9p)
+              echo "=== Verifying 9P debug output (localhost context) ==="
+              if grep -q "active_linux_ref" debug_output.txt; then
+                echo "✓ Found active_linux_ref in 9P debug output"
+              else
+                echo "✗ Missing active_linux_ref in 9P debug output"
+                cat debug_output.txt
+                exit 1
+              fi
+              ;;
+            target|builder)
+              echo "=== Verifying non-9P debug output (per-node context) ==="
+              if grep -q "target_linux_ref" debug_output.txt; then
+                echo "✓ Found target_linux_ref in non-9P debug output"
+              else
+                echo "✗ Missing target_linux_ref in non-9P debug output"
+                cat debug_output.txt
+                exit 1
+              fi
+              ;;
+          esac
+
+      - name: Verify A/B testing Makefile rules
+        run: |
+          echo "=== Verifying A/B testing Makefile structure ==="
+
+          # Check that linux target depends on linux-baseline and linux-dev
+          if grep -A5 "^linux:" workflows/linux/Makefile | grep -q "linux-baseline linux-dev"; then
+            echo "✓ Makefile has correct A/B testing dependencies"
+          else
+            echo "✗ Makefile missing A/B testing dependencies"
+            exit 1
+          fi
+
+          # Verify linux-baseline and linux-dev targets exist
+          if grep -q "^linux-baseline:" workflows/linux/Makefile && \
+             grep -q "^linux-dev:" workflows/linux/Makefile; then
+            echo "✓ Both linux-baseline and linux-dev targets exist"
+          else
+            echo "✗ Missing linux-baseline or linux-dev targets"
+            exit 1
+          fi
+
+      - name: Test variable resolution patterns
+        run: |
+          echo "=== Testing variable resolution patterns ==="
+
+          # Create test playbook to verify variable resolution
+          cat > test_vars.yml << 'EOF'
+          ---
+          - hosts: localhost
+            connection: local
+            tasks:
+              - name: Load extra vars
+                include_vars: extra_vars.yaml
+
+              - name: Display loaded variables
+                debug:
+                  msg: |
+                    bootlinux_tree_ref: {{ bootlinux_tree_ref | default('undefined') }}
+                    bootlinux_dev_tree_ref: {{ bootlinux_dev_tree_ref | default('undefined') }}
+                    kdevops_baseline_and_dev: {{ kdevops_baseline_and_dev | default(false) }}
+                    bootlinux_ab_different_ref: {{ bootlinux_ab_different_ref | default(false) }}
+          EOF
+
+          ansible-playbook test_vars.yml -v
+
+      - name: Summary report
+        if: always()
+        run: |
+          echo "=== A/B Testing Verification Summary ==="
+          echo "Distribution: ${{ matrix.distro_container }}"
+          echo "Build Method: ${{ matrix.build_method }}"
+          echo "Baseline Ref: ${BASELINE_REF:-not set}"
+          echo "Dev Ref: ${DEV_REF:-not set}"
+          echo ""
+
+          if [ -f .config ]; then
+            echo "Key configurations:"
+            grep -E "(BASELINE_AND_DEV|AB_DIFFERENT_REF|BOOTLINUX_TARGET|BOOTLINUX_9P|BOOTLINUX_BUILDER)" .config | head -10
+          fi
+
+          echo ""
+          echo "Test completed successfully ✓"
diff --git a/.github/workflows/linux-ab.yml b/.github/workflows/linux-ab.yml
new file mode 100644
index 00000000..9162c887
--- /dev/null
+++ b/.github/workflows/linux-ab.yml
@@ -0,0 +1,47 @@
+name: Run kdevops linux-ab tests on self-hosted runner
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+  workflow_dispatch:  # Add this for manual triggering of the workflow
+
+jobs:
+  run-kdevops:
+    name: Run kdevops CI
+    runs-on: [self-hosted, Linux, X64]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set CI metadata for kdevops-results-archive
+        run: |
+          echo "$(basename ${{ github.repository }})" > ci.trigger
+          git log -1 --pretty=format:"%s" > ci.subject
+          # Start out pessimistic
+          echo "not ok" > ci.result
+          echo "Nothing to write home about." > ci.commit_extra
+
+      - name: Set kdevops path
+        run: echo "KDEVOPS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
+
+      - name: Configure git
+        run: |
+          git config --global --add safe.directory '*'
+          git config --global user.name "kdevops"
+          git config --global user.email "kdevops@lists.linux.dev"
+
+      - name: Run kdevops check-linux-ab
+        run: |
+          make check-linux-ab
+          echo "ok" > ci.result
+
+      # Ensure make destroy always runs, even on failure
+      - name: Run kdevops make destroy
+        if: always()  # This ensures the step runs even if previous steps failed
+        run: |
+          make destroy
+          make mrproper
diff --git a/Makefile b/Makefile
index c88637c2..8755577e 100644
--- a/Makefile
+++ b/Makefile
@@ -243,6 +243,7 @@ $(KDEVOPS_NODES): .config $(ANSIBLE_CFG_FILE) $(KDEVOPS_NODES_TEMPLATE)
 DEFAULT_DEPS += $(LOCALHOST_SETUP_WORK)
 
 include scripts/tests.Makefile
+include scripts/linux-ab-testing.Makefile
 include scripts/ci.Makefile
 include scripts/archive.Makefile
 include scripts/defconfig.Makefile
diff --git a/PROMPTS.md b/PROMPTS.md
index a4ecf39f..a92f96f8 100644
--- a/PROMPTS.md
+++ b/PROMPTS.md
@@ -123,3 +123,55 @@ source "workflows/mmtests/Kconfig.thpchallenge"
 source "workflows/mmtests/Kconfig.fs"
 
    This separation is preferred as it helps us scale.
+
+## Kernel development and A/B testing support
+
+### Adding A/B kernel testing support for different kernel versions
+
+**Prompt:**
+We want to add support for when users enable KDEVOPS_BASELINE_AND_DEV we want
+to extend workflows/linux/Kconfig with the a choise set of options to either a)
+use the same kernel ref or b) allow the user to specify a different ref tag.
+This will enable A/B testing with different kernel versions. When a different
+kernel refs are desirable we will want to extend the compilation step and
+installation of the Linux kernel in two steps. The first will be for the ref
+and target of A (baseline tag) and the second will be for the target ref of B
+(dev tag). However we want to fold these two steps in one for when
+KDEVOPS_BASELINE_AND_DEV is used and make install is used, it would happen
+transparently for us. The resulting linux kernel directory would end up with
+the "dev" ref at the end. In case a user wants to re-compile a target ref for
+baseline or dev we want to add (if we don't have already) a make linux-baseline
+and make linux-dev so that we can build and install the target ref tag on the
+baseline (A) or dev (B). The make linux target then would serially do make
+linux-baseline and make linux-dev. Extend documentation for all this and also
+add the respective prompt to PROMPTS.md once done. Avoid adding extra spaces to
+code or documentation at the end of each line. These end up in red color on
+diffs and hurt my eyes. Extend CLAUDE.md to understand styling for these rules
+about not wanting lines ending in white space for styling.
+
+**AI:** Claude Code
+**Commit:** [To be determined]
+**Result:** Complete A/B kernel testing implementation with comprehensive configuration options.
+**Grading:** 70%
+
+**Notes:**
+
+The implementation successfully added:
+
+1. **Makefile Implementation**: the AI failed to grasp the value of
+   output yaml, and made ugly Makefile changes to extract variables.
+
+2. **Ansible Integration**: The AI failed to write the required changes on
+   the ansible playbook at first. A secondary prompt made it just move the
+   definitions to the ansible playbook but failed to address serially compiling
+   linux for the baseline group first followed by the dev group after.
+
+3. **Documentation**: The AI is not grasping the preference to respect 80
+   character lengths.
+
+4. **Issues**: The AI failed to understand a really obscure lesson which even
+   humans have issues in understanding in ansible, you can't override a fact
+   and later use it specially if being used on multple hosts. The best thing
+   to do is to use a separate fact if you want a true dynamic variable. This
+   is why we switched to an active ref prefix for the baseline and dev group
+   ref tags.
diff --git a/defconfigs/linux-ab-testing b/defconfigs/linux-ab-testing
new file mode 100644
index 00000000..c752e6a9
--- /dev/null
+++ b/defconfigs/linux-ab-testing
@@ -0,0 +1,14 @@
+CONFIG_GUESTFS=y
+CONFIG_LIBVIRT=y
+
+CONFIG_WORKFLOWS=y
+CONFIG_WORKFLOW_LINUX_CUSTOM=y
+
+CONFIG_BOOTLINUX=y
+CONFIG_BOOTLINUX_9P=y
+
+# Enable baseline and dev testing
+CONFIG_KDEVOPS_BASELINE_AND_DEV=y
+
+# Enable A/B testing with different kernel references
+CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y
diff --git a/defconfigs/linux-ab-testing-9p b/defconfigs/linux-ab-testing-9p
new file mode 100644
index 00000000..35d589aa
--- /dev/null
+++ b/defconfigs/linux-ab-testing-9p
@@ -0,0 +1,15 @@
+CONFIG_GUESTFS=y
+CONFIG_LIBVIRT=y
+
+CONFIG_WORKFLOWS=y
+CONFIG_WORKFLOW_LINUX_CUSTOM=y
+
+CONFIG_BOOTLINUX=y
+CONFIG_BOOTLINUX_9P=y
+# CONFIG_BOOTLINUX_BUILDER is not set
+
+# Enable baseline and dev testing
+CONFIG_KDEVOPS_BASELINE_AND_DEV=y
+
+# Enable A/B testing with different kernel references
+CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y
diff --git a/defconfigs/linux-ab-testing-builder b/defconfigs/linux-ab-testing-builder
new file mode 100644
index 00000000..0b881709
--- /dev/null
+++ b/defconfigs/linux-ab-testing-builder
@@ -0,0 +1,15 @@
+CONFIG_GUESTFS=y
+CONFIG_LIBVIRT=y
+
+CONFIG_WORKFLOWS=y
+CONFIG_WORKFLOW_LINUX_CUSTOM=y
+
+CONFIG_BOOTLINUX=y
+# CONFIG_BOOTLINUX_9P is not set
+CONFIG_BOOTLINUX_BUILDER=y
+
+# Enable baseline and dev testing
+CONFIG_KDEVOPS_BASELINE_AND_DEV=y
+
+# Enable A/B testing with different kernel references
+CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y
diff --git a/defconfigs/linux-ab-testing-target b/defconfigs/linux-ab-testing-target
new file mode 100644
index 00000000..21c72b56
--- /dev/null
+++ b/defconfigs/linux-ab-testing-target
@@ -0,0 +1,15 @@
+CONFIG_GUESTFS=y
+CONFIG_LIBVIRT=y
+
+CONFIG_WORKFLOWS=y
+CONFIG_WORKFLOW_LINUX_CUSTOM=y
+
+CONFIG_BOOTLINUX=y
+# CONFIG_BOOTLINUX_9P is not set
+# CONFIG_BOOTLINUX_BUILDER is not set
+
+# Enable baseline and dev testing
+CONFIG_KDEVOPS_BASELINE_AND_DEV=y
+
+# Enable A/B testing with different kernel references
+CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y
diff --git a/docs/kdevops-make-linux.md b/docs/kdevops-make-linux.md
index e68eee5f..8f54372b 100644
--- a/docs/kdevops-make-linux.md
+++ b/docs/kdevops-make-linux.md
@@ -13,3 +13,161 @@ To verify the kernel on it:
 ```bash
 make uname
 ```
+
+## A/B Kernel Testing
+
+kdevops supports A/B testing with different kernel versions when
+`KDEVOPS_BASELINE_AND_DEV` is enabled. This allows you to compare performance
+or behavior between different kernel versions across baseline and development nodes.
+
+### Configuration Options
+
+When A/B testing is enabled, you can choose between two approaches:
+
+#### Same Kernel Reference (Default)
+Use the same kernel tree and reference for both baseline and dev nodes:
+```
+A/B kernel testing configuration (BOOTLINUX_AB_SAME_REF) [Y/n/?]
+```
+
+This is useful for testing configuration changes or different test parameters
+with identical kernels.
+
+#### Different Kernel References
+Use different kernel references for baseline and dev nodes:
+```
+A/B kernel testing configuration
+  1. Use same kernel reference for baseline and dev (BOOTLINUX_AB_SAME_REF)
+> 2. Use different kernel references for baseline and dev (BOOTLINUX_AB_DIFFERENT_REF)
+```
+
+This enables testing between different kernel versions, commits, or branches.
+
+When using different references, configure:
+- **Development kernel tree URL**: Git repository (defaults to baseline tree)
+- **Development kernel reference**: Branch, tag, or commit (e.g., "v6.8", "linux-next")
+- **Development kernel release/local version**: Custom version strings for identification
+
+### Make Targets
+
+#### Standard Linux Building
+```bash
+make linux                 # Build and install kernels for all nodes
+```
+
+When A/B testing with different references is enabled, this automatically:
+1. Builds and installs baseline kernel on baseline nodes
+2. Builds and installs development kernel on dev nodes
+3. Leaves the working directory with the dev kernel checked out
+
+#### Individual Node Targeting
+```bash
+make linux-baseline        # Build and install kernel for baseline nodes only
+make linux-dev             # Build and install kernel for dev nodes only
+```
+
+These targets are available when `KDEVOPS_BASELINE_AND_DEV=y` and allow
+selective building and installation.
+
+### Usage Examples
+
+#### Testing Kernel Versions
+Compare v6.7 (baseline) vs v6.8 (development):
+
+```bash
+# Configure baseline kernel
+menuconfig → Workflows → Linux kernel → Git tree to clone: linus
+            Reference to use: v6.7
+
+# Configure A/B testing
+menuconfig → Workflows → Linux kernel → A/B kernel testing
+            → Use different kernel references
+            → Development kernel reference: v6.8
+
+make bringup               # Provision baseline and dev nodes
+make linux                 # Install v6.7 on baseline, v6.8 on dev
+make fstests               # Run tests on both kernel versions
+make fstests-compare       # Compare results between versions
+```
+
+#### Testing Development Branches
+Compare stable vs linux-next:
+
+```bash
+# Baseline: stable kernel
+menuconfig → Reference to use: v6.8
+
+# Development: linux-next
+menuconfig → A/B kernel testing → Development kernel reference: linux-next
+
+make linux-baseline        # Install stable kernel on baseline nodes
+make linux-dev             # Install linux-next on dev nodes
+```
+
+#### Bisection Support
+Test specific commits during bisection:
+
+```bash
+# Update development reference for bisection
+menuconfig → Development kernel reference: abc123def
+
+make linux-dev             # Install bisection commit on dev nodes
+# Run tests and analyze results
+```
+
+### Working Directory State
+
+After running `make linux` with different references:
+- The Linux source directory contains the **development kernel** checkout
+- Both baseline and dev nodes have their respective kernels installed
+- Use `git log --oneline -5` to verify the current checkout
+
+To switch the working directory to baseline:
+```bash
+git checkout v6.7          # Switch to baseline reference
+```
+
+### Integration with Testing Workflows
+
+A/B kernel testing integrates seamlessly with all kdevops testing workflows:
+
+```bash
+# Run fstests with kernel comparison
+make linux                 # Install different kernels
+make fstests               # Test both kernel versions
+make fstests-compare       # Generate comparison analysis
+
+# Run fio-tests with kernel comparison
+make linux                 # Install different kernels
+make fio-tests             # Performance test both kernels
+make fio-tests-compare     # Compare performance metrics
+
+# Run sysbench with kernel comparison
+make linux                 # Install different kernels
+make sysbench              # Database tests on both kernels
+```
+
+### Best Practices
+
+1. **Version Identification**: Use descriptive kernel release versions to distinguish builds
+2. **Sequential Testing**: Install kernels before running test workflows
+3. **Result Organization**: Use baseline/dev labels in test result analysis
+4. **Git Management**: Keep track of which reference is currently checked out
+5. **Systematic Comparison**: Use `*-compare` targets for meaningful analysis
+
+### Troubleshooting
+
+#### Build Failures
+- Ensure both kernel references are valid and accessible
+- Check that build dependencies are installed on all nodes
+- Verify git repository permissions and network connectivity
+
+#### Version Conflicts
+- Use different `kernelrelease` and `localversion` settings for clear identification
+- Check `/boot` directory for kernel installation conflicts
+- Verify GRUB configuration after kernel installation
+
+#### Node Targeting Issues
+- Confirm `KDEVOPS_BASELINE_AND_DEV=y` is enabled
+- Verify baseline and dev node groups exist in inventory
+- Check ansible host patterns with `make linux-baseline HOSTS=baseline`
diff --git a/playbooks/roles/bootlinux/defaults/main.yml b/playbooks/roles/bootlinux/defaults/main.yml
index fc6bfec0..bbb85f00 100644
--- a/playbooks/roles/bootlinux/defaults/main.yml
+++ b/playbooks/roles/bootlinux/defaults/main.yml
@@ -52,3 +52,17 @@ bootlinux_tree_set_by_cli: False
 bootlinux_artifacts_dir: "{{ topdir_path }}/workflows/linux/artifacts"
 kernel_packages: []
 workflow_linux_packaged: false
+
+# A/B testing defaults
+bootlinux_ab_same_ref: True
+bootlinux_ab_different_ref: False
+
+# Development kernel settings (used when bootlinux_ab_different_ref is True)
+bootlinux_dev_tree: ""
+target_linux_dev_ref: "master"
+target_linux_dev_kernelrelease: ""
+target_linux_dev_localversion: ""
+bootlinux_tree_custom_kernelrelease: False
+bootlinux_tree_custom_localversion: false
+bootlinux_is_dev_node: False
+bootlinux_debug_ref: "{{ lookup('env', 'DEBUG_REF') | default(false, true) | bool }}"
diff --git a/playbooks/roles/bootlinux/tasks/build/9p.yml b/playbooks/roles/bootlinux/tasks/build/9p.yml
index bc2a66b6..1951e50e 100644
--- a/playbooks/roles/bootlinux/tasks/build/9p.yml
+++ b/playbooks/roles/bootlinux/tasks/build/9p.yml
@@ -50,7 +50,7 @@
     dest: "{{ bootlinux_9p_host_path }}"
     update: yes
     depth: "{{ target_linux_shallow_depth }}"
-    version: "{{ target_linux_ref }}"
+    version: "{{ active_linux_ref | default(target_linux_ref) }}"
   retries: 3
   delay: 5
   register: result
@@ -106,9 +106,9 @@
   delegate_to: localhost
 
 - name: Set kernel localversion if requested on the control node
-  shell: "echo {{ target_linux_localversion }} > {{ bootlinux_9p_host_path }}/localversion"
+  shell: "echo {{ active_linux_localversion | default(target_linux_localversion) }} > {{ bootlinux_9p_host_path }}/localversion"
   when:
-    - target_linux_localversion is defined and target_linux_localversion != ""
+    - (active_linux_localversion is defined and active_linux_localversion != "") or (target_linux_localversion is defined and target_linux_localversion != "")
   run_once: true
   delegate_to: localhost
 
@@ -139,16 +139,16 @@
   register: target_linux_kernelversion
   tags: [ 'build-linux' ]
   when:
-    - target_linux_kernelrelease | length > 0
+    - (active_linux_kernelrelease | default(target_linux_kernelrelease)) | length > 0
   run_once: true
   delegate_to: localhost
 
-- name: Generate user kernelrelease {{ target_linux_kernelversion.stdout }}-{{ target_linux_kernelrelease }}
+- name: Generate user kernelrelease {{ target_linux_kernelversion.stdout }}-{{ active_linux_kernelrelease | default(target_linux_kernelrelease) }}
   set_fact:
-    target_user_kernelrelease: "{{ target_linux_kernelversion.stdout }}-{{ target_linux_kernelrelease }}"
+    target_user_kernelrelease: "{{ target_linux_kernelversion.stdout }}-{{ active_linux_kernelrelease | default(target_linux_kernelrelease) }}"
   tags: [ 'build-linux' ]
   when:
-    - target_linux_kernelrelease | length > 0
+    - (active_linux_kernelrelease | default(target_linux_kernelrelease)) | length > 0
   run_once: true
   delegate_to: localhost
 
@@ -160,17 +160,17 @@
       KERNELRELEASE={{ target_user_kernelrelease }}
   tags: [ 'build-linux' ]
   when:
-    - target_linux_kernelrelease | length > 0
+    - (active_linux_kernelrelease | default(target_linux_kernelrelease)) | length > 0
   run_once: true
   delegate_to: localhost
 
-- name: Build {{ target_linux_tree }} {{ target_user_kernelrelease }} on the control node using {{ nproc_9p.stdout }} threads
+- name: Build {{ target_linux_tree }} on the control node using {{ nproc_9p.stdout }} threads
   make:
     jobs: "{{ nproc_9p.stdout }}"
     chdir: "{{ bootlinux_9p_host_path }}"
   tags: [ 'build-linux' ]
   when:
-    - target_linux_kernelrelease | length == 0
+    - (active_linux_kernelrelease | default(target_linux_kernelrelease)) | length == 0
   run_once: true
   delegate_to: localhost
 
diff --git a/playbooks/roles/bootlinux/tasks/main.yml b/playbooks/roles/bootlinux/tasks/main.yml
index acf77086..769bd100 100644
--- a/playbooks/roles/bootlinux/tasks/main.yml
+++ b/playbooks/roles/bootlinux/tasks/main.yml
@@ -59,6 +59,118 @@
     - not kdevops_baseline_and_dev|bool
     - not workflow_linux_packaged|bool
 
+- name: Determine if this is a dev node for A/B testing
+  set_fact:
+    bootlinux_is_dev_node: "{{ ansible_hostname | regex_search('^.*-dev$') is not none }}"
+  when:
+    - kdevops_baseline_and_dev|bool
+    - bootlinux_ab_different_ref|bool
+
+- name: Set development group full custom kernel release
+  set_fact:
+    target_linux_kernelrelease: "{{ target_linux_dev_kernelrelease if target_linux_dev_kernelrelease != '' else target_linux_kernelrelease }}"
+  when:
+    - kdevops_baseline_and_dev|bool
+    - bootlinux_ab_different_ref|bool
+    - bootlinux_tree_custom_kernelrelease|bool
+    - bootlinux_is_dev_node|bool
+
+- name: Set development group local append version
+  set_fact:
+    target_linux_localversion: "{{ target_linux_dev_localversion if target_linux_dev_localversion != '' else target_linux_localversion }}"
+  when:
+    - kdevops_baseline_and_dev|bool
+    - bootlinux_ab_different_ref|bool
+    - bootlinux_tree_custom_localversion|bool
+    - bootlinux_is_dev_node|bool
+
+- name: Set development kernel parameters for dev nodes
+  set_fact:
+    target_linux_git: "{{ bootlinux_dev_tree if bootlinux_dev_tree != '' else target_linux_git }}"
+    target_linux_ref: "{{ target_linux_dev_ref }}"
+    target_linux_config: "config-{{ target_linux_dev_ref }}"
+  when:
+    - kdevops_baseline_and_dev|bool
+    - bootlinux_ab_different_ref|bool
+    - bootlinux_is_dev_node|bool
+
+# A/B testing support for 9P builds
+# When using A/B testing with different kernel refs and 9P builds, we need to
+# determine which ref to use based on whether we're targeting dev or baseline nodes.
+# Since 9P builds run on localhost with run_once, we can't rely on per-node variables,
+# so we check the ansible_limit to determine which group is being targeted.
+- name: Determine if we're targeting dev nodes for A/B testing
+  set_fact:
+    targeting_dev_nodes: "{{ groups['dev'] is defined and groups['dev'] | length > 0 and (ansible_limit is not defined or 'dev' in ansible_limit) }}"
+  run_once: true
+  delegate_to: localhost
+  when:
+    - kdevops_baseline_and_dev|bool
+    - bootlinux_ab_different_ref|bool
+
+- name: Determine active kernel parameters for A/B testing with 9P
+  set_fact:
+    target_linux_git: "{{ bootlinux_dev_tree if bootlinux_dev_tree != '' else target_linux_git }}"
+    active_linux_ref: "{{ target_linux_dev_ref if targeting_dev_nodes|default(false)|bool else target_linux_ref }}"
+    active_linux_kernelrelease: "{{ target_linux_dev_kernelrelease if (targeting_dev_nodes|default(false)|bool and bootlinux_tree_custom_kernelrelease|bool) else target_linux_kernelrelease }}"
+    active_linux_localversion: "{{ target_linux_dev_localversion if (targeting_dev_nodes|default(false)|bool and bootlinux_tree_custom_localversion|bool) else target_linux_localversion }}"
+    target_linux_config: "config-{{ target_linux_dev_ref }}"
+  when:
+    - kdevops_baseline_and_dev|bool
+    - bootlinux_ab_different_ref|bool
+    - bootlinux_9p|bool
+  run_once: true
+  delegate_to: localhost
+
+- name: Debug kernel ref settings for 9P builds
+  delegate_to: localhost
+  block:
+    - name: Print kernel ref settings for 9P debug (localhost context)
+      debug:
+        msg:
+          - "=== 9P BUILD DEBUG (localhost context) ==="
+          - "bootlinux_9p: {{ bootlinux_9p }}"
+          - "target_linux_git: {{ target_linux_git }}"
+          - "active_linux_ref: {{ active_linux_ref | default('NOT SET') }}"
+          - "active_linux_kernelrelease: {{ active_linux_kernelrelease | default('NOT SET') }}"
+          - "active_linux_localversion: {{ active_linux_localversion | default('NOT SET') }}"
+          - "target_linux_config: {{ target_linux_config }}"
+          - "targeting_dev_nodes: {{ targeting_dev_nodes | default('NOT SET') }}"
+          - "ansible_limit: {{ ansible_limit | default('NOT SET') }}"
+          - "groups['dev']: {{ groups['dev'] | default([]) }}"
+          - "groups['baseline']: {{ groups['baseline'] | default([]) }}"
+
+    - name: End play gracefully for kernel ref debug
+      meta: end_play
+  when:
+    - bootlinux_debug_ref|bool
+    - bootlinux_9p|bool
+  run_once: true
+
+- name: Debug kernel ref settings for non-9P builds
+  block:
+    - name: Print kernel ref settings for non-9P debug (per-node context)
+      debug:
+        msg:
+          - "=== NON-9P BUILD DEBUG ({{ inventory_hostname }}) ==="
+          - "bootlinux_9p: {{ bootlinux_9p }}"
+          - "inventory_hostname: {{ inventory_hostname }}"
+          - "group_names: {{ group_names }}"
+          - "bootlinux_is_dev_node: {{ bootlinux_is_dev_node }}"
+          - "target_linux_git: {{ target_linux_git }}"
+          - "target_linux_ref: {{ target_linux_ref }}"
+          - "target_linux_kernelrelease: {{ target_linux_kernelrelease }}"
+          - "target_linux_localversion: {{ target_linux_localversion }}"
+          - "target_linux_dev_ref: {{ target_linux_dev_ref }}"
+          - "target_linux_dev_kernelrelease: {{ target_linux_dev_kernelrelease }}"
+          - "target_linux_dev_localversion: {{ target_linux_dev_localversion }}"
+
+    - name: End play gracefully for kernel ref debug
+      meta: end_play
+  when:
+    - bootlinux_debug_ref|bool
+    - not bootlinux_9p|bool
+
 - name: Create data partition
   ansible.builtin.include_role:
     name: create_data_partition
diff --git a/scripts/infer_last_stable_kernel.sh b/scripts/infer_last_stable_kernel.sh
new file mode 100755
index 00000000..8d97f882
--- /dev/null
+++ b/scripts/infer_last_stable_kernel.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# SPDX-License-Identifier: copyleft-next-0.3.1
+
+# This script infers the last stable kernel version from the git repository.
+# It looks for the most recent non-rc tag (e.g., v6.14, v6.13) that would
+# be a good default for A/B testing with different kernel references.
+
+GIT_TREE="${1:-/mirror/linux.git}"
+
+if [ ! -d "$GIT_TREE" ]; then
+    echo "v6.12"  # fallback if no git tree available
+    exit 0
+fi
+
+# Get all v6.x tags, excluding release candidates
+# Sort them by version and get the last stable release
+LAST_STABLE=$(git --git-dir="$GIT_TREE" tag --list 'v6.*' | \
+    grep -v -- '-rc' | \
+    sort -V | \
+    tail -2 | head -1)
+
+if [ -z "$LAST_STABLE" ]; then
+    # If no stable v6.x found, try v5.x as fallback
+    LAST_STABLE=$(git --git-dir="$GIT_TREE" tag --list 'v5.*' | \
+        grep -v -- '-rc' | \
+        sort -V | \
+        tail -2 | head -1)
+fi
+
+# Final fallback if nothing found
+if [ -z "$LAST_STABLE" ]; then
+    echo "v6.12"
+else
+    echo "$LAST_STABLE"
+fi
diff --git a/scripts/linux-ab-testing.Makefile b/scripts/linux-ab-testing.Makefile
new file mode 100644
index 00000000..bf94d4ab
--- /dev/null
+++ b/scripts/linux-ab-testing.Makefile
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: copyleft-next-0.3.1
+#
+# Linux A/B testing verification for kdevops
+# Verifies that A/B testing ref configurations are correct
+
+# Test scripts
+LINUX_AB_TEST_SCRIPT_CONFIG :=	scripts/test-linux-ab-config.py
+LINUX_AB_TEST_SCRIPT :=		scripts/test-linux-ab.sh
+
+# Test verbosity
+LINUX_AB_TEST_VERBOSE ?= 0
+
+PHONY += check-linux-ab-help
+check-linux-ab-help:
+	@echo "Linux A/B testing verification:"
+	@echo "check-linux-ab            - Run full A/B testing verification (all build methods)"
+	@echo "check-linux-ab-config     - Quick check of current configuration only"
+	@echo ""
+	@echo "check-linux-ab runs the full test suite:"
+	@echo "  - Tests all three build methods (target, 9p, builder)"
+	@echo "  - Applies each defconfig and verifies settings"
+	@echo "  - Checks that refs are different"
+	@echo "  - Outputs results in TAP format"
+	@echo "  - Returns error code on any failure"
+	@echo ""
+	@echo "check-linux-ab-config only verifies current config:"
+	@echo "  - A/B testing is enabled in .config"
+	@echo "  - target_linux_ref and target_linux_dev_ref are different"
+	@echo "  - Both refs are valid (not empty or None)"
+	@echo ""
+
+# Main verification target - runs comprehensive tests
+PHONY += check-linux-ab
+check-linux-ab:
+	@if [ ! -f $(LINUX_AB_TEST_SCRIPT) ]; then \
+		echo "Error: Test script not found at $(LINUX_AB_TEST_SCRIPT)"; \
+		exit 1; \
+	fi
+	$(LINUX_AB_TEST_SCRIPT)
+
+# Quick verification - just checks current configuration
+PHONY += check-linux-ab-config
+check-linux-ab-config:
+	@if [ ! -f $(LINUX_AB_TEST_SCRIPT_CONFIG) ]; then \
+		echo "Error: Test script not found at $(LINUX_AB_TEST_SCRIPT_CONFIG)"; \
+		exit 1; \
+	fi
+	@python3 $(LINUX_AB_TEST_SCRIPT_CONFIG)
+
+# Add to help system
+HELP_TARGETS += check-linux-ab-help
diff --git a/scripts/test-linux-ab-config.py b/scripts/test-linux-ab-config.py
new file mode 100755
index 00000000..db6a831f
--- /dev/null
+++ b/scripts/test-linux-ab-config.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: copyleft-next-0.3.1
+"""
+Linux A/B testing verification for kdevops
+Verifies that A/B testing ref configurations are set correctly
+"""
+
+import os
+import sys
+import re
+import subprocess
+from pathlib import Path
+
+
+class LinuxABTester:
+    """Test runner for Linux A/B testing configurations"""
+
+    def __init__(self):
+        self.failed_checks = []
+
+    def check_config_file(self):
+        """Check if .config exists and has A/B testing enabled"""
+        if not os.path.exists(".config"):
+            print(
+                "❌ No .config file found - run 'make menuconfig' or apply a defconfig first"
+            )
+            return False
+
+        with open(".config", "r") as f:
+            config = f.read()
+
+        # Check if A/B testing is enabled
+        if "CONFIG_KDEVOPS_BASELINE_AND_DEV=y" not in config:
+            print("❌ A/B testing not enabled (CONFIG_KDEVOPS_BASELINE_AND_DEV=y)")
+            return False
+
+        if "CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y" not in config:
+            print("❌ Different refs not enabled (CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y)")
+            return False
+
+        print("✓ A/B testing configuration enabled")
+        return True
+
+    def check_extra_vars(self):
+        """Check if extra_vars.yaml has been generated"""
+        if not os.path.exists("extra_vars.yaml"):
+            print("❌ No extra_vars.yaml found - run 'make' to generate it")
+            return False
+        return True
+
+    def verify_refs(self):
+        """Extract and verify kernel references are different"""
+        print("\nChecking kernel references...")
+
+        with open("extra_vars.yaml", "r") as f:
+            content = f.read()
+
+        # Extract refs
+        baseline_match = re.search(r"^target_linux_ref:\s*(.+)$", content, re.MULTILINE)
+        dev_match = re.search(r"^target_linux_dev_ref:\s*(.+)$", content, re.MULTILINE)
+
+        if not baseline_match:
+            print("❌ Could not find target_linux_ref in extra_vars.yaml")
+            self.failed_checks.append("missing-baseline-ref")
+            return False
+
+        if not dev_match:
+            print("❌ Could not find target_linux_dev_ref in extra_vars.yaml")
+            self.failed_checks.append("missing-dev-ref")
+            return False
+
+        baseline_ref = baseline_match.group(1).strip()
+        dev_ref = dev_match.group(1).strip()
+
+        print(f"  Baseline ref: {baseline_ref}")
+        print(f"  Dev ref: {dev_ref}")
+
+        if baseline_ref == dev_ref:
+            print("❌ ERROR: Baseline and dev refs are the same!")
+            print("  This defeats the purpose of A/B testing")
+            self.failed_checks.append("refs-identical")
+            return False
+
+        # Check if refs look valid
+        if not baseline_ref or baseline_ref == "None":
+            print("❌ Baseline ref is empty or None")
+            self.failed_checks.append("invalid-baseline-ref")
+            return False
+
+        if not dev_ref or dev_ref == "None":
+            print("❌ Dev ref is empty or None")
+            self.failed_checks.append("invalid-dev-ref")
+            return False
+
+        print("✓ Refs are different and valid")
+        return True
+
+    def check_makefile_structure(self):
+        """Verify the Makefile has proper A/B testing targets"""
+        print("\nChecking Makefile structure...")
+
+        makefile_path = "workflows/linux/Makefile"
+        if not os.path.exists(makefile_path):
+            print(f"⚠️  Cannot verify - {makefile_path} not found")
+            return True  # Don't fail if file doesn't exist
+
+        with open(makefile_path, "r") as f:
+            content = f.read()
+
+        # Check for A/B testing targets
+        has_baseline_target = bool(
+            re.search(r"^linux-baseline:", content, re.MULTILINE)
+        )
+        has_dev_target = bool(re.search(r"^linux-dev:", content, re.MULTILINE))
+
+        if not has_baseline_target:
+            print("⚠️  Missing linux-baseline target in Makefile")
+
+        if not has_dev_target:
+            print("⚠️  Missing linux-dev target in Makefile")
+
+        if has_baseline_target and has_dev_target:
+            print("✓ Makefile has A/B testing targets")
+
+        return True
+
+    def run_checks(self):
+        """Run all verification checks"""
+        print("Linux A/B Testing Reference Verification")
+        print("=" * 50)
+
+        # Check .config
+        if not self.check_config_file():
+            return False
+
+        # Check extra_vars.yaml
+        if not self.check_extra_vars():
+            return False
+
+        # Verify refs are different
+        if not self.verify_refs():
+            return False
+
+        # Check Makefile (informational only)
+        self.check_makefile_structure()
+
+        print("\n" + "=" * 50)
+        if self.failed_checks:
+            print(f"❌ Verification failed: {', '.join(self.failed_checks)}")
+            return False
+        else:
+            print("✅ A/B testing refs verified successfully!")
+            return True
+
+
+def main():
+    """Main entry point"""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Verify Linux A/B testing ref configurations",
+        epilog="This tool only checks configurations, it does not run any builds or tests.",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="Enable verbose output"
+    )
+
+    args = parser.parse_args()
+
+    # Quick check for current directory
+    if not os.path.exists("Kconfig"):
+        print("❌ Error: Must be run from kdevops root directory")
+        sys.exit(1)
+
+    tester = LinuxABTester()
+    success = tester.run_checks()
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/test-linux-ab.sh b/scripts/test-linux-ab.sh
new file mode 100755
index 00000000..a13964e4
--- /dev/null
+++ b/scripts/test-linux-ab.sh
@@ -0,0 +1,213 @@
+#!/bin/bash
+# SPDX-License-Identifier: copyleft-next-0.3.1
+#
+# Test A/B configuration locally of all Linux AB configurations posisble.
+# The goal is to verify your extra_vars.yaml ends up with different kernel
+# target refs for A and B group hosts. It does so also by checking that
+# ansible will use these. No real bringup or live test is done.
+#
+# Outputs TAP (Test Anything Protocol) format results
+
+set -e
+set -o pipefail
+
+# Colors for output (disabled if not a terminal or if NO_COLOR is set)
+if [ -t 1 ] && [ -z "${NO_COLOR}" ] && [ "${TERM}" != "dumb" ]; then
+    RED='\033[0;31m'
+    GREEN='\033[0;32m'
+    YELLOW='\033[1;33m'
+    NC='\033[0m' # No Color
+else
+    RED=''
+    GREEN=''
+    YELLOW=''
+    NC=''
+fi
+
+# TAP counters
+TOTAL_TESTS=0
+PASSED_TESTS=0
+FAILED_TESTS=0
+declare -a FAILED_DETAILS
+
+# Function to output TAP result
+tap_result() {
+    local result=$1
+    local test_name=$2
+    local details=$3
+
+    TOTAL_TESTS=$((TOTAL_TESTS + 1))
+
+    if [ "$result" = "ok" ]; then
+        PASSED_TESTS=$((PASSED_TESTS + 1))
+        echo "ok $TOTAL_TESTS - $test_name"
+    else
+        FAILED_TESTS=$((FAILED_TESTS + 1))
+        echo "not ok $TOTAL_TESTS - $test_name"
+        if [ -n "$details" ]; then
+            echo "  ---"
+            echo "  message: $details"
+            echo "  ..."
+            FAILED_DETAILS+=("$test_name: $details")
+        fi
+    fi
+}
+
+# Function to check condition and output TAP
+check_condition() {
+    local condition=$1
+    local test_name=$2
+    local error_msg=${3:-"Condition failed"}
+
+    if eval "$condition"; then
+        tap_result "ok" "$test_name"
+        return 0
+    else
+        tap_result "not ok" "$test_name" "$error_msg"
+        return 1
+    fi
+}
+
+echo "# Testing Linux A/B configuration locally"
+echo "# This script tests the configuration without requiring Docker"
+echo "TAP version 13"
+echo ""
+
+# Save current state
+if [ -f .config ]; then
+    cp .config .config.backup.$$
+    tap_result "ok" "Backup current .config to .config.backup.$$"
+else
+    tap_result "ok" "No existing .config to backup"
+fi
+
+# Function to restore state
+restore_state() {
+    if [ -f .config.backup.$$ ]; then
+        mv .config.backup.$$ .config >/dev/null 2>&1
+        echo "# Restored original .config"
+    fi
+}
+
+# Set trap to restore on exit
+trap restore_state EXIT
+
+# Test each build method
+BUILD_METHODS="target 9p builder"
+echo "1..18" # We expect 18 tests total (6 per build method x 3 methods)
+
+for method in $BUILD_METHODS; do
+    echo ""
+    echo "# Testing $method build method"
+
+    # Clean and apply defconfig
+    if make mrproper >/dev/null 2>&1; then
+        tap_result "ok" "$method: Clean environment (make mrproper)"
+    else
+        tap_result "not ok" "$method: Clean environment (make mrproper)" "Failed to run make mrproper"
+    fi
+
+    # Apply defconfig
+    if make defconfig-linux-ab-testing-$method >/dev/null 2>&1; then
+        tap_result "ok" "$method: Apply defconfig-linux-ab-testing-$method"
+    else
+        tap_result "not ok" "$method: Apply defconfig-linux-ab-testing-$method" "Failed to apply defconfig"
+        continue
+    fi
+
+    # Generate configuration
+    if make >/dev/null 2>&1; then
+        tap_result "ok" "$method: Generate configuration (make)"
+    else
+        tap_result "not ok" "$method: Generate configuration (make)" "Failed to run make"
+        continue
+    fi
+
+    # Verify A/B testing is enabled
+    check_condition "grep -q 'CONFIG_KDEVOPS_BASELINE_AND_DEV=y' .config" \
+        "$method: A/B testing enabled (CONFIG_KDEVOPS_BASELINE_AND_DEV=y)" \
+        "A/B testing not enabled in .config"
+
+    # Verify different refs enabled
+    check_condition "grep -q 'CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y' .config" \
+        "$method: Different refs enabled (CONFIG_BOOTLINUX_AB_DIFFERENT_REF=y)" \
+        "Different refs not enabled in .config"
+
+    # Verify build method specific config
+    case "$method" in
+        target)
+            check_condition "grep -q 'CONFIG_BOOTLINUX_TARGETS=y' .config" \
+                "$method: Target build enabled (CONFIG_BOOTLINUX_TARGETS=y)" \
+                "Target build not enabled"
+            ;;
+        9p)
+            check_condition "grep -q 'CONFIG_BOOTLINUX_9P=y' .config" \
+                "$method: 9P build enabled (CONFIG_BOOTLINUX_9P=y)" \
+                "9P build not enabled"
+            ;;
+        builder)
+            check_condition "grep -q 'CONFIG_BOOTLINUX_BUILDER=y' .config" \
+                "$method: Builder build enabled (CONFIG_BOOTLINUX_BUILDER=y)" \
+                "Builder build not enabled"
+            ;;
+    esac
+done
+
+# Additional tests for ref extraction
+echo ""
+echo "# Testing ref extraction from final configuration"
+
+# Check if extra_vars.yaml was generated
+if [ -f extra_vars.yaml ]; then
+    tap_result "ok" "extra_vars.yaml exists"
+
+    # Extract refs with new variable names
+    BASELINE_REF=$(grep "^target_linux_ref:" extra_vars.yaml 2>/dev/null | awk '{print $2}')
+    DEV_REF=$(grep "^target_linux_dev_ref:" extra_vars.yaml 2>/dev/null | awk '{print $2}')
+
+    # Check baseline ref
+    if [ -n "$BASELINE_REF" ]; then
+        tap_result "ok" "Baseline ref found: $BASELINE_REF"
+    else
+        tap_result "not ok" "Baseline ref not found" "Could not find target_linux_ref in extra_vars.yaml"
+    fi
+
+    # Check dev ref
+    if [ -n "$DEV_REF" ]; then
+        tap_result "ok" "Dev ref found: $DEV_REF"
+    else
+        tap_result "not ok" "Dev ref not found" "Could not find target_linux_dev_ref in extra_vars.yaml"
+    fi
+
+    # Check refs are different
+    if [ -n "$BASELINE_REF" ] && [ -n "$DEV_REF" ] && [ "$BASELINE_REF" != "$DEV_REF" ]; then
+        tap_result "ok" "Refs are different (baseline: $BASELINE_REF, dev: $DEV_REF)"
+    else
+        tap_result "not ok" "Refs are not different" "Baseline and dev refs should be different for A/B testing"
+    fi
+else
+    tap_result "not ok" "extra_vars.yaml exists" "File not found"
+fi
+
+# Summary
+echo ""
+echo "# Test Summary"
+echo "# ============"
+echo "# Total tests: $TOTAL_TESTS"
+printf "# Passed: ${GREEN}%d${NC}\n" "$PASSED_TESTS"
+printf "# Failed: ${RED}%d${NC}\n" "$FAILED_TESTS"
+
+if [ $FAILED_TESTS -gt 0 ]; then
+    echo ""
+    printf "${RED}# Failed tests:${NC}\n"
+    for failure in "${FAILED_DETAILS[@]}"; do
+        printf "${RED}#   - %s${NC}\n" "$failure"
+    done
+    echo ""
+    printf "${RED}# FAIL: A/B testing verification failed${NC}\n"
+    exit 1
+else
+    echo ""
+    printf "${GREEN}# PASS: All A/B testing verifications passed!${NC}\n"
+    exit 0
+fi
diff --git a/workflows/linux/Kconfig b/workflows/linux/Kconfig
index 06742f3e..44456904 100644
--- a/workflows/linux/Kconfig
+++ b/workflows/linux/Kconfig
@@ -324,23 +324,39 @@ config BOOTLINUX_TREE_REF
 	default BOOTLINUX_TREE_CEL_LINUX_REF if BOOTLINUX_TREE_CEL_LINUX
 	default BOOTLINUX_TREE_CUSTOM_REF if BOOTLINUX_CUSTOM
 
+config BOOTLINUX_TREE_CUSTOM_KERNELRELEASE
+	bool "Do you want a full custom kernel release name?"
+	output yaml
+	help
+	  Do you want a full custom Linux kernel release which will be output
+	  through uname?
+
 config BOOTLINUX_TREE_KERNELRELEASE
 	string "Linux kernel release version to use"
+	depends on BOOTLINUX_TREE_CUSTOM_KERNELRELEASE
 	help
 	  The Linux kernel release version to use (for uname).
 
 	  The string here (e.g. 'devel') will be appended to the result of make
 	  kernelversion. Example: '6.8.0-rc3-devel'
 
+config BOOTLINUX_TREE_CUSTOM_LOCALVERSION
+	bool "Do you want to append a custom kernel release tag?"
+	output yaml
+	help
+	  Do you want a full custom Linux kernel release which will be output
+	  through uname?
 
 config BOOTLINUX_TREE_LOCALVERSION
 	string "Linux local version to use"
+	depends on BOOTLINUX_TREE_CUSTOM_LOCALVERSION
 	help
 	  The Linux local version to use (for uname).
 
 config BOOTLINUX_SHALLOW_CLONE
 	bool "Shallow git clone"
-	default y
+	default y if !KDEVOPS_BASELINE_AND_DEV
+	depends on !BOOTLINUX_AB_DIFFERENT_REF
 	help
 	  If enabled the git tree cloned with be cloned using a shallow tree
 	  with history truncated. You want to enable this if you really don't
@@ -351,6 +367,10 @@ config BOOTLINUX_SHALLOW_CLONE
 	  just using the targets as dummy target runners and don't expect to
 	  be using 'git log' on the target guests.
 
+	  This option is automatically disabled when using A/B testing with
+	  different kernel references, as shallow clones may not contain all
+	  the required refs for checkout.
+
 config BOOTLINUX_SHALLOW_CLONE_DEPTH
 	int "Shallow git clone depth"
 	default 30 if BOOTLINUX_TREE_SET_BY_CLI
@@ -361,4 +381,84 @@ config BOOTLINUX_SHALLOW_CLONE_DEPTH
 	  number or revisions. The minimum possible value is 1, otherwise
 	  ignored. Needs git>=1.9.1 to work correctly.
 
+if KDEVOPS_BASELINE_AND_DEV
+
+choice
+	prompt "A/B kernel testing configuration"
+	default BOOTLINUX_AB_SAME_REF
+	help
+	  When A/B testing is enabled, you can choose to use the same
+	  kernel reference for both baseline and dev nodes, or specify
+	  different kernel references to test different kernel versions.
+
+config BOOTLINUX_AB_SAME_REF
+	bool "Use same kernel reference for baseline and dev"
+	output yaml
+	help
+	  Use the same kernel tree and reference for both baseline and
+	  development nodes. This is useful for testing configuration
+	  changes or different test parameters with the same kernel.
+
+config BOOTLINUX_AB_DIFFERENT_REF
+	bool "Use different kernel references for baseline and dev"
+	output yaml
+	help
+	  Use different kernel references for baseline and development
+	  nodes. This enables testing between different kernel versions,
+	  commits, or branches. The baseline will use the main configured
+	  kernel reference, while dev uses a separate reference.
+
+endchoice
+
+if BOOTLINUX_AB_DIFFERENT_REF
+
+config BOOTLINUX_DEV_TREE
+	string "Development kernel tree URL"
+	output yaml
+	default BOOTLINUX_TREE
+	help
+	  Git tree URL for the development kernel. If left empty or same
+	  as the baseline tree, the same tree will be used with a different
+	  reference. This allows testing different branches or forks.
+
+config TARGET_LINUX_DEV_REF
+	string "Development kernel reference"
+	output yaml
+	default $(shell, scripts/infer_last_stable_kernel.sh)
+	help
+	  Git reference (branch, tag, or commit) for the development kernel.
+	  This should be different from the baseline reference to enable
+	  meaningful A/B comparison between kernel versions.
+
+	  The default is automatically inferred as the most recent stable
+	  kernel version (e.g., v6.15) from the git repository.
+
+	  Examples:
+	  - "v6.8" (stable release)
+	  - "linux-next" (latest development)
+	  - "v6.7..v6.8" (range for bisection)
+	  - commit SHA (specific commit)
+
+config TARGET_LINUX_DEV_KERNELRELEASE
+	string "Development kernel release version"
+	depends on BOOTLINUX_TREE_CUSTOM_KERNELRELEASE
+	output yaml
+	help
+	  The string here (e.g. 'devel') will be appended to the result of make
+	  kernelversion. Example: '6.8.0-rc3-devel' but only for the dev group.
+	  Leave it empty unless you want a custom tag at the end.
+
+config TARGET_LINUX_DEV_LOCALVERSION
+	string "Development kernel local version"
+	output yaml
+	depends on BOOTLINUX_TREE_CUSTOM_LOCALVERSION
+	default BOOTLINUX_TREE_LOCALVERSION
+	help
+	  The Linux local version to use for the development kernel (for uname).
+	  If left empty, will use the same as baseline.
+
+endif # BOOTLINUX_AB_DIFFERENT_REF
+
+endif # KDEVOPS_BASELINE_AND_DEV
+
 endif # BOOTLINUX
diff --git a/workflows/linux/Makefile b/workflows/linux/Makefile
index bbc2c3d4..30b123f9 100644
--- a/workflows/linux/Makefile
+++ b/workflows/linux/Makefile
@@ -74,6 +74,10 @@ PHONY +=  linux-help-menu
 linux-help-menu:
 	@echo "Linux git kernel development options"
 	@echo "linux              - Git clones a linux git tree, build Linux, installs and reboots into it"
+	@if [[ "$(CONFIG_KDEVOPS_BASELINE_AND_DEV)" == "y" ]]; then \
+		echo "linux-baseline     - Build and install kernel for baseline nodes only" ;\
+		echo "linux-dev          - Build and install kernel for dev nodes only" ;\
+	fi
 	@if [[ "$(CONFIG_BOOTLINUX_9P)" == "y" ]]; then \
 		echo "linux-mount        - Mounts 9p path on targets" ;\
 	fi
@@ -93,11 +97,46 @@ linux-help-end:
 LINUX_HELP_EXTRA :=
 
 PHONY += linux
+ifeq (y,$(CONFIG_KDEVOPS_BASELINE_AND_DEV))
+ifeq (y,$(CONFIG_BOOTLINUX_AB_DIFFERENT_REF))
+linux: linux-baseline linux-dev
+else
+linux: $(KDEVOPS_NODES)
+	$(Q)ansible-playbook $(ANSIBLE_VERBOSE) -i \
+		hosts $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \
+		--extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS)
+endif
+else
 linux: $(KDEVOPS_NODES)
 	$(Q)ansible-playbook $(ANSIBLE_VERBOSE) \
 		--limit 'baseline:dev' \
 		$(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \
 		--extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS)
+endif
+
+PHONY += linux-baseline
+ifeq (y,$(CONFIG_KDEVOPS_BASELINE_AND_DEV))
+linux-baseline: $(KDEVOPS_NODES)
+	$(Q)ansible-playbook $(ANSIBLE_VERBOSE) -i \
+		hosts $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \
+		--extra-vars="$(BOOTLINUX_ARGS)" --limit baseline
+else
+linux-baseline:
+	@echo "linux-baseline requires KDEVOPS_BASELINE_AND_DEV=y"
+	@exit 1
+endif
+
+PHONY += linux-dev
+ifeq (y,$(CONFIG_KDEVOPS_BASELINE_AND_DEV))
+linux-dev: $(KDEVOPS_NODES)
+	$(Q)ansible-playbook $(ANSIBLE_VERBOSE) -i \
+		hosts $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \
+		--extra-vars="$(BOOTLINUX_ARGS)" --limit dev
+else
+linux-dev:
+	@echo "linux-dev requires KDEVOPS_BASELINE_AND_DEV=y"
+	@exit 1
+endif
 
 PHONY += linux-mount
 linux-mount:
-- 
2.47.2