[PATCH 1/3] devtool: upgrade: extract changelog between versions

[<daniel.turull@ericsson.com>]

From: Daniel Turull <daniel.turull@ericsson.com>

Automatically extract changelog information when upgrading a recipe.
Uses the devtool-base tags created during upgrade to diff known
changelog files (NEWS, ChangeLog, CHANGES, etc.) between the old and
new versions. For git-based sources, falls back to git log --oneline
if no changelog file changed.

Output is written to workspace/changelogs/<pn>.txt and cleaned up on
devtool reset. This allows AUH and other tools to pick up the changelog
without implementing their own extraction logic.

Assisted-by: kiro:claude-opus-4.6
Signed-off-by: Daniel Turull <daniel.turull@ericsson.com>
---
 meta/lib/oeqa/selftest/cases/devtool.py | 24 +++++++
 scripts/lib/devtool/standard.py         |  8 +++
 scripts/lib/devtool/upgrade.py          | 84 +++++++++++++++++++++++++
 3 files changed, 116 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index 5ed69aee1b..ea788021e6 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -2028,6 +2028,30 @@ class DevtoolUpgradeTests(DevtoolBase):
     def test_devtool_upgrade_gitsm(self):
         self._test_devtool_upgrade_git_by_recipe('devtool-upgrade-test5', '0a60d6af95d22b4c50446559cd41942a8acd2d57')
 
+    def test_devtool_upgrade_changelog(self):
+        # Check preconditions
+        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # dbus-wait has ChangeLog/NEWS files and one commit between these revisions
+        recipe = 'devtool-upgrade-test2'
+        commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        # Perform upgrade
+        runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit))
+        # Check changelog file was created with expected content
+        changelog_file = os.path.join(self.workspacedir, 'changelogs', '%s.txt' % recipe)
+        self.assertExists(changelog_file, 'Changelog file should exist after upgrade')
+        with open(changelog_file, 'r') as f:
+            content = f.read()
+        self.assertIn(recipe, content)
+        # The commit between versions fixes a typo - verify we got real content
+        self.assertIn('typo', content)
+        # Check devtool reset cleans up changelog
+        runCmd('devtool reset %s -n' % recipe)
+        self.assertNotExists(changelog_file, 'Changelog file should be removed after reset')
+
     def test_devtool_upgrade_drop_md5sum(self):
         # Check preconditions
         self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 42fb13872d..2a3a62d081 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -2046,6 +2046,14 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
 
         clean_preferred_provider(pn, config.workspace_path)
 
+        # Clean up changelog if present
+        changelog_file = os.path.join(config.workspace_path, 'changelogs', '%s.txt' % pn)
+        if os.path.exists(changelog_file):
+            os.remove(changelog_file)
+            changelog_dir = os.path.dirname(changelog_file)
+            if not os.listdir(changelog_dir):
+                os.rmdir(changelog_dir)
+
 def reset(args, config, basepath, workspace):
     """Entry point for the devtool 'reset' subcommand"""
 
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 8930fde5d6..6adaf5185e 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -9,6 +9,7 @@
 import os
 import sys
 import re
+import shlex
 import shutil
 import tempfile
 import logging
@@ -26,6 +27,31 @@ from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_r
 
 logger = logging.getLogger('devtool')
 
+# Common changelog filenames found in upstream source trees (matched case-insensitively):
+# changelog - util-linux, coreutils, dbus, acpid, hdparm
+# changelog.md - libslirp, ttyrun, python3-maturin, libjpeg-turbo
+# changelog.rst - python3-pluggy, python3-packaging
+# changes - openssl, python3-babel, icu, tcl
+# changes.md - openssl
+# changes.rst - python3-babel, python3-pathspec
+# changes.txt - python3-lxml, icu
+# news - systemd, glib-2.0, libxml2, dbus
+# news.md - libxml2
+# news.rst - python3-sphinx
+# news.adoc - ccache
+# history.md - python3-requests, python3-hatch-vcs
+# history.rst - python3-idna, python3-docutils
+# releases.md - rust, cargo (includes CVEs)
+# whatsnew.txt - libsdl2
+_CHANGELOG_BASENAMES = {
+    'changelog', 'changelog.md', 'changelog.rst', 'changelog.txt',
+    'changes', 'changes.md', 'changes.rst', 'changes.txt',
+    'news', 'news.md', 'news.rst', 'news.adoc',
+    'history', 'history.md', 'history.rst',
+    'releases.md',
+    'whatsnew.txt',
+}
+
 def _run(cmd, cwd=''):
     logger.debug("Running command %s> %s" % (cwd,cmd))
     return bb.process.run('%s' % cmd, cwd=cwd)
@@ -529,6 +555,52 @@ def _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil):
         if not res:
             raise DevtoolError('Running extra recipe upgrade task %s for %s failed' % (task, pn))
 
+def _extract_changelog(srctree, pn, old_ver, new_ver, old_tag, new_tag, workspace_path, is_git_source):
+    """Extract changelog between old and new version using devtool git tags."""
+    changelog_content = None
+
+    # Try to find a changelog file that changed between versions
+    try:
+        stdout, _ = _run('git diff --name-only %s %s' % (old_tag, new_tag), srctree)
+        for fname in stdout.splitlines():
+            fname = fname.strip()  # strip whitespace/CR from git output
+            if not fname:
+                continue
+            basename = os.path.basename(fname).lower()
+            if basename in _CHANGELOG_BASENAMES:
+                diff_out, _ = _run('git diff %s %s -- %s' % (old_tag, new_tag, shlex.quote(fname)), srctree)
+                if diff_out.strip():
+                    # Extract only the added lines from the diff
+                    lines = [line[1:] for line in diff_out.splitlines()
+                             if line.startswith('+') and not line.startswith('+++')]
+                    if lines:
+                        changelog_content = '\n'.join(lines)
+                        break
+    except bb.process.ExecutionError as e:
+        logger.warning('Changelog file extraction failed: %s' % str(e))
+
+    # For git sources, fall back to git log if no changelog file was found
+    if not changelog_content and is_git_source:
+        try:
+            stdout, _ = _run('git log --oneline %s..%s' % (old_tag, new_tag), srctree)
+            if stdout.strip():
+                changelog_content = stdout.strip()
+        except bb.process.ExecutionError as e:
+            logger.warning('Changelog git log extraction failed: %s' % str(e))
+
+    if not changelog_content:
+        return None
+
+    changelog_dir = os.path.join(workspace_path, 'changelogs')
+    bb.utils.mkdirhier(changelog_dir)
+    changelog_path = os.path.join(changelog_dir, '%s.txt' % pn)
+    with open(changelog_path, 'w') as f:
+        f.write('Changelog for %s: %s -> %s\n\n' % (pn, old_ver, new_ver))
+        f.write(changelog_content)
+        f.write('\n')
+
+    return changelog_path
+
 def upgrade(args, config, basepath, workspace):
     """Entry point for the devtool 'upgrade' subcommand"""
 
@@ -610,6 +682,18 @@ def upgrade(args, config, basepath, workspace):
 
         logger.info('Upgraded source extracted to %s' % srctree)
         logger.info('New recipe is %s' % rf)
+
+        # Extract changelog between versions using the tags created by
+        # _extract_new_source(): devtool-base-new for git, devtool-base-<pv> for tarballs
+        is_git = old_srcrev is not None
+        newpv = args.version or rd.getVar('PV')
+        new_tag = 'devtool-base-new' if is_git else 'devtool-base-%s' % newpv
+        changelog_file = _extract_changelog(srctree, pn, old_ver, newpv,
+                                            'devtool-base', new_tag,
+                                            config.workspace_path, is_git)
+        if changelog_file:
+            logger.info('Changelog extracted to %s' % changelog_file)
+
         if license_diff:
             logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
         preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN'))
-- 
2.34.1