[PATCH 0/3] devtool: add changelog extraction

[PATCH 0/3] devtool: add changelog extraction

[daniel.turull]

From: Daniel Turull <daniel.turull@ericsson.com>

This series aims to extract the changelog between 2 versions when
devtool is upgrade. The aim is that later to backport the feature into
LTS branches, so we can have more detailed descriptions with AUH.

For testing I reverted the last 200 updates of recipes and master and
try to do devtool upgrade <recipe> --no-patch, since I'm only interested
in the changelog output.

From the last run: 83 with changelog, 21 without, 41 failed.

Excluding failures: 83 out of 104 = 80% of changelog were generated.

Daniel Turull (3):
  devtool: upgrade: extract changelog between versions
  devtool: upgrade: detect per-version release notes files
  devtool: upgrade: clean up extracted changelog content

 meta/lib/oeqa/selftest/cases/devtool.py | 24 +++++++
 scripts/lib/devtool/standard.py         |  8 +++
 scripts/lib/devtool/upgrade.py          | 95 +++++++++++++++++++++++++
 3 files changed, 127 insertions(+)

-- 
2.34.1

[PATCH 1/3] devtool: upgrade: extract changelog between versions

[daniel.turull]

From: Daniel Turull <daniel.turull@ericsson.com>

Automatically extract changelog information when upgrading a recipe.
Uses the devtool-base tags created during upgrade to diff known
changelog files (NEWS, ChangeLog, CHANGES, etc.) between the old and
new versions. For git-based sources, falls back to git log --oneline
if no changelog file changed.

Output is written to workspace/changelogs/<pn>.txt and cleaned up on
devtool reset. This allows AUH and other tools to pick up the changelog
without implementing their own extraction logic.

Assisted-by: kiro:claude-opus-4.6
Signed-off-by: Daniel Turull <daniel.turull@ericsson.com>
---
 meta/lib/oeqa/selftest/cases/devtool.py | 24 +++++++
 scripts/lib/devtool/standard.py         |  8 +++
 scripts/lib/devtool/upgrade.py          | 84 +++++++++++++++++++++++++
 3 files changed, 116 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index 5ed69aee1b..ea788021e6 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -2028,6 +2028,30 @@ class DevtoolUpgradeTests(DevtoolBase):
     def test_devtool_upgrade_gitsm(self):
         self._test_devtool_upgrade_git_by_recipe('devtool-upgrade-test5', '0a60d6af95d22b4c50446559cd41942a8acd2d57')
 
+    def test_devtool_upgrade_changelog(self):
+        # Check preconditions
+        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        # dbus-wait has ChangeLog/NEWS files and one commit between these revisions
+        recipe = 'devtool-upgrade-test2'
+        commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        # Perform upgrade
+        runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit))
+        # Check changelog file was created with expected content
+        changelog_file = os.path.join(self.workspacedir, 'changelogs', '%s.txt' % recipe)
+        self.assertExists(changelog_file, 'Changelog file should exist after upgrade')
+        with open(changelog_file, 'r') as f:
+            content = f.read()
+        self.assertIn(recipe, content)
+        # The commit between versions fixes a typo - verify we got real content
+        self.assertIn('typo', content)
+        # Check devtool reset cleans up changelog
+        runCmd('devtool reset %s -n' % recipe)
+        self.assertNotExists(changelog_file, 'Changelog file should be removed after reset')
+
     def test_devtool_upgrade_drop_md5sum(self):
         # Check preconditions
         self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 42fb13872d..2a3a62d081 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -2046,6 +2046,14 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
 
         clean_preferred_provider(pn, config.workspace_path)
 
+        # Clean up changelog if present
+        changelog_file = os.path.join(config.workspace_path, 'changelogs', '%s.txt' % pn)
+        if os.path.exists(changelog_file):
+            os.remove(changelog_file)
+            changelog_dir = os.path.dirname(changelog_file)
+            if not os.listdir(changelog_dir):
+                os.rmdir(changelog_dir)
+
 def reset(args, config, basepath, workspace):
     """Entry point for the devtool 'reset' subcommand"""
 
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 8930fde5d6..6adaf5185e 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -9,6 +9,7 @@
 import os
 import sys
 import re
+import shlex
 import shutil
 import tempfile
 import logging
@@ -26,6 +27,31 @@ from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_r
 
 logger = logging.getLogger('devtool')
 
+# Common changelog filenames found in upstream source trees (matched case-insensitively):
+# changelog - util-linux, coreutils, dbus, acpid, hdparm
+# changelog.md - libslirp, ttyrun, python3-maturin, libjpeg-turbo
+# changelog.rst - python3-pluggy, python3-packaging
+# changes - openssl, python3-babel, icu, tcl
+# changes.md - openssl
+# changes.rst - python3-babel, python3-pathspec
+# changes.txt - python3-lxml, icu
+# news - systemd, glib-2.0, libxml2, dbus
+# news.md - libxml2
+# news.rst - python3-sphinx
+# news.adoc - ccache
+# history.md - python3-requests, python3-hatch-vcs
+# history.rst - python3-idna, python3-docutils
+# releases.md - rust, cargo (includes CVEs)
+# whatsnew.txt - libsdl2
+_CHANGELOG_BASENAMES = {
+    'changelog', 'changelog.md', 'changelog.rst', 'changelog.txt',
+    'changes', 'changes.md', 'changes.rst', 'changes.txt',
+    'news', 'news.md', 'news.rst', 'news.adoc',
+    'history', 'history.md', 'history.rst',
+    'releases.md',
+    'whatsnew.txt',
+}
+
 def _run(cmd, cwd=''):
     logger.debug("Running command %s> %s" % (cwd,cmd))
     return bb.process.run('%s' % cmd, cwd=cwd)
@@ -529,6 +555,52 @@ def _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil):
         if not res:
             raise DevtoolError('Running extra recipe upgrade task %s for %s failed' % (task, pn))
 
+def _extract_changelog(srctree, pn, old_ver, new_ver, old_tag, new_tag, workspace_path, is_git_source):
+    """Extract changelog between old and new version using devtool git tags."""
+    changelog_content = None
+
+    # Try to find a changelog file that changed between versions
+    try:
+        stdout, _ = _run('git diff --name-only %s %s' % (old_tag, new_tag), srctree)
+        for fname in stdout.splitlines():
+            fname = fname.strip()  # strip whitespace/CR from git output
+            if not fname:
+                continue
+            basename = os.path.basename(fname).lower()
+            if basename in _CHANGELOG_BASENAMES:
+                diff_out, _ = _run('git diff %s %s -- %s' % (old_tag, new_tag, shlex.quote(fname)), srctree)
+                if diff_out.strip():
+                    # Extract only the added lines from the diff
+                    lines = [line[1:] for line in diff_out.splitlines()
+                             if line.startswith('+') and not line.startswith('+++')]
+                    if lines:
+                        changelog_content = '\n'.join(lines)
+                        break
+    except bb.process.ExecutionError as e:
+        logger.warning('Changelog file extraction failed: %s' % str(e))
+
+    # For git sources, fall back to git log if no changelog file was found
+    if not changelog_content and is_git_source:
+        try:
+            stdout, _ = _run('git log --oneline %s..%s' % (old_tag, new_tag), srctree)
+            if stdout.strip():
+                changelog_content = stdout.strip()
+        except bb.process.ExecutionError as e:
+            logger.warning('Changelog git log extraction failed: %s' % str(e))
+
+    if not changelog_content:
+        return None
+
+    changelog_dir = os.path.join(workspace_path, 'changelogs')
+    bb.utils.mkdirhier(changelog_dir)
+    changelog_path = os.path.join(changelog_dir, '%s.txt' % pn)
+    with open(changelog_path, 'w') as f:
+        f.write('Changelog for %s: %s -> %s\n\n' % (pn, old_ver, new_ver))
+        f.write(changelog_content)
+        f.write('\n')
+
+    return changelog_path
+
 def upgrade(args, config, basepath, workspace):
     """Entry point for the devtool 'upgrade' subcommand"""
 
@@ -610,6 +682,18 @@ def upgrade(args, config, basepath, workspace):
 
         logger.info('Upgraded source extracted to %s' % srctree)
         logger.info('New recipe is %s' % rf)
+
+        # Extract changelog between versions using the tags created by
+        # _extract_new_source(): devtool-base-new for git, devtool-base-<pv> for tarballs
+        is_git = old_srcrev is not None
+        newpv = args.version or rd.getVar('PV')
+        new_tag = 'devtool-base-new' if is_git else 'devtool-base-%s' % newpv
+        changelog_file = _extract_changelog(srctree, pn, old_ver, newpv,
+                                            'devtool-base', new_tag,
+                                            config.workspace_path, is_git)
+        if changelog_file:
+            logger.info('Changelog extracted to %s' % changelog_file)
+
         if license_diff:
             logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
         preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN'))
-- 
2.34.1

Re: [OE-core] [PATCH 1/3] devtool: upgrade: extract changelog between versions

[Alexander Kanavin]

On Tue, 12 May 2026 at 09:25, Daniel Turull via lists.openembedded.org
<daniel.turull=ericsson.com@lists.openembedded.org> wrote:
> Automatically extract changelog information when upgrading a recipe.
> Uses the devtool-base tags created during upgrade to diff known
> changelog files (NEWS, ChangeLog, CHANGES, etc.) between the old and
> new versions. For git-based sources, falls back to git log --oneline
> if no changelog file changed.
>
> Output is written to workspace/changelogs/<pn>.txt and cleaned up on
> devtool reset. This allows AUH and other tools to pick up the changelog
> without implementing their own extraction logic.

Thanks, I like this a lot, and I think this should be fairly easy to
get merged. Just a couple notes:

- probably patches 2/3 and 3/3 should all be folded into the main
commit? (but also add those commit messages to the main commit
message, don't discard the additional information)

- please see below this about testing the functionality

> +    def test_devtool_upgrade_changelog(self):
> +        # Check preconditions
> +        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
> +        self.track_for_cleanup(self.workspacedir)
> +        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
> +        # dbus-wait has ChangeLog/NEWS files and one commit between these revisions
> +        recipe = 'devtool-upgrade-test2'
> +        commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
> +        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> +        self.track_for_cleanup(tempdir)
> +        # Perform upgrade
> +        runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit))
> +        # Check changelog file was created with expected content
> +        changelog_file = os.path.join(self.workspacedir, 'changelogs', '%s.txt' % recipe)
> +        self.assertExists(changelog_file, 'Changelog file should exist after upgrade')
> +        with open(changelog_file, 'r') as f:
> +            content = f.read()
> +        self.assertIn(recipe, content)
> +        # The commit between versions fixes a typo - verify we got real content
> +        self.assertIn('typo', content)
> +        # Check devtool reset cleans up changelog
> +        runCmd('devtool reset %s -n' % recipe)
> +        self.assertNotExists(changelog_file, 'Changelog file should be removed after reset')

Rather than write a whole new test (which adds to the overall testing
time, copy-pastes code, and repeats the upgrade procedure already
happening in a different test), I think the changelog checks should be
added to existing tests in DevtoolUpgradeTests class. At least these
tests:

- test_devtool_upgrade
- _test_devtool_upgrade_git_by_recipe

but ideally everything where upgrade operation runs and changelogs are
extracted.

Also, don't look for simple strings as a way to check the changelog
sanity. Instead, write a 'reference changelog' into meta-selftest, and
compare that with the extracted changelog in the workspace, similar to
how upgraded recipes are compared with reference data:

        # Check recipe got changed as expected
        with open(oldrecipefile + '.upgraded', 'r') as f:
            desiredlines = f.readlines()
        with open(newrecipefile, 'r') as f:
            newlines = f.readlines()
        self.assertEqual(desiredlines, newlines)

Such a comparison could probably be in a helper function to avoid
repeating code lines.

It's okay if some of the upgrades are unable to extract anything
useful into the changelog, then the reference data should simply stay
empty. If you can cover all of these, would be awesome:

meta-selftest/recipes-devtools/python/python3-guessing-game-crates.inc.upgraded
meta-selftest/recipes-devtools/python/python3-guessing-game_git.bb.upgraded
meta-selftest/recipes-test/devtool/devtool-upgrade-test1_1.5.3.bb.upgraded
meta-selftest/recipes-test/devtool/devtool-upgrade-test2_git.bb.upgraded
meta-selftest/recipes-test/devtool/devtool-upgrade-test3_1.5.3.bb.upgraded
meta-selftest/recipes-test/devtool/devtool-upgrade-test4_1.5.3.bb.upgraded
meta-selftest/recipes-test/devtool/devtool-upgrade-test5_git.bb.upgraded

Cheers,
Alex

RE: [OE-core] [PATCH 1/3] devtool: upgrade: extract changelog between versions

[Daniel Turull]

> -----Original Message-----
> From: Alexander Kanavin <alex.kanavin@gmail.com>
> Sent: Tuesday, 12 May 2026 18:28
> To: Daniel Turull <daniel.turull@ericsson.com>
> Cc: openembedded-core@lists.openembedded.org
> Subject: Re: [OE-core] [PATCH 1/3] devtool: upgrade: extract changelog between
> versions
> 
> On Tue, 12 May 2026 at 09:25, Daniel Turull via lists.openembedded.org
> <daniel.turull=ericsson.com@lists.openembedded.org> wrote:
> > Automatically extract changelog information when upgrading a recipe.
> > Uses the devtool-base tags created during upgrade to diff known
> > changelog files (NEWS, ChangeLog, CHANGES, etc.) between the old and
> > new versions. For git-based sources, falls back to git log --oneline
> > if no changelog file changed.
> >
> > Output is written to workspace/changelogs/<pn>.txt and cleaned up on
> > devtool reset. This allows AUH and other tools to pick up the
> > changelog without implementing their own extraction logic.
> 
> Thanks, I like this a lot, and I think this should be fairly easy to get merged. Just a
> couple notes:
> 
> - probably patches 2/3 and 3/3 should all be folded into the main commit? (but
> also add those commit messages to the main commit message, don't discard the
> additional information)
> 

Sure, I'll squash them into one.

> - please see below this about testing the functionality
> 
> > +    def test_devtool_upgrade_changelog(self):
> > +        # Check preconditions
> > +        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be
> run with a workspace directory under the build directory')
> > +        self.track_for_cleanup(self.workspacedir)
> > +        self.add_command_to_tearDown('bitbake-layers remove-layer
> */workspace')
> > +        # dbus-wait has ChangeLog/NEWS files and one commit between these
> revisions
> > +        recipe = 'devtool-upgrade-test2'
> > +        commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
> > +        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
> > +        self.track_for_cleanup(tempdir)
> > +        # Perform upgrade
> > +        runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit))
> > +        # Check changelog file was created with expected content
> > +        changelog_file = os.path.join(self.workspacedir, 'changelogs', '%s.txt' %
> recipe)
> > +        self.assertExists(changelog_file, 'Changelog file should exist after
> upgrade')
> > +        with open(changelog_file, 'r') as f:
> > +            content = f.read()
> > +        self.assertIn(recipe, content)
> > +        # The commit between versions fixes a typo - verify we got real content
> > +        self.assertIn('typo', content)
> > +        # Check devtool reset cleans up changelog
> > +        runCmd('devtool reset %s -n' % recipe)
> > +        self.assertNotExists(changelog_file, 'Changelog file should
> > + be removed after reset')
> 
> Rather than write a whole new test (which adds to the overall testing time, copy-
> pastes code, and repeats the upgrade procedure already happening in a different
> test), I think the changelog checks should be added to existing tests in
> DevtoolUpgradeTests class. At least these
> tests:
> 
> - test_devtool_upgrade
> - _test_devtool_upgrade_git_by_recipe
> 
> but ideally everything where upgrade operation runs and changelogs are
> extracted.
> 
> Also, don't look for simple strings as a way to check the changelog sanity. Instead,
> write a 'reference changelog' into meta-selftest, and compare that with the
> extracted changelog in the workspace, similar to how upgraded recipes are
> compared with reference data:
> 
>         # Check recipe got changed as expected
>         with open(oldrecipefile + '.upgraded', 'r') as f:
>             desiredlines = f.readlines()
>         with open(newrecipefile, 'r') as f:
>             newlines = f.readlines()
>         self.assertEqual(desiredlines, newlines)
> 
> Such a comparison could probably be in a helper function to avoid repeating code
> lines.
> 
> It's okay if some of the upgrades are unable to extract anything useful into the
> changelog, then the reference data should simply stay empty. If you can cover all
> of these, would be awesome:
> 
> meta-selftest/recipes-devtools/python/python3-guessing-game-
> crates.inc.upgraded
> meta-selftest/recipes-devtools/python/python3-guessing-game_git.bb.upgraded
> meta-selftest/recipes-test/devtool/devtool-upgrade-test1_1.5.3.bb.upgraded
> meta-selftest/recipes-test/devtool/devtool-upgrade-test2_git.bb.upgraded
> meta-selftest/recipes-test/devtool/devtool-upgrade-test3_1.5.3.bb.upgraded
> meta-selftest/recipes-test/devtool/devtool-upgrade-test4_1.5.3.bb.upgraded
> meta-selftest/recipes-test/devtool/devtool-upgrade-test5_git.bb.upgraded
> 

I'll redo the tests with your suggestions and send a v2

Thanks

Daniel
> Cheers,
> Alex

[PATCH 2/3] devtool: upgrade: detect per-version release notes files

[daniel.turull]

From: Daniel Turull <daniel.turull@ericsson.com>

Extend the changelog file detection loop to also match per-version
release notes files like git's Documentation/RelNotes/2.53.0.adoc
or mesa's docs/relnotes/26.0.3.rst.

These are detected by matching filenames that contain a version
number with a documentation extension (.txt, .md, .rst, .adoc).
Their full content is read via git show since they are new files
rather than modifications to an existing changelog.

Assisted-by: kiro:claude-opus-4.6
Signed-off-by: Daniel Turull <daniel.turull@ericsson.com>
---
 scripts/lib/devtool/upgrade.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 6adaf5185e..f2a2c8a6fb 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -576,6 +576,12 @@ def _extract_changelog(srctree, pn, old_ver, new_ver, old_tag, new_tag, workspac
                     if lines:
                         changelog_content = '\n'.join(lines)
                         break
+            # Per-version release notes (e.g., git RelNotes/2.53.0.adoc, mesa relnotes/26.0.3.rst)
+            elif re.search(r'(\d+[.\-])+\d+\.(txt|md|rst|adoc)$', basename):
+                file_content, _ = _run('git show %s' % shlex.quote('%s:%s' % (new_tag, fname)), srctree)
+                if file_content.strip():
+                    changelog_content = file_content.strip()
+                    break
     except bb.process.ExecutionError as e:
         logger.warning('Changelog file extraction failed: %s' % str(e))
 
-- 
2.34.1

[PATCH 3/3] devtool: upgrade: clean up extracted changelog content

[daniel.turull]

From: Daniel Turull <daniel.turull@ericsson.com>

Collapse runs of multiple blank lines into a single blank line and
strip leading/trailing whitespace from the extracted changelog. This
produces cleaner output that is suitable for inclusion in commit
messages by tools like AUH.

Assisted-by: kiro:claude-opus-4.6
Signed-off-by: Daniel Turull <daniel.turull@ericsson.com>
---
 scripts/lib/devtool/upgrade.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index f2a2c8a6fb..e37ad07ea1 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -597,6 +597,11 @@ def _extract_changelog(srctree, pn, old_ver, new_ver, old_tag, new_tag, workspac
     if not changelog_content:
         return None
 
+    # Clean up content for readability and commit message use
+    changelog_content = re.sub(r'\n{3,}', '\n\n', changelog_content).strip()
+    if not changelog_content:
+        return None
+
     changelog_dir = os.path.join(workspace_path, 'changelogs')
     bb.utils.mkdirhier(changelog_dir)
     changelog_path = os.path.join(changelog_dir, '%s.txt' % pn)
-- 
2.34.1