[yocto-autobuilder-helper][dunfell 0/2] Patch review

[yocto-autobuilder-helper][dunfell 0/2] Patch review

[Steve Sakoman]

This series updates to a more recent version of buildtools

Tested over the past serveral weeks on autobuilder, most recently:

https://autobuilder.yoctoproject.org/typhoon/#/builders/83/builds/3445

The following changes since commit bdbc5f2e4eb45e29c8a957963c8431cdc86c12ce:

  shared-repos: Use tar instead of rsync for speed (2022-02-22 15:26:49 +0000)

are available in the Git repository at:

  git://git.yoctoproject.org/yocto-autobuilder-helper contrib/sakoman
  http://git.yoctoproject.org/cgit.cgi/yocto-autobuilder-helper/log/?h=contrib/sakoman

Richard Purdie (2):
  config.json: Switch to 3.4 M2 buildtools
  config.json: Update to 3.4 buildtools tarball

 config.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

-- 
2.25.1

[yocto-autobuilder-helper][dunfell 0/2] Patch review

[Steve Sakoman]

These two patches are required to fix performance builds on the autobuilder.

There will also be a couple of related patches in an upcoming dunfell
oe-core patchset.

The following changes since commit 76cb7aefd0724d59b6c1bbd9fc483e766624cee2:

  config.json: Update perf worker target names (2022-04-25 21:53:48 +0100)

are available in the Git repository at:

  git://git.yoctoproject.org/yocto-autobuilder-helper contrib/sakoman
  http://git.yoctoproject.org/cgit.cgi/yocto-autobuilder-helper/log/?h=contrib/sakoman

Richard Purdie (2):
  scripts/build-perf-test-wrapper: Update to only send text based emails
  oe-build-perf-report-email: Drop phantomjs email generation support

 scripts/build-perf-test-wrapper       |  14 +--
 scripts/oe-build-perf-report-email.py | 158 +-------------------------
 2 files changed, 7 insertions(+), 165 deletions(-)

-- 
2.25.1

[yocto-autobuilder-helper][dunfell 1/2] scripts/build-perf-test-wrapper: Update to only send text based emails

[Steve Sakoman]

From: Richard Purdie <richard.purdie@linuxfoundation.org>

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit c77f588773924fd724b3c42daace24d6cce50d52)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
 scripts/build-perf-test-wrapper | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/scripts/build-perf-test-wrapper b/scripts/build-perf-test-wrapper
index 660ef82..53a47ea 100755
--- a/scripts/build-perf-test-wrapper
+++ b/scripts/build-perf-test-wrapper
@@ -68,18 +68,6 @@ if args.results_dir:
     git_repo = args.results_dir + "/archive-repo"
     global_results = args.results_dir
 
-if args.email_addr:
-    try:
-        subprocess.check_output(["which", "phantomjs"])
-    except subprocess.CalledProcessError:
-        print("Please install phantomjs to email reports")
-        sys.exit(1)
-    try:
-        subprocess.check_output(["which", "optipng"])
-    except subprocess.CalledProcessError:
-        print("Please install optipng to email reports")
-        sys.exit(1)
-
 op = fcntl.LOCK_EX
 try:
     lf = open("/tmp/oe-build-perf-test-wrapper.lock", 'a+')
@@ -218,7 +206,7 @@ if git_repo:
         os_name = subprocess.check_output(". /etc/os-release; eval echo '$'PRETTY_NAME", shell=True).decode("utf-8").strip()
         cmd = scriptsdir + "/oe-build-perf-report-email.py --to '" + args.email_addr + \
               "' --subject 'Build Perf Test Report for " + os_name + "' --text " + \
-              report_txt + " --html " + report_html
+              report_txt
         try:
             subprocess.check_call(cmd, shell=True)
         except subprocess.CalledProcessError:
-- 
2.25.1

[yocto-autobuilder-helper][dunfell 2/2] oe-build-perf-report-email: Drop phantomjs email generation support

[Steve Sakoman]

From: Richard Purdie <richard.purdie@linuxfoundation.org>

This approach didn't work reliably so switch to text based emails
pointing at the website version with graphs and data.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit c0a7cac39acba54e106fa74143c5de7c5eeefab8)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
 scripts/oe-build-perf-report-email.py | 158 +-------------------------
 1 file changed, 6 insertions(+), 152 deletions(-)

diff --git a/scripts/oe-build-perf-report-email.py b/scripts/oe-build-perf-report-email.py
index f4476f5..6d16892 100755
--- a/scripts/oe-build-perf-report-email.py
+++ b/scripts/oe-build-perf-report-email.py
@@ -25,8 +25,6 @@ import socket
 import subprocess
 import sys
 import tempfile
-from email.mime.image import MIMEImage
-from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
 
@@ -42,22 +40,6 @@ if not os.path.isfile(SCRAPE_JS):
     sys.exit(1)
 
 
-class ReportError(Exception):
-    """Local errors"""
-    pass
-
-
-def check_utils():
-    """Check that all needed utils are installed in the system"""
-    missing = []
-    for cmd in ('phantomjs', 'optipng'):
-        if not shutil.which(cmd):
-            missing.append(cmd)
-    if missing:
-        log.error("The following tools are missing: %s", ' '.join(missing))
-        sys.exit(1)
-
-
 def parse_args(argv):
     """Parse command line arguments"""
     description = """Email build perf test report"""
@@ -82,137 +64,20 @@ def parse_args(argv):
                              "the email parts")
     parser.add_argument('--text',
                         help="Plain text message")
-    parser.add_argument('--html',
-                        help="HTML peport generated by oe-build-perf-report")
     parser.add_argument('--phantomjs-args', action='append',
                         help="Extra command line arguments passed to PhantomJS")
 
     args = parser.parse_args(argv)
 
-    if not args.html and not args.text:
-        parser.error("Please specify --html and/or --text")
+    if not args.text:
+        parser.error("Please specify --text")
 
     return args
 
 
-def decode_png(infile, outfile):
-    """Parse/decode/optimize png data from a html element"""
-    with open(infile) as f:
-        raw_data = f.read()
-
-    # Grab raw base64 data
-    b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1)
-    b64_data = re.sub('">.+$', '', b64_data, 1)
-
-    # Replace file with proper decoded png
-    with open(outfile, 'wb') as f:
-        f.write(base64.b64decode(b64_data))
-
-    subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT)
-
-
-def mangle_html_report(infile, outfile, pngs):
-    """Mangle html file into a email compatible format"""
-    paste = True
-    png_dir = os.path.dirname(outfile)
-    with open(infile) as f_in:
-        with open(outfile, 'w') as f_out:
-            for line in f_in.readlines():
-                stripped = line.strip()
-                # Strip out scripts
-                if stripped == '<!--START-OF-SCRIPTS-->':
-                    paste = False
-                elif stripped == '<!--END-OF-SCRIPTS-->':
-                    paste = True
-                elif paste:
-                    if re.match('^.+href="data:image/png;base64', stripped):
-                        # Strip out encoded pngs (as they're huge in size)
-                        continue
-                    elif 'www.gstatic.com' in stripped:
-                        # HACK: drop references to external static pages
-                        continue
-
-                    # Replace charts with <img> elements
-                    match = re.match('<div id="(?P<id>\w+)"', stripped)
-                    if match and match.group('id') in pngs:
-                        f_out.write('<img src="cid:{}"\n'.format(match.group('id')))
-                    else:
-                        f_out.write(line)
-
-
-def scrape_html_report(report, outdir, phantomjs_extra_args=None):
-    """Scrape html report into a format sendable by email"""
-    tmpdir = tempfile.mkdtemp(dir='.')
-    log.debug("Using tmpdir %s for phantomjs output", tmpdir)
-
-    if not os.path.isdir(outdir):
-        os.mkdir(outdir)
-    if os.path.splitext(report)[1] not in ('.html', '.htm'):
-        raise ReportError("Invalid file extension for report, needs to be "
-                          "'.html' or '.htm'")
-
-    try:
-        log.info("Scraping HTML report with PhangomJS")
-        extra_args = phantomjs_extra_args if phantomjs_extra_args else []
-        subprocess.check_output(['phantomjs', '--debug=true'] + extra_args +
-                                [SCRAPE_JS, report, tmpdir],
-                                stderr=subprocess.STDOUT)
-
-        pngs = []
-        images = []
-        for fname in os.listdir(tmpdir):
-            base, ext = os.path.splitext(fname)
-            if ext == '.png':
-                log.debug("Decoding %s", fname)
-                decode_png(os.path.join(tmpdir, fname),
-                           os.path.join(outdir, fname))
-                pngs.append(base)
-                images.append(fname)
-            elif ext in ('.html', '.htm'):
-                report_file = fname
-            else:
-                log.warning("Unknown file extension: '%s'", ext)
-                #shutil.move(os.path.join(tmpdir, fname), outdir)
-
-        log.debug("Mangling html report file %s", report_file)
-        mangle_html_report(os.path.join(tmpdir, report_file),
-                           os.path.join(outdir, report_file), pngs)
-        return (os.path.join(outdir, report_file),
-                [os.path.join(outdir, i) for i in images])
-    finally:
-        shutil.rmtree(tmpdir)
-
-def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[],
-               blind_copy=[]):
-    """Send email"""
-    # Generate email message
-    text_msg = html_msg = None
-    if text_fn:
-        with open(text_fn) as f:
-            text_msg = MIMEText("Yocto build performance test report.\n" +
-                                f.read(), 'plain')
-    if html_fn:
-        html_msg = msg = MIMEMultipart('related')
-        with open(html_fn) as f:
-            html_msg.attach(MIMEText(f.read(), 'html'))
-        for img_fn in image_fns:
-            # Expect that content id is same as the filename
-            cid = os.path.splitext(os.path.basename(img_fn))[0]
-            with open(img_fn, 'rb') as f:
-                image_msg = MIMEImage(f.read())
-            image_msg['Content-ID'] = '<{}>'.format(cid)
-            html_msg.attach(image_msg)
-
-    if text_msg and html_msg:
-        msg = MIMEMultipart('alternative')
-        msg.attach(text_msg)
-        msg.attach(html_msg)
-    elif text_msg:
-        msg = text_msg
-    elif html_msg:
-        msg = html_msg
-    else:
-        raise ReportError("Neither plain text nor html body specified")
+def send_email(text_fn, subject, recipients, copy=[], blind_copy=[]):
+    with open(text_fn) as f:
+        msg = MIMEText("Yocto build performance test report.\n" + f.read(), 'plain')
 
     pw_data = pwd.getpwuid(os.getuid())
     full_name = pw_data.pw_gecos.split(',')[0]
@@ -239,8 +104,6 @@ def main(argv=None):
     if args.debug:
         log.setLevel(logging.DEBUG)
 
-    check_utils()
-
     if args.outdir:
         outdir = args.outdir
         if not os.path.exists(outdir):
@@ -250,25 +113,16 @@ def main(argv=None):
 
     try:
         log.debug("Storing email parts in %s", outdir)
-        html_report = images = None
-        if args.html:
-            html_report, images = scrape_html_report(args.html, outdir,
-                                                     args.phantomjs_args)
-
         if args.to:
             log.info("Sending email to %s", ', '.join(args.to))
             if args.cc:
                 log.info("Copying to %s", ', '.join(args.cc))
             if args.bcc:
                 log.info("Blind copying to %s", ', '.join(args.bcc))
-            send_email(args.text, html_report, images, args.subject,
-                       args.to, args.cc, args.bcc)
+            send_email(args.text, args.subject, args.to, args.cc, args.bcc)
     except subprocess.CalledProcessError as err:
         log.error("%s, with output:\n%s", str(err), err.output.decode())
         return 1
-    except ReportError as err:
-        log.error(err)
-        return 1
     finally:
         if not args.outdir:
             log.debug("Wiping %s", outdir)
-- 
2.25.1