[poky][PATCH 1/2] devtool sdk-update: Support Basic Auth

[poky][PATCH 1/2] devtool sdk-update: Support Basic Auth

[Adrian Freihofer]

Support HTTP Basic Auth for plain HTTP downloads as well as for
downloads performed by git. If the server returns a 401 error
and Basic Auth is supported by the server, the user is asked for
credentials.

Signed-off-by: Adrian Freihofer <adrian.freihofer@gmail.com>
---
 scripts/lib/devtool/sdk.py | 105 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 97 insertions(+), 8 deletions(-)

diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index 4616753797..fbf69264e6 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -21,13 +21,93 @@ import logging
 import glob
 import shutil
 import errno
-import sys
 import tempfile
 import re
+import urllib.request
+import base64
+import getpass
+
 from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
+
+class SdkUpdateHelper(object):
+    """Chache the basic auth credentials of an http connection for http file and git downloads.
+    """
+    def __init__(self, user_name=None):
+        self.__user_name = user_name
+        self.__user_password = None
+        self.__ba_credentials = None
+
+    def ask_ba_credentials(self, user_name=None):
+        '''Ask user for Basic Auth Credentials'''
+        if not self.__user_name:
+            self.__user_name = input("User Name: ")
+        self.__user_password = getpass.getpass()
+
+        credentials = ('%s:%s' % (self.__user_name, self.__user_password))
+        b64_credentials = base64.b64encode(credentials.encode('ascii'))
+        self.__ba_credentials = 'Basic %s' % b64_credentials.decode("ascii")
+
+    def download(self, dl_url, out_file_name):
+        """If the HTTP server returns a 401 error code, a second trial with credentials is started.
+        Currently only basic auth is supported.
+        The implementation does not use pythons advanced authentication and password features since
+        the credentials are used by git as well.
+        git_ functions use credentials if the credentials are prepared by this http download function.
+
+        return 0 = success, 1 = failed
+        """
+        logger.debug("Downloading: %s to %s" % (dl_url, out_file_name))
+        http_con_trials = 2
+        while http_con_trials > 0:
+            req = urllib.request.Request(dl_url)
+            if self.__ba_credentials:
+                req.add_header('Authorization', self.__ba_credentials)
+            try:
+                with urllib.request.urlopen(req) as response, open(out_file_name, 'wb') as out_file:
+                    data = response.read()
+                    out_file.write(data)
+                    return 0  # success
+            except urllib.error.HTTPError as hexp:
+                if hexp.code == 401:
+                    hdrs_lower = {k.lower():v for k,v in hexp.hdrs.items()}
+                    try:
+                        auth_type = hdrs_lower['www-authenticate']
+                        logger.debug("auth_type: %s", auth_type)
+                        if auth_type.lower().startswith('basic'):
+                            self.ask_ba_credentials()  # re-try with credentials
+                        else:
+                            logger.error("HTTP download permission denied, no supported authentication type (%s)" % auth_type)
+                            return 1
+                    except KeyError:
+                        logger.error("HTTP download permission denied, authentication is not supported by the server.")
+                        return 1
+                else:
+                    print(str(hexp.code))
+                    print(str(hexp.msg))
+                    print(str(hexp.hdrs))
+                    return 1
+            http_con_trials -= 1
+        return 1
+
+    def _git_http_command(self, command, updateserver="", cwd=None):
+        git_ba_header = ""
+        if self.__ba_credentials:
+            git_ba_header = "-c http.extraheader=\"Authorization: %s\" " % self.__ba_credentials
+
+        git_cmd = "git %s %s %s" % (git_ba_header, command, updateserver)
+        logger.debug("Running: %s", git_cmd)
+        return subprocess.call(git_cmd, shell=True, cwd=cwd)
+
+    def git_clone(self, updateserver="", cwd=None):
+        return self._git_http_command("clone", updateserver, cwd)
+
+    def git_fetch_all(self, cwd=None):
+        return self._git_http_command("fetch --all", cwd=cwd)
+
+
 def parse_locked_sigs(sigfile_path):
     """Return <pn:task>:<hash> dictionary"""
     sig_dict = {}
@@ -138,13 +218,16 @@ def sdk_update(args, config, basepath, workspace):
     finally:
         tinfoil.shutdown()
 
+    sdk_update_helper = SdkUpdateHelper()
     tmpsdk_dir = tempfile.mkdtemp()
     try:
         os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
         new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
         # Fetch manifest from server
-        tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
-        ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
+        sdk_conf_mf = 'sdk-conf-manifest'
+        tmpmanifest = os.path.join(tmpsdk_dir, 'conf', sdk_conf_mf)
+        tmpmanifest_url = "%s/conf/sdk-conf-manifest" % updateserver
+        ret = sdk_update_helper.download(tmpmanifest_url, tmpmanifest)
         if ret != 0:
             logger.error("Cannot dowload files from %s" % updateserver)
             return ret
@@ -155,10 +238,14 @@ def sdk_update(args, config, basepath, workspace):
         # Update metadata
         logger.debug("Updating metadata via git ...")
         #Check for the status before doing a fetch and reset
-        if os.path.exists(os.path.join(basepath, 'layers/.git')):
+        if os.path.exists(os.path.join(layers_dir, '.git')):
             out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir)
             if not out:
-                ret = subprocess.call("git fetch --all; git reset --hard @{u}", shell=True, cwd=layers_dir)
+                ret = sdk_update_helper.git_fetch_all(layers_dir)
+                if ret == 0:
+                    git_cmd = "git reset --hard @{u}"
+                    logger.debug("Running: %s", git_cmd)
+                    ret = subprocess.call(git_cmd, shell=True, cwd=layers_dir)
             else:
                 logger.error("Failed to update metadata as there have been changes made to it. Aborting.");
                 logger.error("Changed files:\n%s" % out);
@@ -166,13 +253,15 @@ def sdk_update(args, config, basepath, workspace):
         else:
             ret = -1
         if ret != 0:
-            ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
+            ret = sdk_update_helper.git_clone("%s/layers/.git" % updateserver, tmpsdk_dir)
             if ret != 0:
                 logger.error("Updating metadata via git failed")
                 return ret
         logger.debug("Updating conf files ...")
         for changedfile in changedfiles:
-            ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
+            changedfile_url = "%s/%s" % (updateserver, changedfile)
+            logger.debug("Downloading %s from %s", changedfile, changedfile_url)
+            ret = sdk_update_helper.download(changedfile_url, os.path.join(tmpsdk_dir, changedfile))
             if ret != 0:
                 logger.error("Updating %s failed" % changedfile)
                 return ret
@@ -197,7 +286,7 @@ def sdk_update(args, config, basepath, workspace):
                 for buildarch, chksum in newsums:
                     uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch)
                     mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file)))
-                    ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir)
+                    ret = sdk_update_helper.download("%s/%s" % (updateserver, uninative_file), os.path.join(tmpsdk_dir, uninative_file))
 
         # Ok, all is well at this point - move everything over
         tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
-- 
2.14.1

[poky][PATCH 2/2] devtool sdk-update: --updateserver-insecure-tls

[Adrian Freihofer]

Support insecure HTTPS connections for devtool sdk-update. This
adds a boolean --updateserver-insecure-tls option the the command
line parameters and to the devtool.conf file's SDK section.

This addresses a common szenario:
* Authorization (HTTPS + user credentials) is required by security
  policy.
* Linux development workstations are not domain members, so they
  don't get the company's ca.cert enrolled.
* The build server has a self signed certificate

Signed-off-by: Adrian Freihofer <adrian.freihofer@gmail.com>
---
 scripts/lib/devtool/sdk.py | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index fbf69264e6..1b703dd21f 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -23,6 +23,7 @@ import shutil
 import errno
 import tempfile
 import re
+import ssl
 import urllib.request
 import base64
 import getpass
@@ -39,6 +40,13 @@ class SdkUpdateHelper(object):
         self.__user_name = user_name
         self.__user_password = None
         self.__ba_credentials = None
+        self.__ssl_context = ssl.create_default_context();
+        self.__git_ssl_args = ""
+
+    def enable_insecure_ssl(self):
+        self.__ssl_context.check_hostname=False
+        self.__ssl_context.verify_mode=ssl.CERT_NONE
+        self.__git_ssl_args = "-c http.sslVerify=false"
 
     def ask_ba_credentials(self, user_name=None):
         '''Ask user for Basic Auth Credentials'''
@@ -66,7 +74,7 @@ class SdkUpdateHelper(object):
             if self.__ba_credentials:
                 req.add_header('Authorization', self.__ba_credentials)
             try:
-                with urllib.request.urlopen(req) as response, open(out_file_name, 'wb') as out_file:
+                with urllib.request.urlopen(req, context=self.__ssl_context) as response, open(out_file_name, 'wb') as out_file:
                     data = response.read()
                     out_file.write(data)
                     return 0  # success
@@ -93,11 +101,15 @@ class SdkUpdateHelper(object):
         return 1
 
     def _git_http_command(self, command, updateserver="", cwd=None):
+        git_ssl_args = ""
+        if self.__git_ssl_args:
+            git_ssl_args = "%s " % self.__git_ssl_args
+
         git_ba_header = ""
         if self.__ba_credentials:
             git_ba_header = "-c http.extraheader=\"Authorization: %s\" " % self.__ba_credentials
 
-        git_cmd = "git %s %s %s" % (git_ba_header, command, updateserver)
+        git_cmd = "git %s%s %s %s" % (git_ssl_args, git_ba_header, command, updateserver)
         logger.debug("Running: %s", git_cmd)
         return subprocess.call(git_cmd, shell=True, cwd=cwd)
 
@@ -193,6 +205,12 @@ def sdk_update(args, config, basepath, workspace):
         updateserver = config.get('SDK', 'updateserver', '')
     logger.debug("updateserver: %s" % updateserver)
 
+    updateserver_insecure_tls = args.updateserver_insecure_tls
+    if not updateserver_insecure_tls:
+        updateserver_insecure_tls = config.get('SDK', 'updateserver-insecure-tls', '')
+    if updateserver_insecure_tls:
+        logger.info("Certificate of update server is not validated.")
+
     # Make sure we are using sdk-update from within SDK
     logger.debug("basepath = %s" % basepath)
     old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
@@ -219,6 +237,8 @@ def sdk_update(args, config, basepath, workspace):
         tinfoil.shutdown()
 
     sdk_update_helper = SdkUpdateHelper()
+    if updateserver_insecure_tls:
+        sdk_update_helper.enable_insecure_ssl()
     tmpsdk_dir = tempfile.mkdtemp()
     try:
         os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
@@ -416,6 +436,13 @@ def register_commands(subparsers, context):
             parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?')
         else:
             parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from')
+
+        try:
+            updateserver_insecure_tls = context.config.get('SDK', 'updateserver-insecure-tls', '')
+        except AttributeError:
+            updateserver_insecure_tls = False
+        parser_sdk.add_argument('--updateserver-insecure-tls', action="store_true", help='Do not validate the SSL Certificate of the update server (default %s)' % updateserver_insecure_tls)
+
         parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
         parser_sdk.set_defaults(func=sdk_update)
 
-- 
2.14.1