[poky][PATCH 1/2] devtool sdk-update: Support Basic Auth

[Adrian Freihofer <adrian.freihofer@gmail.com>]

Support HTTP Basic Auth for plain HTTP downloads as well as for
downloads performed by git. If the server returns a 401 error
and Basic Auth is supported by the server, the user is asked for
credentials.

Signed-off-by: Adrian Freihofer <adrian.freihofer@gmail.com>
---
 scripts/lib/devtool/sdk.py | 105 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 97 insertions(+), 8 deletions(-)

diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index 4616753797..fbf69264e6 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -21,13 +21,93 @@ import logging
 import glob
 import shutil
 import errno
-import sys
 import tempfile
 import re
+import urllib.request
+import base64
+import getpass
+
 from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
 
+
+class SdkUpdateHelper(object):
+    """Chache the basic auth credentials of an http connection for http file and git downloads.
+    """
+    def __init__(self, user_name=None):
+        self.__user_name = user_name
+        self.__user_password = None
+        self.__ba_credentials = None
+
+    def ask_ba_credentials(self, user_name=None):
+        '''Ask user for Basic Auth Credentials'''
+        if not self.__user_name:
+            self.__user_name = input("User Name: ")
+        self.__user_password = getpass.getpass()
+
+        credentials = ('%s:%s' % (self.__user_name, self.__user_password))
+        b64_credentials = base64.b64encode(credentials.encode('ascii'))
+        self.__ba_credentials = 'Basic %s' % b64_credentials.decode("ascii")
+
+    def download(self, dl_url, out_file_name):
+        """If the HTTP server returns a 401 error code, a second trial with credentials is started.
+        Currently only basic auth is supported.
+        The implementation does not use pythons advanced authentication and password features since
+        the credentials are used by git as well.
+        git_ functions use credentials if the credentials are prepared by this http download function.
+
+        return 0 = success, 1 = failed
+        """
+        logger.debug("Downloading: %s to %s" % (dl_url, out_file_name))
+        http_con_trials = 2
+        while http_con_trials > 0:
+            req = urllib.request.Request(dl_url)
+            if self.__ba_credentials:
+                req.add_header('Authorization', self.__ba_credentials)
+            try:
+                with urllib.request.urlopen(req) as response, open(out_file_name, 'wb') as out_file:
+                    data = response.read()
+                    out_file.write(data)
+                    return 0  # success
+            except urllib.error.HTTPError as hexp:
+                if hexp.code == 401:
+                    hdrs_lower = {k.lower():v for k,v in hexp.hdrs.items()}
+                    try:
+                        auth_type = hdrs_lower['www-authenticate']
+                        logger.debug("auth_type: %s", auth_type)
+                        if auth_type.lower().startswith('basic'):
+                            self.ask_ba_credentials()  # re-try with credentials
+                        else:
+                            logger.error("HTTP download permission denied, no supported authentication type (%s)" % auth_type)
+                            return 1
+                    except KeyError:
+                        logger.error("HTTP download permission denied, authentication is not supported by the server.")
+                        return 1
+                else:
+                    print(str(hexp.code))
+                    print(str(hexp.msg))
+                    print(str(hexp.hdrs))
+                    return 1
+            http_con_trials -= 1
+        return 1
+
+    def _git_http_command(self, command, updateserver="", cwd=None):
+        git_ba_header = ""
+        if self.__ba_credentials:
+            git_ba_header = "-c http.extraheader=\"Authorization: %s\" " % self.__ba_credentials
+
+        git_cmd = "git %s %s %s" % (git_ba_header, command, updateserver)
+        logger.debug("Running: %s", git_cmd)
+        return subprocess.call(git_cmd, shell=True, cwd=cwd)
+
+    def git_clone(self, updateserver="", cwd=None):
+        return self._git_http_command("clone", updateserver, cwd)
+
+    def git_fetch_all(self, cwd=None):
+        return self._git_http_command("fetch --all", cwd=cwd)
+
+
 def parse_locked_sigs(sigfile_path):
     """Return <pn:task>:<hash> dictionary"""
     sig_dict = {}
@@ -138,13 +218,16 @@ def sdk_update(args, config, basepath, workspace):
     finally:
         tinfoil.shutdown()
 
+    sdk_update_helper = SdkUpdateHelper()
     tmpsdk_dir = tempfile.mkdtemp()
     try:
         os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
         new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
         # Fetch manifest from server
-        tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
-        ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
+        sdk_conf_mf = 'sdk-conf-manifest'
+        tmpmanifest = os.path.join(tmpsdk_dir, 'conf', sdk_conf_mf)
+        tmpmanifest_url = "%s/conf/sdk-conf-manifest" % updateserver
+        ret = sdk_update_helper.download(tmpmanifest_url, tmpmanifest)
         if ret != 0:
             logger.error("Cannot dowload files from %s" % updateserver)
             return ret
@@ -155,10 +238,14 @@ def sdk_update(args, config, basepath, workspace):
         # Update metadata
         logger.debug("Updating metadata via git ...")
         #Check for the status before doing a fetch and reset
-        if os.path.exists(os.path.join(basepath, 'layers/.git')):
+        if os.path.exists(os.path.join(layers_dir, '.git')):
             out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir)
             if not out:
-                ret = subprocess.call("git fetch --all; git reset --hard @{u}", shell=True, cwd=layers_dir)
+                ret = sdk_update_helper.git_fetch_all(layers_dir)
+                if ret == 0:
+                    git_cmd = "git reset --hard @{u}"
+                    logger.debug("Running: %s", git_cmd)
+                    ret = subprocess.call(git_cmd, shell=True, cwd=layers_dir)
             else:
                 logger.error("Failed to update metadata as there have been changes made to it. Aborting.");
                 logger.error("Changed files:\n%s" % out);
@@ -166,13 +253,15 @@ def sdk_update(args, config, basepath, workspace):
         else:
             ret = -1
         if ret != 0:
-            ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
+            ret = sdk_update_helper.git_clone("%s/layers/.git" % updateserver, tmpsdk_dir)
             if ret != 0:
                 logger.error("Updating metadata via git failed")
                 return ret
         logger.debug("Updating conf files ...")
         for changedfile in changedfiles:
-            ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
+            changedfile_url = "%s/%s" % (updateserver, changedfile)
+            logger.debug("Downloading %s from %s", changedfile, changedfile_url)
+            ret = sdk_update_helper.download(changedfile_url, os.path.join(tmpsdk_dir, changedfile))
             if ret != 0:
                 logger.error("Updating %s failed" % changedfile)
                 return ret
@@ -197,7 +286,7 @@ def sdk_update(args, config, basepath, workspace):
                 for buildarch, chksum in newsums:
                     uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch)
                     mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file)))
-                    ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir)
+                    ret = sdk_update_helper.download("%s/%s" % (updateserver, uninative_file), os.path.join(tmpsdk_dir, uninative_file))
 
         # Ok, all is well at this point - move everything over
         tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
-- 
2.14.1