[meta-swupd] allow username/password encoded in SWUPD_VERSION_URL and SWUPD_CONTENT_URL

[meta-swupd] allow username/password encoded in SWUPD_VERSION_URL and SWUPD_CONTENT_URL

[Ingo Flaschberger]

[-- Attachment #1: Type: text/plain, Size: 183 bytes --]

This patch allows basic authentication of swupd SWUPD_VERSION_URL and 
SWUPD_CONTENT_URL.
swupd-client already support urlencoded username/password, but 
buildlayer does not.



[-- Attachment #2: basic-auth.patch --]
[-- Type: text/plain, Size: 2500 bytes --]

diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index b4c6f49..223fd3c 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -4,6 +4,8 @@ import subprocess
 import shutil
 import urllib.request
 import urllib.error
+import urllib.parse
+import re
 from bb.utils import export_proxies
 from oe.package_manager import RpmPM
 from oe.package_manager import OpkgPM
@@ -164,6 +166,15 @@ def download_manifests(content_url, version, component, to_dir):
     base_versions = set()
     if not os.path.exists(target):
         bb.debug(1, 'Downloading %s -> %s' % (source, target))
+        parsed_source = urllib.parse.urlsplit(source)
+        if( parsed_source.username != None):
+            new_source = ( parsed_source.scheme, re.sub( re.escape( parsed_source.username+':'+parsed_source.password+'@'), '',parsed_source.netloc), parsed_source.path, parsed_source.query, parsed_source.fragment)
+            source = urllib.parse.urlunsplit( new_source)
+            manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+            manager.add_password(None, new_source, parsed_source.username, parsed_source.password)
+            authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+            opener = urllib.request.build_opener(authHandler)
+            urllib.request.install_opener(opener)
         response = urllib.request.urlopen(source)
         archive = response.read()
         bb.utils.mkdirhier(to_dir)
@@ -228,6 +239,15 @@ def download_old_versions(d):
     for format in range(3, current_format + 1):
         try:
             url = '%s/version/format%d/latest' % (content_url, format)
+            parsed_url = urllib.parse.urlsplit(url)
+            if( parsed_url.username != None):
+                new_url = ( parsed_url.scheme, re.sub( re.escape( parsed_url.username+':'+parsed_url.password+'@'), '',parsed_url.netloc), parsed_url.path, parsed_url.query, parsed_url.fragment)
+                url = urllib.parse.urlunsplit( new_url)
+                manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+                manager.add_password(None, new_url, parsed_url.username, parsed_url.password)
+                authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+                opener = urllib.request.build_opener(authHandler)
+                urllib.request.install_opener(opener)
             response = urllib.request.urlopen(url)
             version = int(response.read())
             latest_versions[format] = version

Re: [meta-swupd] allow username/password encoded in SWUPD_VERSION_URL and SWUPD_CONTENT_URL

[Patrick Ohly]

On Thu, 2017-03-23 at 16:04 +0100, Ingo Flaschberger wrote:
> This patch allows basic authentication of swupd SWUPD_VERSION_URL and 
> SWUPD_CONTENT_URL.
> swupd-client already support urlencoded username/password, but 
> buildlayer does not.

Can you resend in "git format-patch" format? Don't forget the
signed-off-by and add short prefix to the summary line, perhaps like
this:
bundles.py: allow username/password encoded into HTTP server URLs

An example how username/password need to be encoded in the URL would
also be useful.

-- 
Best Regards, Patrick Ohly

The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.

Re: [meta-swupd] allow username/password encoded in SWUPD_VERSION_URL and SWUPD_CONTENT_URL

[Ingo Flaschberger]

[-- Attachment #1: Type: text/plain, Size: 28 bytes --]

requested patch attached


[-- Attachment #2: 0001-bundles.py-allow-username-password-encoded-into-HTTP.patch --]
[-- Type: text/plain, Size: 2876 bytes --]

From f2526a7ed47b3f3c8f0cb893eadb5e6981255d4c Mon Sep 17 00:00:00 2001
From: ingo <ingo.flaschberger@gmail.com>
Date: Sat, 25 Mar 2017 21:13:33 +0100
Subject: [PATCH] bundles.py: allow username/password encoded into HTTP server
 URLs example: https://user:password@server/path

---
 lib/swupd/bundles.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index b4c6f49..223fd3c 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -4,6 +4,8 @@ import subprocess
 import shutil
 import urllib.request
 import urllib.error
+import urllib.parse
+import re
 from bb.utils import export_proxies
 from oe.package_manager import RpmPM
 from oe.package_manager import OpkgPM
@@ -164,6 +166,15 @@ def download_manifests(content_url, version, component, to_dir):
     base_versions = set()
     if not os.path.exists(target):
         bb.debug(1, 'Downloading %s -> %s' % (source, target))
+        parsed_source = urllib.parse.urlsplit(source)
+        if( parsed_source.username != None):
+            new_source = ( parsed_source.scheme, re.sub( re.escape( parsed_source.username+':'+parsed_source.password+'@'), '',parsed_source.netloc), parsed_source.path, parsed_source.query, parsed_source.fragment)
+            source = urllib.parse.urlunsplit( new_source)
+            manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+            manager.add_password(None, new_source, parsed_source.username, parsed_source.password)
+            authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+            opener = urllib.request.build_opener(authHandler)
+            urllib.request.install_opener(opener)
         response = urllib.request.urlopen(source)
         archive = response.read()
         bb.utils.mkdirhier(to_dir)
@@ -228,6 +239,15 @@ def download_old_versions(d):
     for format in range(3, current_format + 1):
         try:
             url = '%s/version/format%d/latest' % (content_url, format)
+            parsed_url = urllib.parse.urlsplit(url)
+            if( parsed_url.username != None):
+                new_url = ( parsed_url.scheme, re.sub( re.escape( parsed_url.username+':'+parsed_url.password+'@'), '',parsed_url.netloc), parsed_url.path, parsed_url.query, parsed_url.fragment)
+                url = urllib.parse.urlunsplit( new_url)
+                manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+                manager.add_password(None, new_url, parsed_url.username, parsed_url.password)
+                authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+                opener = urllib.request.build_opener(authHandler)
+                urllib.request.install_opener(opener)
             response = urllib.request.urlopen(url)
             version = int(response.read())
             latest_versions[format] = version
-- 
2.7.4

Re: [meta-swupd] allow username/password encoded in SWUPD_VERSION_URL and SWUPD_CONTENT_URL

[Patrick Ohly]

Hello Ingo!

Sorry for the late reply. There were quite a few things in the patch
that needed further discussion, so I kept postponing dealing with it.
That, and I am not sure due to staffing questions whether I am really
supposed to maintain meta-swupd at the moment :-/

Right now I refrain from applying patches to it until that gets
clarified.

On Sat, 2017-03-25 at 21:14 +0100, Ingo Flaschberger wrote:
> requested patch attached

Instead of attaching patches, please use "git send-email" to send the
patch directly. I needs a proper commit message, too:
https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines#Patch_Headers_and_Commit_Messages

> From f2526a7ed47b3f3c8f0cb893eadb5e6981255d4c Mon Sep 17 00:00:00
> 2001
> From: ingo <ingo.flaschberger@gmail.com>
> Date: Sat, 25 Mar 2017 21:13:33 +0100
> Subject: [PATCH] bundles.py: allow username/password encoded into
> HTTP server
>  URLs example: https://user:password@server/path
> 
> ---
>  lib/swupd/bundles.py | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
> index b4c6f49..223fd3c 100644
> --- a/lib/swupd/bundles.py
> +++ b/lib/swupd/bundles.py
> @@ -4,6 +4,8 @@ import subprocess
>  import shutil
>  import urllib.request
>  import urllib.error
> +import urllib.parse
> +import re
>  from bb.utils import export_proxies
>  from oe.package_manager import RpmPM
>  from oe.package_manager import OpkgPM
> @@ -164,6 +166,15 @@ def download_manifests(content_url, version,
> component, to_dir):
>      base_versions = set()
>      if not os.path.exists(target):
>          bb.debug(1, 'Downloading %s -> %s' % (source, target))
> +        parsed_source = urllib.parse.urlsplit(source)
> +        if( parsed_source.username != None):
> +            new_source = ( parsed_source.scheme, re.sub( re.escape(
> parsed_source.username+':'+parsed_source.password+'@'),
> '',parsed_source.netloc), parsed_source.path, parsed_source.query,
> parsed_source.fragment)

Wouldn't it be simpler to do this?
   parsed_source.netloc = parsed_source.hostname

We want the original URL, just with a simpler netloc part. Mucking
around with a regex to achieve that seems overly complicated when
urllib.parse() has already done the parsing for us.

> +            source = urllib.parse.urlunsplit( new_source)
> +            manager =
> urllib.request.HTTPPasswordMgrWithDefaultRealm()
> +            manager.add_password(None, new_source,
> parsed_source.username, parsed_source.password)
> +            authHandler =
> urllib.request.HTTPBasicAuthHandler(manager)
> +            opener = urllib.request.build_opener(authHandler)
> +            urllib.request.install_opener(opener)

This opener gets installed over and over again, each time some URL
derived from content_url is used. Isn't it enough to check content_url
once and then use a simpler version of it for constructing URLs?

>          response = urllib.request.urlopen(source)
>          archive = response.read()
>          bb.utils.mkdirhier(to_dir)
> @@ -228,6 +239,15 @@ def download_old_versions(d):
>      for format in range(3, current_format + 1):
>          try:
>              url = '%s/version/format%d/latest' % (content_url,
> format)
> +            parsed_url = urllib.parse.urlsplit(url)
> +            if( parsed_url.username != None):
> +                new_url = ( parsed_url.scheme, re.sub( re.escape(
> parsed_url.username+':'+parsed_url.password+'@'),
> '',parsed_url.netloc), parsed_url.path, parsed_url.query,
> parsed_url.fragment)
> +                url = urllib.parse.urlunsplit( new_url)
> +                manager =
> urllib.request.HTTPPasswordMgrWithDefaultRealm()
> +                manager.add_password(None, new_url,
> parsed_url.username, parsed_url.password)
> +                authHandler =
> urllib.request.HTTPBasicAuthHandler(manager)
> +                opener = urllib.request.build_opener(authHandler)
> +                urllib.request.install_opener(opener)

Cut-and-paste... this should be in a helper function.

I tried to come up with a cleaner patch that implements the same
behavior. But I don't have a way to test it. Can you perhaps try out
the patch that I will post as a followup?

Note that it applies cleanly only on top of
https://github.com/pohly/meta-swupd/tree/master

You can also check out the patch from
https://github.com/pohly/meta-swupd/tree/basic_auth

-- 
Best Regards, Patrick Ohly

The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.

[meta-swupd][PATCH 1/1] bundles.py: allow username/password encoded in URLs

[Patrick Ohly]

Downloading content and version information via HTTP may need a
username/password for basic authentication. To support this,
SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the
form http(s)://<user>:<password>@<host>/.

Original patch from: Ingo Flaschberger <ingo.flaschberger@gmail.com>

Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
 lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index e1eec5a..48c7455 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -5,6 +5,7 @@ import subprocess
 import shutil
 import urllib.request
 import urllib.error
+import urllib.parse
 from bb.utils import export_proxies
 from oe.package_manager import RpmPM
 from oe.package_manager import OpkgPM
@@ -153,6 +154,27 @@ def copy_bundle_contents(d):
     for bndl in bundles:
         stage_empty_bundle(d, bndl)
 
+def handle_plain_auth(url):
+    """
+    Check for special urls with username/password (as in http://user:password@host/),
+    extract those and install an auth handler which will provide them
+    to the HTTP server when needed. Returns the URL that is to be instead of the original one.
+    """
+    parsed_url = urllib.parse.urlsplit(url)
+    if parsed_url.username != None:
+        # Use the netloc with just the hostname, without username/password.
+        parsed_url.netloc = parsed_url.hostname
+        # The username/password are installed permanently in the urllib.request module
+        # for future use with all URLs beneath url.
+        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+        manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
+        authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+        opener = urllib.request.build_opener(authHandler)
+        urllib.request.install_opener(opener)
+        return urllib.parse.urlunsplit(new_source)
+    else:
+        return url
+
 def download_manifests(content_url, version, component, to_dir):
     """
     Download one manifest file and recursively all manifests referenced by it.
@@ -204,8 +226,8 @@ def download_old_versions(d):
     a normal build and thus is not on the critical path.
     """
 
-    content_url = d.getVar('SWUPD_CONTENT_BUILD_URL', True)
-    version_url = d.getVar('SWUPD_VERSION_BUILD_URL', True)
+    content_url = handle_plain_auth(d.getVar('SWUPD_CONTENT_BUILD_URL', True))
+    version_url = handle_plain_auth(d.getVar('SWUPD_VERSION_BUILD_URL', True))
     current_format = int(d.getVar('SWUPD_FORMAT', True))
     deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True)
     www_dir = os.path.join(deploy_dir, 'www')
-- 
2.11.0

Re: [meta-swupd][PATCH 1/1] bundles.py: allow username/password encoded in URLs

[Ingo Flaschberger]

Dear Patrick,

this doesn't work:
      0162:    """
      0163:    parsed_url = urllib.parse.urlsplit(url)
      0164:    if parsed_url.username != None:
      0165:        # Use the netloc with just the hostname, without 
username/password.
  *** 0166:        parsed_url.netloc = parsed_url.hostname
      0167:        # The username/password are installed permanently in 
the urllib.request module
      0168:        # for future use with all URLs beneath url.
      0169:        manager = 
urllib.request.HTTPPasswordMgrWithDefaultRealm()
      0170:        manager.add_password(None, parsed_url, 
parsed_url.username, parsed_url.password)
Exception: AttributeError: can't set attribute


Using hostname as netloc will also remove an additional portnumber - 
could this be a problem?

Bye,
     Ingo


Am 20.12.2017 um 16:50 schrieb Patrick Ohly:
> Downloading content and version information via HTTP may need a
> username/password for basic authentication. To support this,
> SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the
> form http(s)://<user>:<password>@<host>/.
>
> Original patch from: Ingo Flaschberger <ingo.flaschberger@gmail.com>
>
> Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
> ---
>   lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--
>   1 file changed, 24 insertions(+), 2 deletions(-)
>
> diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
> index e1eec5a..48c7455 100644
> --- a/lib/swupd/bundles.py
> +++ b/lib/swupd/bundles.py
> @@ -5,6 +5,7 @@ import subprocess
>   import shutil
>   import urllib.request
>   import urllib.error
> +import urllib.parse
>   from bb.utils import export_proxies
>   from oe.package_manager import RpmPM
>   from oe.package_manager import OpkgPM
> @@ -153,6 +154,27 @@ def copy_bundle_contents(d):
>       for bndl in bundles:
>           stage_empty_bundle(d, bndl)
>   
> +def handle_plain_auth(url):
> +    """
> +    Check for special urls with username/password (as in http://user:password@host/),
> +    extract those and install an auth handler which will provide them
> +    to the HTTP server when needed. Returns the URL that is to be instead of the original one.
> +    """
> +    parsed_url = urllib.parse.urlsplit(url)
> +    if parsed_url.username != None:
> +        # Use the netloc with just the hostname, without username/password.
> +        parsed_url.netloc = parsed_url.hostname
> +        # The username/password are installed permanently in the urllib.request module
> +        # for future use with all URLs beneath url.
> +        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
> +        manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
> +        authHandler = urllib.request.HTTPBasicAuthHandler(manager)
> +        opener = urllib.request.build_opener(authHandler)
> +        urllib.request.install_opener(opener)
> +        return urllib.parse.urlunsplit(new_source)
> +    else:
> +        return url
> +
>   def download_manifests(content_url, version, component, to_dir):
>       """
>       Download one manifest file and recursively all manifests referenced by it.
> @@ -204,8 +226,8 @@ def download_old_versions(d):
>       a normal build and thus is not on the critical path.
>       """
>   
> -    content_url = d.getVar('SWUPD_CONTENT_BUILD_URL', True)
> -    version_url = d.getVar('SWUPD_VERSION_BUILD_URL', True)
> +    content_url = handle_plain_auth(d.getVar('SWUPD_CONTENT_BUILD_URL', True))
> +    version_url = handle_plain_auth(d.getVar('SWUPD_VERSION_BUILD_URL', True))
>       current_format = int(d.getVar('SWUPD_FORMAT', True))
>       deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True)
>       www_dir = os.path.join(deploy_dir, 'www')

Re: [meta-swupd][PATCH 1/1] bundles.py: allow username/password encoded in URLs

[Patrick Ohly]

On Thu, 2017-12-21 at 00:18 +0100, Ingo Flaschberger wrote:
> Dear Patrick,
> 
> this doesn't work:
>       0162:    """
>       0163:    parsed_url = urllib.parse.urlsplit(url)
>       0164:    if parsed_url.username != None:
>       0165:        # Use the netloc with just the hostname, without 
> username/password.
>   *** 0166:        parsed_url.netloc = parsed_url.hostname
>       0167:        # The username/password are installed permanently
> in 
> the urllib.request module
>       0168:        # for future use with all URLs beneath url.
>       0169:        manager = 
> urllib.request.HTTPPasswordMgrWithDefaultRealm()
>       0170:        manager.add_password(None, parsed_url, 
> parsed_url.username, parsed_url.password)
> Exception: AttributeError: can't set attribute

Looks like netloc is a read-only attribute. That means one has to
construct a new urllib.parse.SplitResult instead of updating the old
one.

> Using hostname as netloc will also remove an additional portnumber - 
> could this be a problem?

Yes, that's also something that needs to be fixed.

Can you update the patch as shown below, test it, and it if works send
the final version to the list? Obviously I am not doing a good job with
posting code that I can't test :-/

Am 20.12.2017 um 16:50 schrieb Patrick Ohly:
> > Downloading content and version information via HTTP may need a
> > username/password for basic authentication. To support this,
> > SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the
> > form http(s)://<user>:<password>@<host>/.
> > 
> > Original patch from: Ingo Flaschberger <ingo.flaschberger@gmail.com
> > >
> > 
> > Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
> > ---
> >   lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--
> >   1 file changed, 24 insertions(+), 2 deletions(-)
> > 
> > diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
> > index e1eec5a..48c7455 100644
> > --- a/lib/swupd/bundles.py
> > +++ b/lib/swupd/bundles.py
> > @@ -5,6 +5,7 @@ import subprocess
> >   import shutil
> >   import urllib.request
> >   import urllib.error
> > +import urllib.parse
> >   from bb.utils import export_proxies
> >   from oe.package_manager import RpmPM
> >   from oe.package_manager import OpkgPM
> > @@ -153,6 +154,27 @@ def copy_bundle_contents(d):
> >       for bndl in bundles:
> >           stage_empty_bundle(d, bndl)
> >   
> > +def handle_plain_auth(url):
> > +    """
> > +    Check for special urls with username/password (as in http://us
> > er:password@host/),
> > +    extract those and install an auth handler which will provide
> > them
> > +    to the HTTP server when needed. Returns the URL that is to be
> > instead of the original one.
> > +    """
> > +    parsed_url = urllib.parse.urlsplit(url)
> > +    if parsed_url.username != None:
> > +        # Use the netloc with just the hostname, without
> > username/password.
> > +        parsed_url.netloc = parsed_url.hostname

Instead:

netloc = parsed_url.hostname
if parsed_url.port is not None:
    netloc += ":%d" % parsed_url.port
new_url = urllib.parse.SplitResult(parsed_url.scheme, netloc, parsed_url.path, parsed_url.query, parsed_url.fragment)

> > +        # The username/password are installed permanently in the
> > urllib.request module
> > +        # for future use with all URLs beneath url.
> > +        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
> > +        manager.add_password(None, parsed_url,
> > parsed_url.username, parsed_url.password)
> > +        authHandler = urllib.request.HTTPBasicAuthHandler(manager)
> > +        opener = urllib.request.build_opener(authHandler)
> > +        urllib.request.install_opener(opener)
> > +        return urllib.parse.urlunsplit(new_source)

Instead:

return urllib.parse.urlunsplit(new_url)


-- 
Best Regards, Patrick Ohly

The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.

Re: [meta-swupd][PATCH 1/1] bundles.py: allow username/password encoded in URLs

[Ingo Flaschberger]

Dear Patrick,

it works if you replace:
     manager.add_password(None, parsed_url, parsed_url.username, 
parsed_url.password)
with:
      manager.add_password(None, new_url, parsed_url.username, 
parsed_url.password)

Kind regards,
     Ingo Flaschberger

Am 09.01.2018 um 17:57 schrieb Patrick Ohly:
> On Thu, 2017-12-21 at 00:18 +0100, Ingo Flaschberger wrote:
>> Dear Patrick,
>>
>> this doesn't work:
>>        0162:    """
>>        0163:    parsed_url = urllib.parse.urlsplit(url)
>>        0164:    if parsed_url.username != None:
>>        0165:        # Use the netloc with just the hostname, without
>> username/password.
>>    *** 0166:        parsed_url.netloc = parsed_url.hostname
>>        0167:        # The username/password are installed permanently
>> in
>> the urllib.request module
>>        0168:        # for future use with all URLs beneath url.
>>        0169:        manager =
>> urllib.request.HTTPPasswordMgrWithDefaultRealm()
>>        0170:        manager.add_password(None, parsed_url,
>> parsed_url.username, parsed_url.password)
>> Exception: AttributeError: can't set attribute
> Looks like netloc is a read-only attribute. That means one has to
> construct a new urllib.parse.SplitResult instead of updating the old
> one.
>
>> Using hostname as netloc will also remove an additional portnumber -
>> could this be a problem?
> Yes, that's also something that needs to be fixed.
>
> Can you update the patch as shown below, test it, and it if works send
> the final version to the list? Obviously I am not doing a good job with
> posting code that I can't test :-/
>
> Am 20.12.2017 um 16:50 schrieb Patrick Ohly:
>>> Downloading content and version information via HTTP may need a
>>> username/password for basic authentication. To support this,
>>> SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the
>>> form http(s)://<user>:<password>@<host>/.
>>>
>>> Original patch from: Ingo Flaschberger <ingo.flaschberger@gmail.com
>>> Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
>>> ---
>>>    lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--
>>>    1 file changed, 24 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
>>> index e1eec5a..48c7455 100644
>>> --- a/lib/swupd/bundles.py
>>> +++ b/lib/swupd/bundles.py
>>> @@ -5,6 +5,7 @@ import subprocess
>>>    import shutil
>>>    import urllib.request
>>>    import urllib.error
>>> +import urllib.parse
>>>    from bb.utils import export_proxies
>>>    from oe.package_manager import RpmPM
>>>    from oe.package_manager import OpkgPM
>>> @@ -153,6 +154,27 @@ def copy_bundle_contents(d):
>>>        for bndl in bundles:
>>>            stage_empty_bundle(d, bndl)
>>>    
>>> +def handle_plain_auth(url):
>>> +    """
>>> +    Check for special urls with username/password (as in http://us
>>> er:password@host/),
>>> +    extract those and install an auth handler which will provide
>>> them
>>> +    to the HTTP server when needed. Returns the URL that is to be
>>> instead of the original one.
>>> +    """
>>> +    parsed_url = urllib.parse.urlsplit(url)
>>> +    if parsed_url.username != None:
>>> +        # Use the netloc with just the hostname, without
>>> username/password.
>>> +        parsed_url.netloc = parsed_url.hostname
> Instead:
>
> netloc = parsed_url.hostname
> if parsed_url.port is not None:
>      netloc += ":%d" % parsed_url.port
> new_url = urllib.parse.SplitResult(parsed_url.scheme, netloc, parsed_url.path, parsed_url.query, parsed_url.fragment)
>
>>> +        # The username/password are installed permanently in the
>>> urllib.request module
>>> +        # for future use with all URLs beneath url.
>>> +        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
>>> +        manager.add_password(None, parsed_url,
>>> parsed_url.username, parsed_url.password)
>>> +        authHandler = urllib.request.HTTPBasicAuthHandler(manager)
>>> +        opener = urllib.request.build_opener(authHandler)
>>> +        urllib.request.install_opener(opener)
>>> +        return urllib.parse.urlunsplit(new_source)
> Instead:
>
> return urllib.parse.urlunsplit(new_url)
>
>

Re: [meta-swupd][PATCH 1/1] bundles.py: allow username/password encoded in URLs

[Patrick Ohly]

On Tue, 2018-01-09 at 19:28 +0100, Ingo Flaschberger wrote:
> Dear Patrick,
> 
> it works if you replace:
>      manager.add_password(None, parsed_url, parsed_url.username, 
> parsed_url.password)
> with:
>       manager.add_password(None, new_url, parsed_url.username, 
> parsed_url.password)

I assume this works on top of my proposal. To avoid such ambiguity and
the risk that something gets merged that is still incomplete, please
post the entire patch as tested by you. The right way would be "git
send-email", but I can also take "git diff" attached to an email.

-- 
Best Regards, Patrick Ohly

The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.

Re: [meta-swupd][PATCH 1/1] bundles.py: allow username/password encoded in URLs

[Ingo Flaschberger]

[-- Attachment #1: Type: text/plain, Size: 748 bytes --]

Dear Patrick,

attached requested patch.

Kind regards,
     Ingo Flaschberger

Am 10.01.2018 um 10:58 schrieb Patrick Ohly:
> On Tue, 2018-01-09 at 19:28 +0100, Ingo Flaschberger wrote:
>> Dear Patrick,
>>
>> it works if you replace:
>>       manager.add_password(None, parsed_url, parsed_url.username,
>> parsed_url.password)
>> with:
>>        manager.add_password(None, new_url, parsed_url.username,
>> parsed_url.password)
> I assume this works on top of my proposal. To avoid such ambiguity and
> the risk that something gets merged that is still incomplete, please
> post the entire patch as tested by you. The right way would be "git
> send-email", but I can also take "git diff" attached to an email.
>


[-- Attachment #2: swupd_auth.patch --]
[-- Type: text/plain, Size: 2880 bytes --]

diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index b4c6f49..bec95d4 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -4,6 +4,7 @@ import subprocess
 import shutil
 import urllib.request
 import urllib.error
+import urllib.parse
 from bb.utils import export_proxies
 from oe.package_manager import RpmPM
 from oe.package_manager import OpkgPM
@@ -152,6 +153,30 @@ def copy_bundle_contents(d):
     for bndl in bundles:
         stage_empty_bundle(d, bndl)
 
+def handle_plain_auth(url):
+    """
+    Check for special urls with username/password (as in http://user:password@host/),
+    extract those and install an auth handler which will provide them
+    to the HTTP server when needed. Returns the URL that is to be instead of the original one.
+    """
+    parsed_url = urllib.parse.urlsplit(url)
+    if parsed_url.username != None:
+        # Use the netloc with just the hostname, without username/password.
+        netloc = parsed_url.hostname
+        if parsed_url.port is not None:
+            netloc += ":%d" % parsed_url.port
+        new_url = urllib.parse.SplitResult(parsed_url.scheme, netloc, parsed_url.path, parsed_url.query, parsed_url.fragment)
+        # The username/password are installed permanently in the urllib.request module
+        # for future use with all URLs beneath url.
+        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+        manager.add_password(None, new_url, parsed_url.username, parsed_url.password)
+        authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+        opener = urllib.request.build_opener(authHandler)
+        urllib.request.install_opener(opener)
+        return urllib.parse.urlunsplit(new_url)
+    else:
+        return url
+
 def download_manifests(content_url, version, component, to_dir):
     """
     Download one manifest file and recursively all manifests referenced by it.
@@ -203,7 +228,7 @@ def download_old_versions(d):
     a normal build and thus is not on the critical path.
     """
 
-    content_url = d.getVar('SWUPD_CONTENT_URL', True)
+    content_url = handle_plain_auth(d.getVar('SWUPD_CONTENT_URL', True))
     version_url = d.getVar('SWUPD_VERSION_URL', True)
     current_format = int(d.getVar('SWUPD_FORMAT', True))
     deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True)
@@ -240,6 +265,11 @@ def download_old_versions(d):
                 bb.debug(1, '%s does not exist, skipping that format' % url)
             else:
                 raise
+        except urllib.error.URLError as url_error:
+            if re.search( 'No such file or directory', str(url_error.reason)):
+                bb.debug(1, '%s does not exist, skipping that format' % url)
+            else:
+                raise
 
     # Now get the Manifests of the latest versions and the
     # versions we are supposed to provide a delta for, as a starting point.