[PATCH 1/2] fetch2/wget: allow identifying with a bitbake user agent

[Johan Anderholm <johanam@axis.com>]

Certain hosts do not allow identifying as wget, but want an application
specific user agent.

This change causes a slight change in how checkstatus will identify.
Instead of simply using bitbake/<version>, it will now also include
project contact information:

bitbake/<version> (https://lists.openembedded.org/g/bitbake-devel; bitbake-devel@lists.openembedded.org)

This change was made to comply with the data access policy of crates.io,
while not introducing too many user agent variants.

Some more information on user agents can be found in 234f9e810494

Signed-off-by: Johan Anderholm <johanam@axis.com>
---
 lib/bb/fetch2/wget.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index 6ac4306c0..93c8b73f4 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -60,6 +60,19 @@ class Wget(FetchMethod):
         """
         return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0"
 
+    def identify_as_bitbake(self):
+        """
+        Should wget identify as bitbake?
+        """
+        return False
+
+    def bitbake_user_agent(self):
+        """
+        Certain hosts require users to identify with an application specific user-agent.
+        E.g. crates.io according to https://crates.io/data-access
+        """
+        return "bitbake/{} (https://lists.openembedded.org/g/bitbake-devel; bitbake-devel@lists.openembedded.org)".format(bb.__version__)
+
     def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with wget.
@@ -122,6 +135,10 @@ class Wget(FetchMethod):
                 # Authorization header.
                 fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
 
+        # Some hosts, e.g. crates.io, may require using a special user-agent.
+        if self.identify_as_bitbake():
+            fetchcmd += " --user-agent='%s'" % self.bitbake_user_agent()
+
         uri = ud.url.split(";")[0]
         fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri)
         self._runwget(ud, d, fetchcmd, False)
@@ -371,7 +388,7 @@ class Wget(FetchMethod):
                 # Some servers (FusionForge, as used on Alioth) require that the
                 # optional Accept header is set.
                 r.add_header("Accept", "*/*")
-                r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
+                r.add_header("User-Agent", self.bitbake_user_agent())
                 def add_basic_auth(login_str, request):
                     '''Adds Basic auth to http request, pass in login:password as string'''
                     import base64
-- 
2.34.1