[Buildroot] [PATCH buildroot-test 1/2] scripts/autobuild-run: make the HTTP URL really configurable

[Buildroot] [PATCH buildroot-test 1/2] scripts/autobuild-run: make the HTTP URL really configurable

[Thomas Petazzoni]

The --http-url option allowed to customize the URL at which build
results are submitted. However, there were two other places in the
script where autobuild.buildroot.org was hardcoded: when checking the
script version, and when getting the list of branches to build.

This commit changes the --http-url to be the base URL of the autobuild
server, and it is used everywhere instead of hardcoding
autobuild.buildroot.org.

Note: users of autobuild-run that were passing a custom --http-url
option such as http://foo.com/submit/ should change it to just
http://foo.com/

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
---
 scripts/autobuild-run | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/autobuild-run b/scripts/autobuild-run
index 601fb31..6fea389 100755
--- a/scripts/autobuild-run
+++ b/scripts/autobuild-run
@@ -66,7 +66,7 @@ defaults = {
     '--make-opts': '',
     '--nice': 0,
     '--pid-file': '/tmp/buildroot-autobuild.pid',
-    '--http-url': 'http://autobuild.buildroot.org/submit/',
+    '--http-url': 'http://autobuild.buildroot.org/',
     '--toolchains-csv': 'support/config-fragments/autobuild/toolchain-configs.csv',
     '--repo': 'https://github.com/buildroot/buildroot.git',
 }
@@ -145,6 +145,7 @@ from distutils.version import StrictVersion
 import platform
 from threading import Thread, Event
 import datetime
+import urlparse
 
 if sys.hexversion >= 0x3000000:
     import configparser
@@ -178,23 +179,23 @@ def log_write(logf, msg):
     logf.write("[%s] %s\n" % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), msg))
     logf.flush()
 
-def check_version():
-    with urlopen_closing('http://autobuild.buildroot.org/version') as r:
+def check_version(**kwargs):
+    with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'version')) as r:
         version = int(decode_bytes(r.readline()).strip())
     if version > VERSION:
         print("ERROR: script version too old, please upgrade.")
         sys.exit(1)
 
-def get_branch():
+def get_branch(**kwargs):
     """Returns the branch that should be built. It fetches a CSV file from
-    autobuild.buildroot.org that provides the list of branches to test
+    the autobuild server that provides the list of branches to test
     (first field) and their weight (second field). We build a list of
     branches, where the branch name is repeated N times, with N being
     the weight of the branch. We then pick a random branch in this
     list. This way, branches with a higher weight are more likely to
     be selected.
     """
-    with urlopen_closing('http://autobuild.buildroot.org/branches') as r:
+    with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'branches')) as r:
         csv_branches = r.readlines()
     branches = []
     for branch in csv.reader(csv_branches):
@@ -314,7 +315,7 @@ def prepare_build(**kwargs):
                   os.path.relpath(f, dldir))
         os.remove(f)
 
-    branch = get_branch()
+    branch = get_branch(**kwargs)
     log_write(log, "INFO: testing branch '%s'" % branch)
 
     # Clone Buildroot. This only happens if the source directory
@@ -679,7 +680,7 @@ def send_results(result, **kwargs):
                                "-H", "Expect:",
                                "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"),
                                "-F", "uploadsubmit=1",
-                               kwargs['http_url']],
+                               urlparse.urljoin(kwargs['http_url'], 'submit/')],
                               stdout=log, stderr=log)
         if ret != 0:
             log_write(log, "INFO: results could not be submitted, %d" % ret)
@@ -713,7 +714,7 @@ def run_instance(**kwargs):
     log_write(kwargs['log'], "INFO: instance started")
 
     while True:
-        check_version()
+        check_version(**kwargs)
 
         ret = prepare_build(**kwargs)
         if ret != 0:
@@ -774,7 +775,6 @@ def main():
     # showing error messages in another language.
     os.environ['LC_ALL'] = 'C'
 
-    check_version()
     sysinfo = SystemInfo()
 
     args = docopt.docopt(doc, version=VERSION)
-- 
2.21.0

[Buildroot] [PATCH buildroot-test 2/2] scripts/autobuild-run: support changing repo

[Thomas Petazzoni]

The current logic in prepare_build() assumes that the "origin" repo
never changes. However, if one regularly changes his autobuild-run
configuration, switching being repository, this is not
true. Currently, it requires manually wiping out the Buildroot clone
in every autobuild instance when changing the repository to pull from.

So instead, use:

 git fetch <repo> <branch>
 git checkout FETCH_HEAD

which will easily allow switching from one repo to the other.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
---
 scripts/autobuild-run | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/autobuild-run b/scripts/autobuild-run
index 6fea389..ce88ea4 100755
--- a/scripts/autobuild-run
+++ b/scripts/autobuild-run
@@ -330,12 +330,12 @@ def prepare_build(**kwargs):
 
     # Update the Buildroot sources.
     abssrcdir = os.path.abspath(srcdir)
-    ret = subprocess.call(["git", "fetch", "origin"], cwd=abssrcdir, stdout=log, stderr=log)
+    ret = subprocess.call(["git", "fetch", kwargs['repo'], branch], cwd=abssrcdir, stdout=log, stderr=log)
     if ret != 0:
         log_write(log, "ERROR: could not fetch Buildroot sources")
         return -1
 
-    ret = subprocess.call(["git", "checkout", "--detach", "origin/%s" % branch], cwd=abssrcdir, stdout=log, stderr=log)
+    ret = subprocess.call(["git", "checkout", "FETCH_HEAD"], cwd=abssrcdir, stdout=log, stderr=log)
     if ret != 0:
         log_write(log, "ERROR: could not check out Buildroot sources")
         return -1
-- 
2.21.0

[Buildroot] [PATCH buildroot-test 1/2] scripts/autobuild-run: make the HTTP URL really configurable

[Yann E. MORIN]

Thomas, All,

On 2019-06-17 20:22 +0200, Thomas Petazzoni spake thusly:
> The --http-url option allowed to customize the URL at which build
> results are submitted. However, there were two other places in the
> script where autobuild.buildroot.org was hardcoded: when checking the
> script version, and when getting the list of branches to build.
> 
> This commit changes the --http-url to be the base URL of the autobuild
> server, and it is used everywhere instead of hardcoding
> autobuild.buildroot.org.
> 
> Note: users of autobuild-run that were passing a custom --http-url
> option such as http://foo.com/submit/ should change it to just
> http://foo.com/
> 
> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
> ---
[--SNIP--]
> @@ -178,23 +179,23 @@ def log_write(logf, msg):
>      logf.write("[%s] %s\n" % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), msg))
>      logf.flush()
>  
> -def check_version():
> -    with urlopen_closing('http://autobuild.buildroot.org/version') as r:
> +def check_version(**kwargs):

There is a current work by Atharva to actually remove usage of kwargs
elsewhere in the code.

I think it would be nice to have a new object that contains the server
config:

    class ServerConfig():
        def __init__(self, base_url):
            self.base_url = base_url
        def branches_url(self):
            return urlparse.urljoin(self.base_url, 'branches')
        def submit_url(self):
            return urlparse.urljoin(self.base_url, 'submit')

And then in main (or elsewhere it is meaningful):

    server_cfg = ServerConfig(args['http_url'])

and pass that object to run_instance().

Then the series from Atharva can more easily account for this new
variable.

Regards,
Yann E. MORIN.

> +    with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'version')) as r:
>          version = int(decode_bytes(r.readline()).strip())
>      if version > VERSION:
>          print("ERROR: script version too old, please upgrade.")
>          sys.exit(1)
>  
> -def get_branch():
> +def get_branch(**kwargs):
>      """Returns the branch that should be built. It fetches a CSV file from
> -    autobuild.buildroot.org that provides the list of branches to test
> +    the autobuild server that provides the list of branches to test
>      (first field) and their weight (second field). We build a list of
>      branches, where the branch name is repeated N times, with N being
>      the weight of the branch. We then pick a random branch in this
>      list. This way, branches with a higher weight are more likely to
>      be selected.
>      """
> -    with urlopen_closing('http://autobuild.buildroot.org/branches') as r:
> +    with urlopen_closing(urlparse.urljoin(kwargs['http_url'], 'branches')) as r:
>          csv_branches = r.readlines()
>      branches = []
>      for branch in csv.reader(csv_branches):
> @@ -314,7 +315,7 @@ def prepare_build(**kwargs):
>                    os.path.relpath(f, dldir))
>          os.remove(f)
>  
> -    branch = get_branch()
> +    branch = get_branch(**kwargs)
>      log_write(log, "INFO: testing branch '%s'" % branch)
>  
>      # Clone Buildroot. This only happens if the source directory
> @@ -679,7 +680,7 @@ def send_results(result, **kwargs):
>                                 "-H", "Expect:",
>                                 "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"),
>                                 "-F", "uploadsubmit=1",
> -                               kwargs['http_url']],
> +                               urlparse.urljoin(kwargs['http_url'], 'submit/')],
>                                stdout=log, stderr=log)
>          if ret != 0:
>              log_write(log, "INFO: results could not be submitted, %d" % ret)
> @@ -713,7 +714,7 @@ def run_instance(**kwargs):
>      log_write(kwargs['log'], "INFO: instance started")
>  
>      while True:
> -        check_version()
> +        check_version(**kwargs)
>  
>          ret = prepare_build(**kwargs)
>          if ret != 0:
> @@ -774,7 +775,6 @@ def main():
>      # showing error messages in another language.
>      os.environ['LC_ALL'] = 'C'
>  
> -    check_version()
>      sysinfo = SystemInfo()
>  
>      args = docopt.docopt(doc, version=VERSION)
> -- 
> 2.21.0
> 
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'