[PATCH] Make client behavior configurable: drop_caches

[PATCH] Make client behavior configurable: drop_caches

[Lucas Meneghel Rodrigues]

Right now autotest will drop caches between:
 * Test executions
 * Same test iterations

This change turns those into configurable options on
global_config.ini. Default configuration:

[CLIENT]
drop_caches: True
drop_caches_between_iterations: True

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/bin/autotest |    7 ++++++-
 client/bin/job.py   |   11 ++++++++---
 global_config.ini   |    7 ++++++-
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/client/bin/autotest b/client/bin/autotest
index fe8f2c7..2d47843 100755
--- a/client/bin/autotest
+++ b/client/bin/autotest
@@ -6,6 +6,7 @@ import os, sys, shutil
 import common
 from optparse import OptionParser
 from autotest_lib.client.bin import job
+from autotest_lib.client.common_lib import global_config
 
 
 # Use the name of the binary to find the real installation directory
@@ -58,5 +59,9 @@ options, args = parser.parse_args()
 if len(args) != 1:
     usage()
 
+drop_caches = global_config.global_config.get_config_value('CLIENT',
+                                                           'drop_caches',
+                                                           type=bool)
+
 # JOB: run the specified job control file.
-job.runjob(os.path.realpath(args[0]), options)
+job.runjob(os.path.realpath(args[0]), drop_caches, options)
diff --git a/client/bin/job.py b/client/bin/job.py
index ebfb3a3..6099188 100755
--- a/client/bin/job.py
+++ b/client/bin/job.py
@@ -15,6 +15,8 @@ from autotest_lib.client.bin import config, sysinfo, test, local_host
 from autotest_lib.client.bin import partition as partition_lib
 from autotest_lib.client.common_lib import error, barrier, log, logging_manager
 from autotest_lib.client.common_lib import base_packages, packages
+from autotest_lib.client.common_lib import global_config
+
 
 LAST_BOOT_TAG = object()
 NO_DEFAULT = object()
@@ -251,7 +253,10 @@ class base_job(object):
         """
         Perform the drop caches initialization.
         """
-        self.drop_caches_between_iterations = True
+        self.drop_caches_between_iterations = (
+                       global_config.global_config.get_config_value('CLIENT',
+                                            'drop_caches_between_iterations',
+                                            type=bool))
         self.drop_caches = drop_caches
         if self.drop_caches:
             logging.debug("Dropping caches")
@@ -1339,7 +1344,7 @@ class disk_usage_monitor:
         return decorator
 
 
-def runjob(control, options):
+def runjob(control, drop_caches, options):
     """
     Run a job using the given control file.
 
@@ -1367,7 +1372,7 @@ def runjob(control, options):
         if options.cont and not os.path.exists(state):
             raise error.JobComplete("all done")
 
-        myjob = job(control, options)
+        myjob = job(control=control, drop_caches=drop_caches, options=options)
 
         # Load in the users control file, may do any one of:
         #  1) execute in toto
diff --git a/global_config.ini b/global_config.ini
index cc20a96..d018374 100644
--- a/global_config.ini
+++ b/global_config.ini
@@ -28,7 +28,6 @@ parse_failed_repair_default: 0
 # Autotest potential install paths
 client_autodir_paths: /usr/local/autotest,/home/autotest
 
-
 [SERVER]
 hostname: autotest
 # Turn on RPC Logging
@@ -48,6 +47,12 @@ smtp_port:
 smtp_user:
 smtp_password:
 
+[CLIENT]
+# Drop test client caches between every test execution
+drop_caches: True
+# Drop test client caches between every test iteration execution
+drop_caches_between_iterations: True
+
 [SCHEDULER]
 die_on_orphans: False
 enable_scheduler: True
-- 
1.6.2.5

Re: [PATCH] Make client behavior configurable: drop_caches

[John Admanski]

Will this code actually work on a standalone client job? I'm not sure
we've ever used global_config stuff outside of the server (despite the
fact that the code lives in the common_lib).

-- John

On Thu, Nov 5, 2009 at 12:23 PM, Lucas Meneghel Rodrigues
<lmr@redhat.com> wrote:
> Right now autotest will drop caches between:
>  * Test executions
>  * Same test iterations
>
> This change turns those into configurable options on
> global_config.ini. Default configuration:
>
> [CLIENT]
> drop_caches: True
> drop_caches_between_iterations: True
>
> Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
> ---
>  client/bin/autotest |    7 ++++++-
>  client/bin/job.py   |   11 ++++++++---
>  global_config.ini   |    7 ++++++-
>  3 files changed, 20 insertions(+), 5 deletions(-)
>
> diff --git a/client/bin/autotest b/client/bin/autotest
> index fe8f2c7..2d47843 100755
> --- a/client/bin/autotest
> +++ b/client/bin/autotest
> @@ -6,6 +6,7 @@ import os, sys, shutil
>  import common
>  from optparse import OptionParser
>  from autotest_lib.client.bin import job
> +from autotest_lib.client.common_lib import global_config
>
>
>  # Use the name of the binary to find the real installation directory
> @@ -58,5 +59,9 @@ options, args = parser.parse_args()
>  if len(args) != 1:
>     usage()
>
> +drop_caches = global_config.global_config.get_config_value('CLIENT',
> +                                                           'drop_caches',
> +                                                           type=bool)
> +
>  # JOB: run the specified job control file.
> -job.runjob(os.path.realpath(args[0]), options)
> +job.runjob(os.path.realpath(args[0]), drop_caches, options)
> diff --git a/client/bin/job.py b/client/bin/job.py
> index ebfb3a3..6099188 100755
> --- a/client/bin/job.py
> +++ b/client/bin/job.py
> @@ -15,6 +15,8 @@ from autotest_lib.client.bin import config, sysinfo, test, local_host
>  from autotest_lib.client.bin import partition as partition_lib
>  from autotest_lib.client.common_lib import error, barrier, log, logging_manager
>  from autotest_lib.client.common_lib import base_packages, packages
> +from autotest_lib.client.common_lib import global_config
> +
>
>  LAST_BOOT_TAG = object()
>  NO_DEFAULT = object()
> @@ -251,7 +253,10 @@ class base_job(object):
>         """
>         Perform the drop caches initialization.
>         """
> -        self.drop_caches_between_iterations = True
> +        self.drop_caches_between_iterations = (
> +                       global_config.global_config.get_config_value('CLIENT',
> +                                            'drop_caches_between_iterations',
> +                                            type=bool))
>         self.drop_caches = drop_caches
>         if self.drop_caches:
>             logging.debug("Dropping caches")
> @@ -1339,7 +1344,7 @@ class disk_usage_monitor:
>         return decorator
>
>
> -def runjob(control, options):
> +def runjob(control, drop_caches, options):
>     """
>     Run a job using the given control file.
>
> @@ -1367,7 +1372,7 @@ def runjob(control, options):
>         if options.cont and not os.path.exists(state):
>             raise error.JobComplete("all done")
>
> -        myjob = job(control, options)
> +        myjob = job(control=control, drop_caches=drop_caches, options=options)
>
>         # Load in the users control file, may do any one of:
>         #  1) execute in toto
> diff --git a/global_config.ini b/global_config.ini
> index cc20a96..d018374 100644
> --- a/global_config.ini
> +++ b/global_config.ini
> @@ -28,7 +28,6 @@ parse_failed_repair_default: 0
>  # Autotest potential install paths
>  client_autodir_paths: /usr/local/autotest,/home/autotest
>
> -
>  [SERVER]
>  hostname: autotest
>  # Turn on RPC Logging
> @@ -48,6 +47,12 @@ smtp_port:
>  smtp_user:
>  smtp_password:
>
> +[CLIENT]
> +# Drop test client caches between every test execution
> +drop_caches: True
> +# Drop test client caches between every test iteration execution
> +drop_caches_between_iterations: True
> +
>  [SCHEDULER]
>  die_on_orphans: False
>  enable_scheduler: True
> --
> 1.6.2.5
>
>

Re: [PATCH] Make client behavior configurable: drop_caches

[Lucas Meneghel Rodrigues]

On Tue, 2009-11-10 at 10:28 -0800, John Admanski wrote:
> Will this code actually work on a standalone client job? I'm not sure
> we've ever used global_config stuff outside of the server (despite the
> fact that the code lives in the common_lib).

Whoops, I just forgot that global_config.ini is a top level file. 2
possible solutions:

1 - Move global_config.ini to client/
2 - Create a specific configuration file for client.

But yes, the patch has already been applied and introduces a regression
since the stand alone client won't be able to see global_config.ini. I
am sorry.

I believe solution 1) is not bad, client/global_config.ini is just
slightly non intuitive, but splitting conf files does not sound like a
better idea.

> -- John
> 
> On Thu, Nov 5, 2009 at 12:23 PM, Lucas Meneghel Rodrigues
> <lmr@redhat.com> wrote:
> > Right now autotest will drop caches between:
> >  * Test executions
> >  * Same test iterations
> >
> > This change turns those into configurable options on
> > global_config.ini. Default configuration:
> >
> > [CLIENT]
> > drop_caches: True
> > drop_caches_between_iterations: True
> >
> > Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
> > ---
> >  client/bin/autotest |    7 ++++++-
> >  client/bin/job.py   |   11 ++++++++---
> >  global_config.ini   |    7 ++++++-
> >  3 files changed, 20 insertions(+), 5 deletions(-)
> >
> > diff --git a/client/bin/autotest b/client/bin/autotest
> > index fe8f2c7..2d47843 100755
> > --- a/client/bin/autotest
> > +++ b/client/bin/autotest
> > @@ -6,6 +6,7 @@ import os, sys, shutil
> >  import common
> >  from optparse import OptionParser
> >  from autotest_lib.client.bin import job
> > +from autotest_lib.client.common_lib import global_config
> >
> >
> >  # Use the name of the binary to find the real installation directory
> > @@ -58,5 +59,9 @@ options, args = parser.parse_args()
> >  if len(args) != 1:
> >     usage()
> >
> > +drop_caches = global_config.global_config.get_config_value('CLIENT',
> > +                                                           'drop_caches',
> > +                                                           type=bool)
> > +
> >  # JOB: run the specified job control file.
> > -job.runjob(os.path.realpath(args[0]), options)
> > +job.runjob(os.path.realpath(args[0]), drop_caches, options)
> > diff --git a/client/bin/job.py b/client/bin/job.py
> > index ebfb3a3..6099188 100755
> > --- a/client/bin/job.py
> > +++ b/client/bin/job.py
> > @@ -15,6 +15,8 @@ from autotest_lib.client.bin import config, sysinfo, test, local_host
> >  from autotest_lib.client.bin import partition as partition_lib
> >  from autotest_lib.client.common_lib import error, barrier, log, logging_manager
> >  from autotest_lib.client.common_lib import base_packages, packages
> > +from autotest_lib.client.common_lib import global_config
> > +
> >
> >  LAST_BOOT_TAG = object()
> >  NO_DEFAULT = object()
> > @@ -251,7 +253,10 @@ class base_job(object):
> >         """
> >         Perform the drop caches initialization.
> >         """
> > -        self.drop_caches_between_iterations = True
> > +        self.drop_caches_between_iterations = (
> > +                       global_config.global_config.get_config_value('CLIENT',
> > +                                            'drop_caches_between_iterations',
> > +                                            type=bool))
> >         self.drop_caches = drop_caches
> >         if self.drop_caches:
> >             logging.debug("Dropping caches")
> > @@ -1339,7 +1344,7 @@ class disk_usage_monitor:
> >         return decorator
> >
> >
> > -def runjob(control, options):
> > +def runjob(control, drop_caches, options):
> >     """
> >     Run a job using the given control file.
> >
> > @@ -1367,7 +1372,7 @@ def runjob(control, options):
> >         if options.cont and not os.path.exists(state):
> >             raise error.JobComplete("all done")
> >
> > -        myjob = job(control, options)
> > +        myjob = job(control=control, drop_caches=drop_caches, options=options)
> >
> >         # Load in the users control file, may do any one of:
> >         #  1) execute in toto
> > diff --git a/global_config.ini b/global_config.ini
> > index cc20a96..d018374 100644
> > --- a/global_config.ini
> > +++ b/global_config.ini
> > @@ -28,7 +28,6 @@ parse_failed_repair_default: 0
> >  # Autotest potential install paths
> >  client_autodir_paths: /usr/local/autotest,/home/autotest
> >
> > -
> >  [SERVER]
> >  hostname: autotest
> >  # Turn on RPC Logging
> > @@ -48,6 +47,12 @@ smtp_port:
> >  smtp_user:
> >  smtp_password:
> >
> > +[CLIENT]
> > +# Drop test client caches between every test execution
> > +drop_caches: True
> > +# Drop test client caches between every test iteration execution
> > +drop_caches_between_iterations: True
> > +
> >  [SCHEDULER]
> >  die_on_orphans: False
> >  enable_scheduler: True
> > --
> > 1.6.2.5
> >
> >
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [Autotest] [PATCH] Make client behavior configurable: drop_caches

[Lucas Meneghel Rodrigues]

On Tue, Nov 10, 2009 at 4:28 PM, John Admanski <jadmanski@google.com> wrote:
> Will this code actually work on a standalone client job? I'm not sure
> we've ever used global_config stuff outside of the server (despite the
> fact that the code lives in the common_lib).

I've sent 2 patches to the mailing list:

1) Fix an incorrect directory reference on the job code (fixes autoserv)
2) Moves the config files to the client directory (fixes autotest stand alone)

Both were tested and does not seem to regress the unittest suites.

I hope that with this patch things will get straightened out.

> -- John
>
> On Thu, Nov 5, 2009 at 12:23 PM, Lucas Meneghel Rodrigues
> <lmr@redhat.com> wrote:
>> Right now autotest will drop caches between:
>>  * Test executions
>>  * Same test iterations
>>
>> This change turns those into configurable options on
>> global_config.ini. Default configuration:
>>
>> [CLIENT]
>> drop_caches: True
>> drop_caches_between_iterations: True
>>
>> Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
>> ---
>>  client/bin/autotest |    7 ++++++-
>>  client/bin/job.py   |   11 ++++++++---
>>  global_config.ini   |    7 ++++++-
>>  3 files changed, 20 insertions(+), 5 deletions(-)
>>
>> diff --git a/client/bin/autotest b/client/bin/autotest
>> index fe8f2c7..2d47843 100755
>> --- a/client/bin/autotest
>> +++ b/client/bin/autotest
>> @@ -6,6 +6,7 @@ import os, sys, shutil
>>  import common
>>  from optparse import OptionParser
>>  from autotest_lib.client.bin import job
>> +from autotest_lib.client.common_lib import global_config
>>
>>
>>  # Use the name of the binary to find the real installation directory
>> @@ -58,5 +59,9 @@ options, args = parser.parse_args()
>>  if len(args) != 1:
>>     usage()
>>
>> +drop_caches = global_config.global_config.get_config_value('CLIENT',
>> +                                                           'drop_caches',
>> +                                                           type=bool)
>> +
>>  # JOB: run the specified job control file.
>> -job.runjob(os.path.realpath(args[0]), options)
>> +job.runjob(os.path.realpath(args[0]), drop_caches, options)
>> diff --git a/client/bin/job.py b/client/bin/job.py
>> index ebfb3a3..6099188 100755
>> --- a/client/bin/job.py
>> +++ b/client/bin/job.py
>> @@ -15,6 +15,8 @@ from autotest_lib.client.bin import config, sysinfo, test, local_host
>>  from autotest_lib.client.bin import partition as partition_lib
>>  from autotest_lib.client.common_lib import error, barrier, log, logging_manager
>>  from autotest_lib.client.common_lib import base_packages, packages
>> +from autotest_lib.client.common_lib import global_config
>> +
>>
>>  LAST_BOOT_TAG = object()
>>  NO_DEFAULT = object()
>> @@ -251,7 +253,10 @@ class base_job(object):
>>         """
>>         Perform the drop caches initialization.
>>         """
>> -        self.drop_caches_between_iterations = True
>> +        self.drop_caches_between_iterations = (
>> +                       global_config.global_config.get_config_value('CLIENT',
>> +                                            'drop_caches_between_iterations',
>> +                                            type=bool))
>>         self.drop_caches = drop_caches
>>         if self.drop_caches:
>>             logging.debug("Dropping caches")
>> @@ -1339,7 +1344,7 @@ class disk_usage_monitor:
>>         return decorator
>>
>>
>> -def runjob(control, options):
>> +def runjob(control, drop_caches, options):
>>     """
>>     Run a job using the given control file.
>>
>> @@ -1367,7 +1372,7 @@ def runjob(control, options):
>>         if options.cont and not os.path.exists(state):
>>             raise error.JobComplete("all done")
>>
>> -        myjob = job(control, options)
>> +        myjob = job(control=control, drop_caches=drop_caches, options=options)
>>
>>         # Load in the users control file, may do any one of:
>>         #  1) execute in toto
>> diff --git a/global_config.ini b/global_config.ini
>> index cc20a96..d018374 100644
>> --- a/global_config.ini
>> +++ b/global_config.ini
>> @@ -28,7 +28,6 @@ parse_failed_repair_default: 0
>>  # Autotest potential install paths
>>  client_autodir_paths: /usr/local/autotest,/home/autotest
>>
>> -
>>  [SERVER]
>>  hostname: autotest
>>  # Turn on RPC Logging
>> @@ -48,6 +47,12 @@ smtp_port:
>>  smtp_user:
>>  smtp_password:
>>
>> +[CLIENT]
>> +# Drop test client caches between every test execution
>> +drop_caches: True
>> +# Drop test client caches between every test iteration execution
>> +drop_caches_between_iterations: True
>> +
>>  [SCHEDULER]
>>  die_on_orphans: False
>>  enable_scheduler: True
>> --
>> 1.6.2.5
>>
>>
> _______________________________________________
> Autotest mailing list
> Autotest@test.kernel.org
> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
>



-- 
Lucas