From a4389e086013f07a3b17f8e8c140dc60b9d4d965 Mon Sep 17 00:00:00 2001 From: Guillaume Tucker Date: Thu, 22 Jun 2023 20:55:31 +0200 Subject: [PATCH 1/3] kernelci.config: move event condition to scheduler Remove the Job.run_on list of conditions and replace it with a single event attribute in the scheduler entries since we can already define multiple entries for a same job. This is as discussed on the mailing list and GitHub. Update unit tests accordingly. Link: https://lore.kernel.org/r/09c2d79d-27f3-1cb1-0d52-df003926a132@collabora.com Signed-off-by: Guillaume Tucker --- kernelci/config/job.py | 11 ++--------- kernelci/config/scheduler.py | 10 ++++++++-- tests/configs/jobs.yaml | 11 ----------- tests/configs/scheduler.yaml | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/kernelci/config/job.py b/kernelci/config/job.py index 405af6a973..1f87338fd0 100644 --- a/kernelci/config/job.py +++ b/kernelci/config/job.py @@ -14,12 +14,10 @@ class Job(YAMLConfigObject): yaml_tag = '!Job' # pylint: disable=too-many-arguments - def __init__(self, name, template, image=None, - run_on=None, params=None): + def __init__(self, name, template, image=None, params=None): self._name = name self._template = template self._image = image - self._run_on = run_on or [] self._params = params or {} @property @@ -37,11 +35,6 @@ def image(self): """Runtime environment image name""" return self._image - @property - def run_on(self): - """Criteria for the job to be run""" - return list(dict(crit) for crit in self._run_on) - @property def params(self): """Arbitrary parameters passed to the template""" @@ -50,7 +43,7 @@ def params(self): @classmethod def _get_yaml_attributes(cls): attrs = super()._get_yaml_attributes() - attrs.update({'template', 'image', 'run_on', 'params'}) + attrs.update({'template', 'image', 'params'}) return attrs diff --git a/kernelci/config/scheduler.py b/kernelci/config/scheduler.py index d0cd18e523..9bcfb28a70 100644 --- a/kernelci/config/scheduler.py +++ b/kernelci/config/scheduler.py @@ -13,9 +13,10 @@ class SchedulerEntry(YAMLConfigObject): yaml_tag = '!SchedulerEntry' - def __init__(self, job, runtime, platforms=None): + def __init__(self, job, runtime, event, platforms=None): self._job = job self._runtime = runtime + self._event = event self._platforms = platforms or [] @property @@ -28,6 +29,11 @@ def runtime(self): """Runtime parameters (name or type)""" return dict(self._runtime) + @property + def event(self): + """Criteria for an event to cause the job to be run""" + return dict(self._event) + @property def platforms(self): """List of platform names""" @@ -36,7 +42,7 @@ def platforms(self): @classmethod def _get_yaml_attributes(cls): attrs = super()._get_yaml_attributes() - attrs.update({'job', 'runtime', 'platforms'}) + attrs.update({'job', 'runtime', 'event', 'platforms'}) return attrs diff --git a/tests/configs/jobs.yaml b/tests/configs/jobs.yaml index 70cef9117b..2a76bca44b 100644 --- a/tests/configs/jobs.yaml +++ b/tests/configs/jobs.yaml @@ -1,24 +1,14 @@ -_aliases: - - checkout: &checkout - - event: node - name: checkout - result: pass - - jobs: kbuild-gcc-10-x86: template: 'kbuild.jinja2' image: 'gcc-10:{arch}{fragments}' - run_on: *checkout params: config: x86_64_defconfig kunit: &kunit-job template: 'kunit.jinja2' image: 'gcc-10:x86-kunit-kernelci' - run_on: *checkout params: {} kunit-x86_64: @@ -30,5 +20,4 @@ jobs: kver: template: 'kver.jinja2' image: 'kernelci' - run_on: *checkout params: {} diff --git a/tests/configs/scheduler.yaml b/tests/configs/scheduler.yaml index f7e66edc25..3023c62e7e 100644 --- a/tests/configs/scheduler.yaml +++ b/tests/configs/scheduler.yaml @@ -1,16 +1,30 @@ +_aliases: + + checkout: &checkout + event: node + name: checkout + result: pass + + scheduler: - job: baseline-x86 + event: + channel: node + name: kbuild-gcc-10-x86 + result: pass runtime: type: lava platforms: [qemu-x86] - job: kunit + event: *checkout runtime: name: k8s-gke-eu-west4 platforms: [] - job: kbuild-gcc-10-x86 + event: *checkout runtime: type: kubernetes platforms: [] From 34074f43f0ba9d396e7042a20bf5b3f9106903a1 Mon Sep 17 00:00:00 2001 From: Guillaume Tucker Date: Thu, 22 Jun 2023 21:10:54 +0200 Subject: [PATCH 2/3] kernelci.scheduler: use scheduler config event attribute Update the Scheduler.get_configs() implementation to rely on the event attribute in scheduler config entries to work out which jobs to run rather than on the Job.run_on attribute which has now been removed. This simplifies the logic and makes it more flexible. Signed-off-by: Guillaume Tucker --- kernelci/scheduler.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/kernelci/scheduler.py b/kernelci/scheduler.py index bb69042b30..918acc97a9 100644 --- a/kernelci/scheduler.py +++ b/kernelci/scheduler.py @@ -29,22 +29,14 @@ def __init__(self, configs, runtimes): runtime_type.append(runtime) self._platforms = configs['device_types'] - def get_jobs(self, event, channel='node'): - """Get the job configs matching a given event""" - for _, job in self._jobs.items(): - for run_on in job.run_on: - run_on_channel = run_on.get('channel') - if run_on_channel == channel: - run_on = run_on.copy() - run_on.pop('channel') - if run_on.items() <= event.items(): - yield job - def get_configs(self, event, channel='node'): - """Get the scheduler configs to run jobs matching a given event""" - for job in self.get_jobs(event, channel): - for entry in self._scheduler: - if entry.job == job.name: + """Get the scheduler configs matching a given event""" + for entry in self._scheduler: + sched_event_channel = entry.event.get('channel') + if sched_event_channel == channel: + sched_event = entry.event.copy() + sched_event.pop('channel') + if sched_event.items() <= event.items(): yield entry def get_schedule(self, event, channel='node'): From 69c52ee3bbd1dff288d5f8694383c72ac201dabf Mon Sep 17 00:00:00 2001 From: Guillaume Tucker Date: Thu, 22 Jun 2023 21:11:09 +0200 Subject: [PATCH 3/3] kernelci.cli: sched: drop list_jobs command Remove the `kci sched list_jobs` command as event criteria for running jobs is now specified in the scheduler config entries rather than in the job configs. Signed-off-by: Guillaume Tucker --- kernelci/cli/sched.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/kernelci/cli/sched.py b/kernelci/cli/sched.py index 3a295f4856..b9d64fcde1 100644 --- a/kernelci/cli/sched.py +++ b/kernelci/cli/sched.py @@ -13,26 +13,6 @@ from .base import Args, Command, sub_main -class cmd_list_jobs(Command): # pylint: disable=invalid-name - """List the jobs to run matching an event provided on stdin""" - opt_args = Command.opt_args + [ - { - 'name': '--channel', - 'help': "Name of the pub/sub channel, or 'node' by default", - }, - ] - - def __call__(self, configs, args): - rconfigs = configs['runtimes'] - runtimes = dict(kernelci.runtime.get_all_runtimes(rconfigs, args)) - sched = kernelci.scheduler.Scheduler(configs, runtimes) - event = json.loads(sys.stdin.read()) - channel = args.channel or 'node' - for job in sched.get_jobs(event, channel): - print(job.name) - return True - - class cmd_get_schedule(Command): # pylint: disable=invalid-name """Get scheduler configs matching an event provided on stdin""" opt_args = Command.opt_args + [