Re: [PATCH v4 11/14] perf pmu-events: Parallelize JSON and metric pre-computation in jevents.py

[Namhyung Kim <namhyung@kernel.org>]

On Fri, May 15, 2026 at 10:38:48AM -0700, Ian Rogers wrote:
> Currently, jevents.py parses hundreds of JSON event and metric files
> sequentially across all CPU architectures during Kbuild startup, taking
> ~3.3 seconds of pure single-core execution time.

I'm curious about this.  Does it really need to parse all architectures?
Oh... is it for perf stat record/report?

Thanks,
Namhyung

> 
> Refactor jevents.py to pre-populate its internal JSON AST cache in parallel
> across all available CPU cores using ProcessPoolExecutor. Define the worker
> process initializer _init_worker at the top-level module scope to guarantee
> flawless pickling and standard event mapping inheritance under spawn
> multiprocessing semantics (avoiding AttributeError crashes when spawn is
> used instead of fork). This accelerates jevents.py execution by over 11x
> (from 3.3s down to ~290ms), fully reclaiming multi-core concurrency during
> the build generation phase.
> 
> Tested-by: James Clark <james.clark@linaro.org>
> Assisted-by: Gemini:gemini-3.1-pro-preview
> Signed-off-by: Ian Rogers <irogers@google.com>
> ---
>  tools/perf/pmu-events/jevents.py | 36 ++++++++++++++++++++++++++++----
>  1 file changed, 32 insertions(+), 4 deletions(-)
> 
> diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py
> index 6adf4b30e1ba..ebacb056524f 100755
> --- a/tools/perf/pmu-events/jevents.py
> +++ b/tools/perf/pmu-events/jevents.py
> @@ -457,8 +457,8 @@ class JsonEvent:
>      return f'{{ { _bcs.offsets[s] } }}, /* {fix_comment(s)} */\n'
>  
>  
> -@lru_cache(maxsize=None)
> -def read_json_events(path: str, topic: str) -> Sequence[JsonEvent]:
> +_json_cache = {}
> +def _read_json_events_impl(path: str, topic: str) -> Sequence[JsonEvent]:
>    """Read json events from the specified file."""
>    try:
>      events = json.load(open(path), object_hook=JsonEvent)
> @@ -474,12 +474,16 @@ def read_json_events(path: str, topic: str) -> Sequence[JsonEvent]:
>    if updates:
>      for event in events:
>        if event.metric_name in updates:
> -        # print(f'Updated {event.metric_name} from\n"{event.metric_expr}"\n'
> -        #       f'to\n"{updates[event.metric_name]}"')
>          event.metric_expr = updates[event.metric_name]
>  
>    return events
>  
> +def read_json_events(path: str, topic: str) -> Sequence[JsonEvent]:
> +  key = (path, topic)
> +  if key not in _json_cache:
> +    _json_cache[key] = _read_json_events_impl(path, topic)
> +  return _json_cache[key]
> +
>  def preprocess_arch_std_files(archpath: str) -> None:
>    """Read in all architecture standard events."""
>    global _arch_std_events
> @@ -1381,6 +1385,14 @@ const char *describe_metricgroup(const char *group)
>  }
>  """)
>  
> +def _parallel_read_json_events(task: Tuple[str, str]) -> Tuple[str, str, Sequence[JsonEvent]]:
> +  path, topic = task
> +  return path, topic, _read_json_events_impl(path, topic)
> +
> +def _init_worker(std_events: dict) -> None:
> +  global _arch_std_events
> +  _arch_std_events = std_events
> +
>  def main() -> None:
>    global _args
>  
> @@ -1459,9 +1471,25 @@ struct pmu_table_entry {
>      raise IOError(f'Missing architecture directory \'{_args.arch}\'')
>  
>    archs.sort()
> +  import concurrent.futures
> +  tasks = []
> +  def collect_json(parents: Sequence[str], item: os.DirEntry) -> None:
> +    if len(parents) == 0:
> +      return
> +    if item.is_file() and item.name.endswith('.json') and not item.name.endswith('metricgroups.json'):
> +      tasks.append((item.path, get_topic(item.name)))
> +
>    for arch in archs:
>      arch_path = f'{_args.starting_dir}/{arch}'
>      preprocess_arch_std_files(arch_path)
> +    ftw(arch_path, [], collect_json)
> +
> +  with concurrent.futures.ProcessPoolExecutor(initializer=_init_worker, initargs=(_arch_std_events,)) as executor:
> +    for path, topic, events in executor.map(_parallel_read_json_events, tasks):
> +      _json_cache[(path, topic)] = events
> +
> +  for arch in archs:
> +    arch_path = f'{_args.starting_dir}/{arch}'
>      ftw(arch_path, [], preprocess_one_file)
>  
>    _bcs.compute()
> -- 
> 2.54.0.563.g4f69b47b94-goog
>