From mboxrd@z Thu Jan 1 00:00:00 1970 From: Jasvinder Singh Subject: [PATCH v4] ip_pipeline: add script file for pipeline to core mappings Date: Fri, 10 Jun 2016 16:55:05 +0100 Message-ID: <1465574105-148429-1-git-send-email-jasvinder.singh@intel.com> References: <1464685085-95410-1-git-send-email-jasvinder.singh@intel.com> Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: quoted-printable Cc: cristian.dumitrescu@intel.com, Guruprasad Mukundarao To: dev@dpdk.org Return-path: Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id D249D2935 for ; Fri, 10 Jun 2016 17:47:55 +0200 (CEST) In-Reply-To: <1464685085-95410-1-git-send-email-jasvinder.singh@intel.com> List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Guruprasad Mukundarao This script parses the application configuration file and detects all the pipelines specified therein, and then, it generates all the possible mapp= ings of those pipelines on the specified CPU core-list. As a result, each of the possible pipeline-to-core mappings is saved as separate output configuration file. For example- if input file is edge_router_downstream.cfg with 3 pipeline (excluding pipeline 0) and core-list is =E2=80=9C1, 2=E2=80=9D, following combinations will be gener= ated- Pipeline 1 Pipeline 2 Pipeline 3 Core =3D 1 Core =3D 1 Core =3D 2 Core =3D 1 Core =3D 2 Core =3D 1 Core =3D 2 Core =3D 1 Core =3D 1 Core =3D 2 Core =3D 2 Core =3D 1 Core =3D 2 Core =3D 1 Core =3D 2 Core =3D 1 Core =3D 2 Core =3D 2 Core =3D C1 Core =3D C1H Core =3D C2 Core =3D C1 Core =3D C2 Core =3D C1H Core =3D C2 Core =3D C1 Core =3D C1H This script will help users to analyse the performance of application by evaluating all the generated configuration files with different pipelines-to-core mappings and obtaining the application configuration fi= le with best performance. To run the script, issue the following command; ./pipeline-to-core-mapping.py -i -pc "sc" Some optional arguments are as follows: -h, --help show this help message and exit -ht {ON,OFF}, --hyper-thread {ON,OFF} enable/disable hyper threading. default is ON -nO, --no-output-file disable output config file generation. Output fil= e generation is enabled by default Signed-off-by: Guruprasad Mukundarao Acked-by: Cristian Dumitrescu --- v4 - conform to PEP8 style conventions - update to make it work with Python2 - remove the unnecessary unicode characters - improve comments, variable names=20 v3 - update the commit message - create folder to save generated output configuration files - fix the duplicate files creation with same pipeline-to-core mappings - fix checkpatch errors - the generated files will have name in the following format- =20 "__s_c_h.cfg" v2 - correct the copyright header .../ip_pipeline/config/pipeline-to-core-mapping.py | 936 +++++++++++++++= ++++++ 1 file changed, 936 insertions(+) create mode 100755 examples/ip_pipeline/config/pipeline-to-core-mapping.= py diff --git a/examples/ip_pipeline/config/pipeline-to-core-mapping.py b/ex= amples/ip_pipeline/config/pipeline-to-core-mapping.py new file mode 100755 index 0000000..9eb97ff --- /dev/null +++ b/examples/ip_pipeline/config/pipeline-to-core-mapping.py @@ -0,0 +1,936 @@ +#! /usr/bin/python2 + +# BSD LICENSE +# +# Copyright(c) 2016 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyrigh= t +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FO= R +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL= , +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE= , +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON AN= Y +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE US= E +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# This script maps the set of pipelines identified (MASTER pipelines are +# ignored) from the input configuration file to the set of cores +# provided as input argument and creates configuration files for each of +# the mapping combinations. +# + +from __future__ import print_function +import sys +import errno +import os +import re +import array +import itertools +import re +import argparse +from collections import namedtuple + +# default values +enable_stage0_traceout =3D 1 +enable_stage1_traceout =3D 1 +enable_stage2_traceout =3D 1 + +enable_stage1_fileout =3D 1 +enable_stage2_fileout =3D 1 + +Constants =3D namedtuple('Constants', ['MAX_CORES', 'MAX_PIPELINES']) +constants =3D Constants(16, 64) + +# pattern for physical core +pattern_phycore =3D '^(s|S)\d(c|C)[1-9][0-9]*$' +reg_phycore =3D re.compile(pattern_phycore) + + +def popcount(mask): + return bin(mask).count("1") + + +def len2mask(length): + if (length =3D=3D 0): + return 0 + + if (length > 64): + sys.exit('error: len2mask - lenght %i > 64. exiting' % length) + + return int('1' * length, 2) + + +def bitstring_write(n, n_bits): + tmpstr =3D "" + if (n_bits > 64): + return + + i =3D n_bits - 1 + while (i >=3D 0): + cond =3D (n & (1 << i)) + if (cond): + print('1', end=3D'') + tmpstr +=3D '1' + else: + print('0', end=3D'') + tmpstr +=3D '0' + i -=3D 1 + return tmpstr + + +class Cores0: + + def __init__(self): + self.n_pipelines =3D 0 + + +class Cores1: + + def __init__(self): + self.pipelines =3D 0 + self.n_pipelines =3D 0 + + +class Cores2: + + def __init__(self): + self.pipelines =3D 0 + self.n_pipelines =3D 0 + self.counter =3D 0 + self.counter_max =3D 0 + self.bitpos =3D array.array( + "L", itertools.repeat(0, constants.MAX_PIPELINES)) + + +class Context0: + + def __init__(self): + self.cores =3D [Cores0() for i in range(0, constants.MAX_CORES)] + self.n_cores =3D 0 + self.n_pipelines =3D 0 + self.n_pipelines0 =3D 0 + self.pos =3D 0 + self.file_comment =3D "" + self.ctx1 =3D None + self.ctx2 =3D None + + def stage0_print(self): + print('printing Context0 obj') + print('c0.cores(n_pipelines) =3D [ ', end=3D'') + for cores_count in range(0, constants.MAX_CORES): + print(self.cores[cores_count].n_pipelines, end=3D' ') + print(']') + print('c0.n_cores =3D %d' % self.n_cores) + print('c0.n_pipelines =3D %d' % self.n_pipelines) + print('c0.n_pipelines0 =3D %d' % self.n_pipelines0) + print('c0.pos =3D %d' % self.pos) + print('c0.file_comment =3D %s' % self.file_comment) + if (self.ctx1 is not None): + print('c0.ctx1 =3D ', end=3D'') + print(repr(self.ctx1)) + else: + print('c0.ctx1 =3D None') + + if (self.ctx2 is not None): + print('c0.ctx2 =3D ', end=3D'') + print(repr(self.ctx2)) + else: + print('c0.ctx2 =3D None') + + def stage0_init(self, num_cores, num_pipelines, ctx1, ctx2): + self.n_cores =3D num_cores + self.n_pipelines =3D num_pipelines + self.ctx1 =3D ctx1 + self.ctx2 =3D ctx2 + + def stage0_process(self): + # stage0 init + self.cores[0].n_pipelines =3D self.n_pipelines + self.n_pipelines0 =3D 0 + self.pos =3D 1 + + while True: + # go forward + while True: + if ((self.pos < self.n_cores) and (self.n_pipelines0 > 0= )): + self.cores[self.pos].n_pipelines =3D min( + self.cores[self.pos - 1].n_pipelines, + self.n_pipelines0) + self.n_pipelines0 -=3D self.cores[self.pos].n_pipeli= nes + self.pos +=3D 1 + else: + break + + # check solution + if (self.n_pipelines0 =3D=3D 0): + self.stage0_log() + self.ctx1.stage1_init(self, self.ctx2) # self is object= c0 + self.ctx1.stage1_process() + + # go backward + while True: + if (self.pos =3D=3D 0): + return + + self.pos -=3D 1 + if ((self.cores[self.pos].n_pipelines > 1) and + (self.pos !=3D (self.n_cores - 1))): + break + + self.n_pipelines0 +=3D self.cores[self.pos].n_pipelines + self.cores[self.pos].n_pipelines =3D 0 + + # rearm + self.cores[self.pos].n_pipelines -=3D 1 + self.n_pipelines0 +=3D 1 + self.pos +=3D 1 + + def stage0_log(self): + tmp_file_comment =3D "" + if(enable_stage0_traceout !=3D 1): + return + + print('STAGE0: ', end=3D'') + tmp_file_comment +=3D 'STAGE0: ' + for cores_count in range(0, self.n_cores): + print('C%d =3D %d\t' + % (cores_count, + self.cores[cores_count].n_pipelines), end=3D'') + tmp_file_comment +=3D "C{} =3D {}\t".format( + cores_count, self.cores[cores_count].n_pipelines) + # end for + print('') + self.ctx1.stage0_file_comment =3D tmp_file_comment + self.ctx2.stage0_file_comment =3D tmp_file_comment + + +class Context1: + _fileTrace =3D None + + def __init__(self): + self.cores =3D [Cores1() for i in range(constants.MAX_CORES)] + self.n_cores =3D 0 + self.n_pipelines =3D 0 + self.pos =3D 0 + self.stage0_file_comment =3D "" + self.stage1_file_comment =3D "" + + self.ctx2 =3D None + self.arr_pipelines2cores =3D [] + + def stage1_reset(self): + for i in range(constants.MAX_CORES): + self.cores[i].pipelines =3D 0 + self.cores[i].n_pipelines =3D 0 + + self.n_cores =3D 0 + self.n_pipelines =3D 0 + self.pos =3D 0 + self.ctx2 =3D None + # clear list + del self.arr_pipelines2cores[:] + + def stage1_print(self): + print('printing Context1 obj') + print('ctx1.cores(pipelines,n_pipelines) =3D [ ', end=3D'') + for cores_count in range(0, constants.MAX_CORES): + print('(%d,%d)' % (self.cores[cores_count].pipelines, + self.cores[cores_count].n_pipelines), end= =3D' ') + print(']') + print('ctx1.n_cores =3D %d' % self.n_cores) + print('ctx1.n_pipelines =3D %d' % self.n_pipelines) + print('ctx1.pos =3D %d' % self.pos) + print('ctx1.stage0_file_comment =3D %s' % self.stage0_file_comme= nt) + print('ctx1.stage1_file_comment =3D %s' % self.stage1_file_comme= nt) + if (self.ctx2 is not None): + print('ctx1.ctx2 =3D ', end=3D'') + print(self.ctx2) + else: + print('ctx1.ctx2 =3D None') + + def stage1_init(self, c0, ctx2): + self.stage1_reset() + self.n_cores =3D 0 + while (c0.cores[self.n_cores].n_pipelines > 0): + self.n_cores +=3D 1 + + self.n_pipelines =3D c0.n_pipelines + self.ctx2 =3D ctx2 + + self.arr_pipelines2cores =3D [0] * self.n_pipelines + + i =3D 0 + while (i < self.n_cores): + self.cores[i].n_pipelines =3D c0.cores[i].n_pipelines + i +=3D 1 + + def stage1_process(self): + pipelines_max =3D len2mask(self.n_pipelines) + while True: + pos =3D 0 + overlap =3D 0 + + if (self.cores[self.pos].pipelines =3D=3D pipelines_max): + if (self.pos =3D=3D 0): + return + + self.cores[self.pos].pipelines =3D 0 + self.pos -=3D 1 + continue + + self.cores[self.pos].pipelines +=3D 1 + if (popcount(self.cores[self.pos].pipelines) !=3D + self.cores[self.pos].n_pipelines): + continue + + overlap =3D 0 + pos =3D 0 + while (pos < self.pos): + if ((self.cores[self.pos].pipelines) & + (self.cores[pos].pipelines)): + overlap =3D 1 + break + pos +=3D 1 + + if (overlap): + continue + + if ((self.pos > 0) and + ((self.cores[self.pos].n_pipelines) =3D=3D + (self.cores[self.pos - 1].n_pipelines)) and + ((self.cores[self.pos].pipelines) < + (self.cores[self.pos - 1].pipelines))): + continue + + if (self.pos =3D=3D self.n_cores - 1): + self.stage1_log() + self.ctx2.stage2_init(self) + self.ctx2.stage2_process() + + if (self.pos =3D=3D 0): + return + + self.cores[self.pos].pipelines =3D 0 + self.pos -=3D 1 + continue + + self.pos +=3D 1 + + def stage1_log(self): + tmp_file_comment =3D "" + if(enable_stage1_traceout =3D=3D 1): + print('STAGE1: ', end=3D'') + tmp_file_comment +=3D 'STAGE1: ' + i =3D 0 + while (i < self.n_cores): + print('C%d =3D [' % i, end=3D'') + tmp_file_comment +=3D "C{} =3D [".format(i) + + j =3D self.n_pipelines - 1 + while (j >=3D 0): + cond =3D ((self.cores[i].pipelines) & (1 << j)) + if (cond): + print('1', end=3D'') + tmp_file_comment +=3D '1' + else: + print('0', end=3D'') + tmp_file_comment +=3D '0' + j -=3D 1 + + print(']\t', end=3D'') + tmp_file_comment +=3D ']\t' + i +=3D 1 + + print('\n', end=3D'') + self.stage1_file_comment =3D tmp_file_comment + self.ctx2.stage1_file_comment =3D tmp_file_comment + + # check if file traceing is enabled + if(enable_stage1_fileout !=3D 1): + return + + # spit out the combination to file + self.stage1_process_file() + + def stage1_updateCoresInBuf(self, nPipeline, sCore): + rePipeline =3D self._fileTrace.arr_pipelines[nPipeline] + rePipeline =3D rePipeline.replace("[", "\[").replace("]", "\]") + reCore =3D 'core\s*=3D\s*((\d*)|(((s|S)\d)?(c|C)[1-9][0-9]*)).*\= n' + sSubs =3D 'core =3D ' + sCore + '\n' + + reg_pipeline =3D re.compile(rePipeline) + search_match =3D reg_pipeline.search(self._fileTrace.in_buf) + + if(search_match): + pos =3D search_match.start() + substr1 =3D self._fileTrace.in_buf[:pos] + substr2 =3D self._fileTrace.in_buf[pos:] + substr2 =3D re.sub(reCore, sSubs, substr2, 1) + self._fileTrace.in_buf =3D substr1 + substr2 + + def stage1_process_file(self): + outFileName =3D os.path.join(self._fileTrace.out_path, + self._fileTrace.prefix_outfile) + outFileName +=3D "_{}CoReS".format(self.n_cores) + + i =3D 0 # represents core number + while (i < self.n_cores): + j =3D self.n_pipelines - 1 + pipeline_idx =3D 0 + while(j >=3D 0): + cond =3D ((self.cores[i].pipelines) & (1 << j)) + if (cond): + # update the pipelines array to match the core + # only in case of cond match + self.arr_pipelines2cores[ + pipeline_idx] =3D fileTrace.in_physical_cores[i] + + j -=3D 1 + pipeline_idx +=3D 1 + + i +=3D 1 + + # update the in_buf as per the arr_pipelines2cores + for pipeline_idx in range(len(self.arr_pipelines2cores)): + outFileName +=3D "_{}".format(self.arr_pipelines2cores[pipel= ine_idx]) + self.stage1_updateCoresInBuf( + pipeline_idx, self.arr_pipelines2cores[pipeline_idx]) + + # by now the in_buf is all set to be written to file + outFileName +=3D self._fileTrace.suffix_outfile + outputFile =3D open(outFileName, "w") + + # write out the comments + strTruncated =3D ("", "(Truncated)")[self._fileTrace.ncores_trun= cated] + outputFile.write( + "; =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D Pipeline-to= -Core Mapping =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D\n" + "; Generated from file {}\n" + "; Input pipelines =3D {}\n" + "; Input cores =3D {}\n" + "; N_PIPELINES =3D {} N_CORES =3D {} {} hyper_thread =3D {}\= n" + .format( + self._fileTrace.in_file_namepath, + fileTrace.arr_pipelines, + fileTrace.in_physical_cores, + self._fileTrace.n_pipelines, + self._fileTrace.n_cores, + strTruncated, + self._fileTrace.hyper_thread)) + + outputFile.write( + "; {stg0cmt}\n" + "; {stg1cmt}\n" + "; =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D\n" + "; \n" + .format( + stg0cmt=3Dself.stage0_file_comment, + stg1cmt=3Dself.stage1_file_comment)) + + # write buffer contents + outputFile.write(self._fileTrace.in_buf) + outputFile.flush() + outputFile.close() + + +class Context2: + _fileTrace =3D None + + def __init__(self): + self.cores =3D [Cores2() for i in range(constants.MAX_CORES)] + self.n_cores =3D 0 + self.n_pipelines =3D 0 + self.pos =3D 0 + self.stage0_file_comment =3D "" + self.stage1_file_comment =3D "" + self.stage2_file_comment =3D "" + + # each array entry is a pipeline mapped to core stored as string + # pipeline ranging from 1 to n, however stored in zero based arr= ay + self.arr2_pipelines2cores =3D [] + + def stage2_print(self): + print('printing Context2 obj') + print('ctx2.cores(pipelines, n_pipelines, counter, counter_max) = =3D') + for cores_count in range(0, constants.MAX_CORES): + print('core[%d] =3D (%d,%d,%d,%d)' % ( + cores_count, + self.cores[cores_count].pipelines, + self.cores[cores_count].n_pipelines, + self.cores[cores_count].counter, + self.cores[cores_count].counter_max)) + + print('ctx2.n_cores =3D %d' % self.n_cores, end=3D'') + print('ctx2.n_pipelines =3D %d' % self.n_pipelines, end=3D''= ) + print('ctx2.pos =3D %d' % self.pos) + print('ctx2.stage0_file_comment =3D %s' % + self.self.stage0_file_comment) + print('ctx2.stage1_file_comment =3D %s' % + self.self.stage1_file_comment) + print('ctx2.stage2_file_comment =3D %s' % + self.self.stage2_file_comment) + + def stage2_reset(self): + for i in range(0, constants.MAX_CORES): + self.cores[i].pipelines =3D 0 + self.cores[i].n_pipelines =3D 0 + self.cores[i].counter =3D 0 + self.cores[i].counter_max =3D 0 + + for idx in range(0, constants.MAX_PIPELINES): + self.cores[i].bitpos[idx] =3D 0 + + self.n_cores =3D 0 + self.n_pipelines =3D 0 + self.pos =3D 0 + # clear list + del self.arr2_pipelines2cores[:] + + def bitpos_load(self, coreidx): + i =3D j =3D 0 + while (i < self.n_pipelines): + if ((self.cores[coreidx].pipelines) & + (1 << i)): + self.cores[coreidx].bitpos[j] =3D i + j +=3D 1 + i +=3D 1 + self.cores[coreidx].n_pipelines =3D j + + def bitpos_apply(self, in_buf, pos, n_pos): + out =3D 0 + for i in range(0, n_pos): + out |=3D (in_buf & (1 << i)) << (pos[i] - i) + + return out + + def stage2_init(self, ctx1): + self.stage2_reset() + self.n_cores =3D ctx1.n_cores + self.n_pipelines =3D ctx1.n_pipelines + + self.arr2_pipelines2cores =3D [''] * self.n_pipelines + + core_idx =3D 0 + while (core_idx < self.n_cores): + self.cores[core_idx].pipelines =3D ctx1.cores[core_idx].pipe= lines + + self.bitpos_load(core_idx) + core_idx +=3D 1 + + def stage2_log(self): + tmp_file_comment =3D "" + if(enable_stage2_traceout =3D=3D 1): + print('STAGE2: ', end=3D'') + tmp_file_comment +=3D 'STAGE2: ' + + for i in range(0, self.n_cores): + mask =3D len2mask(self.cores[i].n_pipelines) + pipelines_ht0 =3D self.bitpos_apply( + (~self.cores[i].counter) & mask, + self.cores[i].bitpos, + self.cores[i].n_pipelines) + + pipelines_ht1 =3D self.bitpos_apply( + self.cores[i].counter, + self.cores[i].bitpos, + self.cores[i].n_pipelines) + + print('C%dHT0 =3D [' % i, end=3D'') + tmp_file_comment +=3D "C{}HT0 =3D [".format(i) + tmp_file_comment +=3D bitstring_write( + pipelines_ht0, self.n_pipelines) + + print(']\tC%dHT1 =3D [' % i, end=3D'') + tmp_file_comment +=3D "]\tC{}HT1 =3D [".format(i) + tmp_file_comment +=3D bitstring_write( + pipelines_ht1, self.n_pipelines) + print(']\t', end=3D'') + tmp_file_comment +=3D ']\t' + + print('') + self.stage2_file_comment =3D tmp_file_comment + + # check if file traceing is enabled + if(enable_stage2_fileout !=3D 1): + return + # spit out the combination to file + self.stage2_process_file() + + def stage2_updateCoresInBuf(self, nPipeline, sCore): + rePipeline =3D self._fileTrace.arr_pipelines[nPipeline] + rePipeline =3D rePipeline.replace("[", "\[").replace("]", "\]") + reCore =3D 'core\s*=3D\s*((\d*)|(((s|S)\d)?(c|C)[1-9][0-9]*)).*\= n' + sSubs =3D 'core =3D ' + sCore + '\n' + + reg_pipeline =3D re.compile(rePipeline) + search_match =3D reg_pipeline.search(self._fileTrace.in_buf) + + if(search_match): + pos =3D search_match.start() + substr1 =3D self._fileTrace.in_buf[:pos] + substr2 =3D self._fileTrace.in_buf[pos:] + substr2 =3D re.sub(reCore, sSubs, substr2, 1) + self._fileTrace.in_buf =3D substr1 + substr2 + + def pipelines2cores(self, n, n_bits, nCore, bHT): + if (n_bits > 64): + return + + i =3D n_bits - 1 + pipeline_idx =3D 0 + while (i >=3D 0): + cond =3D (n & (1 << i)) + if (cond): + # update the pipelines array to match the core + # only in case of cond match + # PIPELINE0 and core 0 are reserved + if(bHT): + tmpCore =3D fileTrace.in_physical_cores[nCore] + 'h' + self.arr2_pipelines2cores[pipeline_idx] =3D tmpCore + else: + self.arr2_pipelines2cores[pipeline_idx] =3D \ + fileTrace.in_physical_cores[nCore] + + i -=3D 1 + pipeline_idx +=3D 1 + + def stage2_process_file(self): + outFileName =3D os.path.join(self._fileTrace.out_path, + self._fileTrace.prefix_outfile) + outFileName +=3D "_{}CoReS".format(self.n_cores) + + for i in range(0, self.n_cores): + mask =3D len2mask(self.cores[i].n_pipelines) + pipelines_ht0 =3D self.bitpos_apply((~self.cores[i].counter)= & mask, + self.cores[i].bitpos, + self.cores[i].n_pipelines) + + pipelines_ht1 =3D self.bitpos_apply(self.cores[i].counter, + self.cores[i].bitpos, + self.cores[i].n_pipelines) + + # update pipelines to core mapping + self.pipelines2cores(pipelines_ht0, self.n_pipelines, i, Fal= se) + self.pipelines2cores(pipelines_ht1, self.n_pipelines, i, Tru= e) + + # update the in_buf as per the arr_pipelines2cores + for pipeline_idx in range(len(self.arr2_pipelines2cores)): + outFileName +=3D "_{}".format( + self.arr2_pipelines2cores[pipeline_idx]) + self.stage2_updateCoresInBuf( + pipeline_idx, self.arr2_pipelines2cores[pipeline_idx]) + + # by now the in_buf is all set to be written to file + outFileName +=3D self._fileTrace.suffix_outfile + outputFile =3D open(outFileName, "w") + + # write the file comments + strTruncated =3D ("", "(Truncated)")[self._fileTrace.ncores_trun= cated] + outputFile.write( + "; =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D Pipeline-to= -Core Mapping =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D\n" + "; Generated from file {}\n" + "; Input pipelines =3D {}\n" + "; Input cores =3D {}\n" + "; N_PIPELINES =3D {} N_CORES =3D {} {} hyper_thread =3D {}= \n" + .format( + self._fileTrace.in_file_namepath, + fileTrace.arr_pipelines, + fileTrace.in_physical_cores, + self._fileTrace.n_pipelines, + self._fileTrace.n_cores, + strTruncated, + self._fileTrace.hyper_thread)) + + outputFile.write( + "; {stg0cmt}\n" + "; {stg1cmt}\n" + "; {stg2cmt}\n" + "; =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D\n" + "; \n" + .format( + stg0cmt=3Dself.stage0_file_comment, + stg1cmt=3Dself.stage1_file_comment, + stg2cmt=3Dself.stage2_file_comment)) + + # write the buffer contents + outputFile.write(self._fileTrace.in_buf) + outputFile.flush() + outputFile.close() + + def stage2_process(self): + i =3D 0 + while(i < self.n_cores): + self.cores[i].counter_max =3D len2mask( + self.cores[i].n_pipelines - 1) + i +=3D 1 + + self.pos =3D self.n_cores - 1 + while True: + if (self.pos =3D=3D self.n_cores - 1): + self.stage2_log() + + if (self.cores[self.pos].counter =3D=3D + self.cores[self.pos].counter_max): + if (self.pos =3D=3D 0): + return + + self.cores[self.pos].counter =3D 0 + self.pos -=3D 1 + continue + + self.cores[self.pos].counter +=3D 1 + if(self.pos < self.n_cores - 1): + self.pos +=3D 1 + + +class FileTrace: + + def __init__(self, filenamepath): + self.in_file_namepath =3D os.path.abspath(filenamepath) + self.in_filename =3D os.path.basename(self.in_file_namepath) + self.in_path =3D os.path.dirname(self.in_file_namepath) + + filenamesplit =3D self.in_filename.split('.') + self.prefix_outfile =3D filenamesplit[0] + self.suffix_outfile =3D ".cfg" + + # output folder: in the same folder as input file + # create new folder in the name of input file + self.out_path =3D os.path.join( + os.path.abspath(os.path.dirname(__file__)), + self.prefix_outfile) + + try: + os.makedirs(self.out_path) + except OSError as excep: + if excep.errno =3D=3D errno.EEXIST and os.path.isdir(self.ou= t_path): + pass + else: + raise + + self.in_buf =3D None + self.arr_pipelines =3D [] # holds the positions of search + + self.max_cores =3D 15 + self.max_pipelines =3D 15 + + self.in_physical_cores =3D None + self.hyper_thread =3D None + + # save the num of pipelines determined from input file + self.n_pipelines =3D 0 + # save the num of cores input (or the truncated value) + self.n_cores =3D 0 + self.ncores_truncated =3D False + + def print_TraceFile(self): + print("self.in_file_namepath =3D ", self.in_file_namepath) + print("self.in_filename =3D ", self.in_filename) + print("self.in_path =3D ", self.in_path) + print("self.out_path =3D ", self.out_path) + print("self.prefix_outfile =3D ", self.prefix_outfile) + print("self.suffix_outfile =3D ", self.suffix_outfile) + print("self.in_buf =3D ", self.in_buf) + print("self.arr_pipelines =3D", self.arr_pipelines) + print("self.in_physical_cores", self.in_physical_cores) + print("self.hyper_thread", self.hyper_thread) + + +def process(n_cores, n_pipelines, fileTrace): + '''proces and map pipelines, cores.''' + if (n_cores =3D=3D 0): + sys.exit('N_CORES is 0, exiting') + + if (n_pipelines =3D=3D 0): + sys.exit('N_PIPELINES is 0, exiting') + + if (n_cores > n_pipelines): + print('\nToo many cores, truncating N_CORES to N_PIPELINES') + n_cores =3D n_pipelines + fileTrace.ncores_truncated =3D True + + fileTrace.n_pipelines =3D n_pipelines + fileTrace.n_cores =3D n_cores + + strTruncated =3D ("", "(Truncated)")[fileTrace.ncores_truncated] + print("N_PIPELINES =3D {}, N_CORES =3D {} {}" + .format(n_pipelines, n_cores, strTruncated)) + print("-------------------------------------------------------------= --") + + ctx0_inst =3D Context0() + ctx1_inst =3D Context1() + ctx2_inst =3D Context2() + + # initialize the class variables + ctx1_inst._fileTrace =3D fileTrace + ctx2_inst._fileTrace =3D fileTrace + + ctx0_inst.stage0_init(n_cores, n_pipelines, ctx1_inst, ctx2_inst) + ctx0_inst.stage0_process() + + +def validate_core(core): + match =3D reg_phycore.match(core) + if(match): + return True + else: + return False + + +def validate_phycores(phy_cores): + '''validate physical cores, check if unique.''' + # eat up whitespaces + phy_cores =3D phy_cores.strip().split(',') + + # check if the core list is unique + if(len(phy_cores) !=3D len(set(phy_cores))): + print('list of physical cores has duplicates') + return None + + for core in phy_cores: + if not validate_core(core): + print('invalid physical core specified.') + return None + return phy_cores + + +def scanconfigfile(fileTrace): + '''scan input file for pipelines, validate then process.''' + # open file + filetoscan =3D open(fileTrace.in_file_namepath, 'r') + fileTrace.in_buf =3D filetoscan.read() + + # reset iterator on open file + filetoscan.seek(0) + + # scan input file for pipelines + # master pipelines to be ignored + pattern_pipeline =3D r'\[PIPELINE\d*\]' + pattern_mastertype =3D r'type\s*=3D\s*MASTER' + + pending_pipeline =3D False + for line in filetoscan: + match_pipeline =3D re.search(pattern_pipeline, line) + match_type =3D re.search('type\s*=3D', line) + match_mastertype =3D re.search(pattern_mastertype, line) + + if(match_pipeline): + sPipeline =3D line[match_pipeline.start():match_pipeline.end= ()] + pending_pipeline =3D True + elif(match_type): + # found a type definition... + if(match_mastertype is None): + # and this is not a master pipeline... + if(pending_pipeline): + # add it to the list of pipelines to be mapped + fileTrace.arr_pipelines.append(sPipeline) + pending_pipeline =3D False + else: + # and this is a master pipeline... + # ignore the current and move on to next + sPipeline =3D "" + pending_pipeline =3D False + filetoscan.close() + + # validate if pipelines are unique + if(len(fileTrace.arr_pipelines) !=3D len(set(fileTrace.arr_pipelines= ))): + sys.exit('Error: duplicate pipelines in input file') + + num_pipelines =3D len(fileTrace.arr_pipelines) + num_cores =3D len(fileTrace.in_physical_cores) + + print("-------------------Pipeline-to-core mapping------------------= --") + print("Input pipelines =3D {}\nInput cores =3D {}" + .format(fileTrace.arr_pipelines, fileTrace.in_physical_cores)) + + # input configuration file validations goes here + if (num_cores > fileTrace.max_cores): + sys.exit('Error: number of cores specified > max_cores (%d)' % + fileTrace.max_cores) + + if (num_pipelines > fileTrace.max_pipelines): + sys.exit('Error: number of pipelines in input \ + cfg file > max_pipelines (%d)' % fileTrace.max_pipelines= ) + + # call process to generate pipeline-to-core mapping, trace and log + process(num_cores, num_pipelines, fileTrace) + + +if __name__ =3D=3D "__main__": + parser =3D argparse.ArgumentParser(description=3D'mappipelines') + + reqNamedGrp =3D parser.add_argument_group('required named args') + reqNamedGrp.add_argument( + '-i', + '--input-file', + type=3Dargparse.FileType('r'), + help=3D'Input config file', + required=3DTrue) + + reqNamedGrp.add_argument( + '-pc', + '--physical-cores', + type=3Dvalidate_phycores, + help=3D'''Enter available CPU cores in + format:\",,...\" + where each core format: \"sc\" + where SOCKETID=3D{0..9}, COREID=3D{1-99}''', + required=3DTrue) + + # add optional arguments + parser.add_argument( + '-ht', + '--hyper-thread', + help=3D'enable/disable hyper threading. default is ON', + default=3D'ON', + choices=3D['ON', 'OFF']) + + parser.add_argument( + '-nO', + '--no-output-file', + help=3D'''disable output config file generation. + Output file generation is enabled by default''', + action=3D"store_true") + + args =3D parser.parse_args() + + if(args.physical_cores is None): + parser.error("invalid physical_cores specified") + + # create object of FileTrace and initialise + fileTrace =3D FileTrace(args.input_file.name) + fileTrace.in_physical_cores =3D args.physical_cores + fileTrace.hyper_thread =3D args.hyper_thread + + if(fileTrace.hyper_thread =3D=3D 'OFF'): + print("!!!!disabling stage2 HT!!!!") + enable_stage2_traceout =3D 0 + enable_stage2_fileout =3D 0 + elif(fileTrace.hyper_thread =3D=3D 'ON'): + print("!!!!HT enabled. disabling stage1 file generation.!!!!") + enable_stage1_fileout =3D 0 + + if(args.no_output_file is True): + print("!!!!disabling stage1 and stage2 fileout!!!!") + enable_stage1_fileout =3D 0 + enable_stage2_fileout =3D 0 + + scanconfigfile(fileTrace) --=20 2.5.5