Openembedded Core Discussions
 help / color / mirror / Atom feed
* [PATCH V4] icu: Add knobs to generate a subset of ICU data
@ 2020-04-04 17:20 Khem Raj
  2020-04-04 17:43 ` Khem Raj
  0 siblings, 1 reply; 3+ messages in thread
From: Khem Raj @ 2020-04-04 17:20 UTC (permalink / raw)
  To: openembedded-core; +Cc: Khem Raj, Wouter Meek

Recent versions of ICU (64+) provides a tool for configuring ICU
locale data file with finer granularity [1]

Default generated size for libicudata.so.66.1 is ~27M, which is quite
large for embedded systems and all of them may not even need all locale
data.

This patch calls the icudata buildtool during configure on the icudata
and utilizes a filter called 'filter.json` ( empty by default) to
create the data, default behavior should remain same but someone can
add a filter.json in own layer to configure this data, e.g.

{
  "localeFilter": {
    "filterType": "language",
    "whitelist": [
      "en",
      "de",
      "it"
    ]
  }
}

would only generate the locale data for english/german/italian

This would reduce the size of libicudata.so.66.1 to 12M

Ensure that icudata is generated using host-tools so it can deal with
endianness correctly, when host and target systems have different
endianness

install the icudtata file back into in/ folder so that main build
can now pickup this data file instead of regenerating it and wiping
out the filter changes that are expected to take effect

[1] https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md

Signed-off-by: Khem Raj <raj.khem@gmail.com>
Suggested-by: Wouter Meek <w.meek@metrological.com>
---
v2: Fix build for BE systems
v3: Unset AR and RANLIB too
v4: Install fresh icudata back into data/in folder

 meta/recipes-support/icu/icu/filter.json |  2 ++
 meta/recipes-support/icu/icu_66.1.bb     | 26 +++++++++++++++++++++---
 2 files changed, 25 insertions(+), 3 deletions(-)
 create mode 100644 meta/recipes-support/icu/icu/filter.json

diff --git a/meta/recipes-support/icu/icu/filter.json b/meta/recipes-support/icu/icu/filter.json
new file mode 100644
index 0000000000..2c63c08510
--- /dev/null
+++ b/meta/recipes-support/icu/icu/filter.json
@@ -0,0 +1,2 @@
+{
+}
diff --git a/meta/recipes-support/icu/icu_66.1.bb b/meta/recipes-support/icu/icu_66.1.bb
index 5018464c14..736bec4605 100644
--- a/meta/recipes-support/icu/icu_66.1.bb
+++ b/meta/recipes-support/icu/icu_66.1.bb
@@ -18,7 +18,10 @@ ARM_INSTRUCTION_SET_armv4 = "arm"
 ARM_INSTRUCTION_SET_armv5 = "arm"
 
 BASE_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-src.tgz"
-SRC_URI = "${BASE_SRC_URI} \
+DATA_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-data.zip"
+SRC_URI = "${BASE_SRC_URI};name=code \
+           ${DATA_SRC_URI};name=data \
+           file://filter.json \
            file://icu-pkgdata-large-cmd.patch \
            file://fix-install-manx.patch \
            file://0001-Fix-big-endian-build.patch \
@@ -28,8 +31,25 @@ SRC_URI = "${BASE_SRC_URI} \
 SRC_URI_append_class-target = "\
            file://0001-Disable-LDFLAGSICUDT-for-Linux.patch \
           "
-SRC_URI[md5sum] = "b33dc6766711517c98d318447e5110f8"
-SRC_URI[sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e"
+SRC_URI[code.sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e"
+SRC_URI[data.sha256sum] = "8be647f738891d2beb79d48f99077b3499948430eae6f1be112553b15ab0243e"
 
 UPSTREAM_CHECK_REGEX = "icu4c-(?P<pver>\d+(_\d+)+)-src"
 UPSTREAM_CHECK_URI = "https://github.com/unicode-org/icu/releases"
+
+do_make_icudata_class-target () {
+    rm -rf ${S}/data
+    cp -a ${WORKDIR}/data ${S}
+    cd ${S}
+    unset AR CC CPP CXX RANLIB CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
+    ICU_DATA_FILTER_FILE=${WORKDIR}/filter.json \
+    ./runConfigureICU Linux --with-data-packaging=archive
+    oe_runmake ${PARALLEL_MAKE}
+    install -Dm644 ${S}/data/out/icudt${ICU_MAJOR_VER}l.dat ${S}/data/in/icudt${ICU_MAJOR_VER}l.dat
+}
+
+do_make_icudata() {
+    :
+}
+
+addtask make_icudata before do_configure after do_patch
-- 
2.26.0


^ permalink raw reply related	[flat|nested] 3+ messages in thread

* Re: [PATCH V4] icu: Add knobs to generate a subset of ICU data
  2020-04-04 17:20 [PATCH V4] icu: Add knobs to generate a subset of ICU data Khem Raj
@ 2020-04-04 17:43 ` Khem Raj
  2020-04-04 18:53   ` Khem Raj
  0 siblings, 1 reply; 3+ messages in thread
From: Khem Raj @ 2020-04-04 17:43 UTC (permalink / raw)
  To: Patches and discussions about the oe-core layer; +Cc: Wouter Meek

ignore this one

On Sat, Apr 4, 2020 at 10:19 AM Khem Raj <raj.khem@gmail.com> wrote:
>
> Recent versions of ICU (64+) provides a tool for configuring ICU
> locale data file with finer granularity [1]
>
> Default generated size for libicudata.so.66.1 is ~27M, which is quite
> large for embedded systems and all of them may not even need all locale
> data.
>
> This patch calls the icudata buildtool during configure on the icudata
> and utilizes a filter called 'filter.json` ( empty by default) to
> create the data, default behavior should remain same but someone can
> add a filter.json in own layer to configure this data, e.g.
>
> {
>   "localeFilter": {
>     "filterType": "language",
>     "whitelist": [
>       "en",
>       "de",
>       "it"
>     ]
>   }
> }
>
> would only generate the locale data for english/german/italian
>
> This would reduce the size of libicudata.so.66.1 to 12M
>
> Ensure that icudata is generated using host-tools so it can deal with
> endianness correctly, when host and target systems have different
> endianness
>
> install the icudtata file back into in/ folder so that main build
> can now pickup this data file instead of regenerating it and wiping
> out the filter changes that are expected to take effect
>
> [1] https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md
>
> Signed-off-by: Khem Raj <raj.khem@gmail.com>
> Suggested-by: Wouter Meek <w.meek@metrological.com>
> ---
> v2: Fix build for BE systems
> v3: Unset AR and RANLIB too
> v4: Install fresh icudata back into data/in folder
>
>  meta/recipes-support/icu/icu/filter.json |  2 ++
>  meta/recipes-support/icu/icu_66.1.bb     | 26 +++++++++++++++++++++---
>  2 files changed, 25 insertions(+), 3 deletions(-)
>  create mode 100644 meta/recipes-support/icu/icu/filter.json
>
> diff --git a/meta/recipes-support/icu/icu/filter.json b/meta/recipes-support/icu/icu/filter.json
> new file mode 100644
> index 0000000000..2c63c08510
> --- /dev/null
> +++ b/meta/recipes-support/icu/icu/filter.json
> @@ -0,0 +1,2 @@
> +{
> +}
> diff --git a/meta/recipes-support/icu/icu_66.1.bb b/meta/recipes-support/icu/icu_66.1.bb
> index 5018464c14..736bec4605 100644
> --- a/meta/recipes-support/icu/icu_66.1.bb
> +++ b/meta/recipes-support/icu/icu_66.1.bb
> @@ -18,7 +18,10 @@ ARM_INSTRUCTION_SET_armv4 = "arm"
>  ARM_INSTRUCTION_SET_armv5 = "arm"
>
>  BASE_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-src.tgz"
> -SRC_URI = "${BASE_SRC_URI} \
> +DATA_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-data.zip"
> +SRC_URI = "${BASE_SRC_URI};name=code \
> +           ${DATA_SRC_URI};name=data \
> +           file://filter.json \
>             file://icu-pkgdata-large-cmd.patch \
>             file://fix-install-manx.patch \
>             file://0001-Fix-big-endian-build.patch \
> @@ -28,8 +31,25 @@ SRC_URI = "${BASE_SRC_URI} \
>  SRC_URI_append_class-target = "\
>             file://0001-Disable-LDFLAGSICUDT-for-Linux.patch \
>            "
> -SRC_URI[md5sum] = "b33dc6766711517c98d318447e5110f8"
> -SRC_URI[sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e"
> +SRC_URI[code.sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e"
> +SRC_URI[data.sha256sum] = "8be647f738891d2beb79d48f99077b3499948430eae6f1be112553b15ab0243e"
>
>  UPSTREAM_CHECK_REGEX = "icu4c-(?P<pver>\d+(_\d+)+)-src"
>  UPSTREAM_CHECK_URI = "https://github.com/unicode-org/icu/releases"
> +
> +do_make_icudata_class-target () {
> +    rm -rf ${S}/data
> +    cp -a ${WORKDIR}/data ${S}
> +    cd ${S}
> +    unset AR CC CPP CXX RANLIB CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
> +    ICU_DATA_FILTER_FILE=${WORKDIR}/filter.json \
> +    ./runConfigureICU Linux --with-data-packaging=archive
> +    oe_runmake ${PARALLEL_MAKE}
> +    install -Dm644 ${S}/data/out/icudt${ICU_MAJOR_VER}l.dat ${S}/data/in/icudt${ICU_MAJOR_VER}l.dat
> +}
> +
> +do_make_icudata() {
> +    :
> +}
> +
> +addtask make_icudata before do_configure after do_patch
> --
> 2.26.0
>

^ permalink raw reply	[flat|nested] 3+ messages in thread

* Re: [PATCH V4] icu: Add knobs to generate a subset of ICU data
  2020-04-04 17:43 ` Khem Raj
@ 2020-04-04 18:53   ` Khem Raj
  0 siblings, 0 replies; 3+ messages in thread
From: Khem Raj @ 2020-04-04 18:53 UTC (permalink / raw)
  To: Patches and discussions about the oe-core layer; +Cc: Wouter Meek

On Sat, Apr 4, 2020 at 10:43 AM Khem Raj <raj.khem@gmail.com> wrote:
>
> ignore this one
>

This now should be ok to take in, I was seeing a issue on musl which I
have sent a separate patch on top


> On Sat, Apr 4, 2020 at 10:19 AM Khem Raj <raj.khem@gmail.com> wrote:
> >
> > Recent versions of ICU (64+) provides a tool for configuring ICU
> > locale data file with finer granularity [1]
> >
> > Default generated size for libicudata.so.66.1 is ~27M, which is quite
> > large for embedded systems and all of them may not even need all locale
> > data.
> >
> > This patch calls the icudata buildtool during configure on the icudata
> > and utilizes a filter called 'filter.json` ( empty by default) to
> > create the data, default behavior should remain same but someone can
> > add a filter.json in own layer to configure this data, e.g.
> >
> > {
> >   "localeFilter": {
> >     "filterType": "language",
> >     "whitelist": [
> >       "en",
> >       "de",
> >       "it"
> >     ]
> >   }
> > }
> >
> > would only generate the locale data for english/german/italian
> >
> > This would reduce the size of libicudata.so.66.1 to 12M
> >
> > Ensure that icudata is generated using host-tools so it can deal with
> > endianness correctly, when host and target systems have different
> > endianness
> >
> > install the icudtata file back into in/ folder so that main build
> > can now pickup this data file instead of regenerating it and wiping
> > out the filter changes that are expected to take effect
> >
> > [1] https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md
> >
> > Signed-off-by: Khem Raj <raj.khem@gmail.com>
> > Suggested-by: Wouter Meek <w.meek@metrological.com>
> > ---
> > v2: Fix build for BE systems
> > v3: Unset AR and RANLIB too
> > v4: Install fresh icudata back into data/in folder
> >
> >  meta/recipes-support/icu/icu/filter.json |  2 ++
> >  meta/recipes-support/icu/icu_66.1.bb     | 26 +++++++++++++++++++++---
> >  2 files changed, 25 insertions(+), 3 deletions(-)
> >  create mode 100644 meta/recipes-support/icu/icu/filter.json
> >
> > diff --git a/meta/recipes-support/icu/icu/filter.json b/meta/recipes-support/icu/icu/filter.json
> > new file mode 100644
> > index 0000000000..2c63c08510
> > --- /dev/null
> > +++ b/meta/recipes-support/icu/icu/filter.json
> > @@ -0,0 +1,2 @@
> > +{
> > +}
> > diff --git a/meta/recipes-support/icu/icu_66.1.bb b/meta/recipes-support/icu/icu_66.1.bb
> > index 5018464c14..736bec4605 100644
> > --- a/meta/recipes-support/icu/icu_66.1.bb
> > +++ b/meta/recipes-support/icu/icu_66.1.bb
> > @@ -18,7 +18,10 @@ ARM_INSTRUCTION_SET_armv4 = "arm"
> >  ARM_INSTRUCTION_SET_armv5 = "arm"
> >
> >  BASE_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-src.tgz"
> > -SRC_URI = "${BASE_SRC_URI} \
> > +DATA_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-data.zip"
> > +SRC_URI = "${BASE_SRC_URI};name=code \
> > +           ${DATA_SRC_URI};name=data \
> > +           file://filter.json \
> >             file://icu-pkgdata-large-cmd.patch \
> >             file://fix-install-manx.patch \
> >             file://0001-Fix-big-endian-build.patch \
> > @@ -28,8 +31,25 @@ SRC_URI = "${BASE_SRC_URI} \
> >  SRC_URI_append_class-target = "\
> >             file://0001-Disable-LDFLAGSICUDT-for-Linux.patch \
> >            "
> > -SRC_URI[md5sum] = "b33dc6766711517c98d318447e5110f8"
> > -SRC_URI[sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e"
> > +SRC_URI[code.sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e"
> > +SRC_URI[data.sha256sum] = "8be647f738891d2beb79d48f99077b3499948430eae6f1be112553b15ab0243e"
> >
> >  UPSTREAM_CHECK_REGEX = "icu4c-(?P<pver>\d+(_\d+)+)-src"
> >  UPSTREAM_CHECK_URI = "https://github.com/unicode-org/icu/releases"
> > +
> > +do_make_icudata_class-target () {
> > +    rm -rf ${S}/data
> > +    cp -a ${WORKDIR}/data ${S}
> > +    cd ${S}
> > +    unset AR CC CPP CXX RANLIB CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
> > +    ICU_DATA_FILTER_FILE=${WORKDIR}/filter.json \
> > +    ./runConfigureICU Linux --with-data-packaging=archive
> > +    oe_runmake ${PARALLEL_MAKE}
> > +    install -Dm644 ${S}/data/out/icudt${ICU_MAJOR_VER}l.dat ${S}/data/in/icudt${ICU_MAJOR_VER}l.dat
> > +}
> > +
> > +do_make_icudata() {
> > +    :
> > +}
> > +
> > +addtask make_icudata before do_configure after do_patch
> > --
> > 2.26.0
> >

^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2020-04-04 18:54 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-04-04 17:20 [PATCH V4] icu: Add knobs to generate a subset of ICU data Khem Raj
2020-04-04 17:43 ` Khem Raj
2020-04-04 18:53   ` Khem Raj

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox