[Buildroot] [PATCH 1/2] package/go/go-cf: new package

[Buildroot] [PATCH 1/2] package/go/go-cf: new package

[James Hilliard]

This is a fork of go needed for cloudflared to work properly.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
 .checkpackageignore                           |   1 +
 ...explicit-option-for-crosscompilation.patch |   1 +
 ...ldvcs-false-when-building-go-bootstr.patch |   1 +
 package/go/go-cf/go-cf.hash                   |   3 +
 package/go/go-cf/go-cf.mk                     | 101 ++++++++++++++++++
 package/pkg-golang.mk                         |  13 ++-
 6 files changed, 117 insertions(+), 3 deletions(-)
 create mode 120000 package/go/go-cf/0001-build.go-explicit-option-for-crosscompilation.patch
 create mode 120000 package/go/go-cf/0002-cmd-dist-set-buildvcs-false-when-building-go-bootstr.patch
 create mode 100644 package/go/go-cf/go-cf.hash
 create mode 100644 package/go/go-cf/go-cf.mk

diff --git a/.checkpackageignore b/.checkpackageignore
index babcb813b4..1213bd1ece 100644
--- a/.checkpackageignore
+++ b/.checkpackageignore
@@ -553,6 +553,7 @@ package/glorytun/0002-aegis256.c-fix-aarch64-build-with-uclibc.patch lib_patch.U
 package/gnu-efi/0001-Make.defaults-don-t-override-ARCH-when-cross-compili.patch lib_patch.Upstream
 package/gnupg/0001-build-Always-use-EXTERN_UNLESS_MAIN_MODULE-pattern.patch lib_patch.Upstream
 package/gnuplot/0001-configure-add-without-demo-option.patch lib_patch.Upstream
+package/go/go-cf/0001-build.go-explicit-option-for-crosscompilation.patch lib_patch.Upstream
 package/go/go-src/0001-build.go-explicit-option-for-crosscompilation.patch lib_patch.Upstream
 package/gob2/0001-dont-include-from-prefix.patch lib_patch.Upstream
 package/gobject-introspection/0001-disable-tests.patch lib_patch.Upstream
diff --git a/package/go/go-cf/0001-build.go-explicit-option-for-crosscompilation.patch b/package/go/go-cf/0001-build.go-explicit-option-for-crosscompilation.patch
new file mode 120000
index 0000000000..e77a438613
--- /dev/null
+++ b/package/go/go-cf/0001-build.go-explicit-option-for-crosscompilation.patch
@@ -0,0 +1 @@
+../go-src/0001-build.go-explicit-option-for-crosscompilation.patch
\ No newline at end of file
diff --git a/package/go/go-cf/0002-cmd-dist-set-buildvcs-false-when-building-go-bootstr.patch b/package/go/go-cf/0002-cmd-dist-set-buildvcs-false-when-building-go-bootstr.patch
new file mode 120000
index 0000000000..ee0c654f08
--- /dev/null
+++ b/package/go/go-cf/0002-cmd-dist-set-buildvcs-false-when-building-go-bootstr.patch
@@ -0,0 +1 @@
+../go-src/0002-cmd-dist-set-buildvcs-false-when-building-go-bootstr.patch
\ No newline at end of file
diff --git a/package/go/go-cf/go-cf.hash b/package/go/go-cf/go-cf.hash
new file mode 100644
index 0000000000..78bf3eec5b
--- /dev/null
+++ b/package/go/go-cf/go-cf.hash
@@ -0,0 +1,3 @@
+# From https://go.dev/dl
+sha256  da1bcdeb86767dab40bf4debd49615a32c795689ba2ac7f6f87d2f389691fdc1  go-cf-ec0a014545f180b0c74dfd687698657a9e86e310.tar.gz
+sha256  2d36597f7117c38b006835ae7f537487207d8ec407aa9d9980794b2030cbc067  LICENSE
diff --git a/package/go/go-cf/go-cf.mk b/package/go/go-cf/go-cf.mk
new file mode 100644
index 0000000000..ac686e4e29
--- /dev/null
+++ b/package/go/go-cf/go-cf.mk
@@ -0,0 +1,101 @@
+################################################################################
+#
+# go-cf
+#
+################################################################################
+
+GO_CF_VERSION = ec0a014545f180b0c74dfd687698657a9e86e310
+GO_CF_SITE = $(call github,cloudflare,go,$(GO_CF_VERSION))
+
+GO_CF_LICENSE = BSD-3-Clause
+GO_CF_LICENSE_FILES = LICENSE
+
+HOST_GO_CF_ROOT = $(HOST_DIR)/lib/go-cf
+
+# We pass an empty GOBIN, otherwise "go install: cannot install
+# cross-compiled binaries when GOBIN is set"
+HOST_GO_CF_COMMON_ENV = \
+	GO111MODULE=on \
+	GOFLAGS=-mod=vendor \
+	GOROOT="$(HOST_GO_CF_ROOT)" \
+	GOPATH="$(HOST_GO_GOPATH)" \
+	GOCACHE="$(HOST_GO_TARGET_CACHE)" \
+	GOMODCACHE="$(HOST_GO_CF_ROOT)/pkg/mod" \
+	GOPROXY=off \
+	GOTOOLCHAIN=local \
+	PATH=$(BR_PATH) \
+	GOBIN= \
+	CGO_ENABLED=$(HOST_GO_CGO_ENABLED)
+
+# For the convenience of target packages.
+HOST_GO_CF_TOOLDIR = $(HOST_GO_CF_ROOT)/pkg/tool/linux_$(GO_GOARCH)
+HOST_GO_CF_TARGET_ENV = \
+	$(HOST_GO_CF_COMMON_ENV) \
+	GOOS="linux" \
+	GOARCH=$(GO_GOARCH) \
+	$(if $(GO_GO386),GO386=$(GO_GO386)) \
+	$(if $(GO_GOARM),GOARM=$(GO_GOARM)) \
+	CC="$(TARGET_CC)" \
+	CXX="$(TARGET_CXX)" \
+	CGO_CFLAGS="$(TARGET_CFLAGS)" \
+	CGO_CXXFLAGS="$(TARGET_CXXFLAGS)" \
+	CGO_LDFLAGS="$(TARGET_LDFLAGS)" \
+	GOTOOLDIR="$(HOST_GO_CF_TOOLDIR)"
+
+HOST_GO_CF_DEPENDENCIES = host-go
+
+ifeq ($(BR2_PACKAGE_HOST_GO_TARGET_ARCH_SUPPORTS),y)
+
+HOST_GO_CF_CROSS_ENV = \
+	CC_FOR_TARGET="$(TARGET_CC)" \
+	CXX_FOR_TARGET="$(TARGET_CXX)" \
+	GOOS="linux" \
+	GOARCH=$(GO_GOARCH) \
+	$(if $(GO_GO386),GO386=$(GO_GO386)) \
+	$(if $(GO_GOARM),GOARM=$(GO_GOARM)) \
+	GO_ASSUME_CROSSCOMPILING=1
+
+endif
+
+# The go build system is not compatible with ccache, so use
+# HOSTCC_NOCCACHE.  See https://github.com/golang/go/issues/11685.
+HOST_GO_CF_MAKE_ENV = \
+	GO111MODULE=off \
+	GOCACHE=$(HOST_GO_HOST_CACHE) \
+	GOROOT_BOOTSTRAP=$(HOST_GO_ROOT) \
+	GOROOT_FINAL=$(HOST_GO_CF_ROOT) \
+	GOROOT="$(@D)" \
+	GOBIN="$(@D)/bin" \
+	GOOS=linux \
+	CC=$(HOSTCC_NOCCACHE) \
+	CXX=$(HOSTCXX_NOCCACHE) \
+	CGO_ENABLED=$(HOST_GO_CGO_ENABLED) \
+	$(HOST_GO_CF_CROSS_ENV)
+
+define HOST_GO_CF_BUILD_CMDS
+	cd $(@D)/src && \
+		$(HOST_GO_CF_MAKE_ENV) ./make.bash $(if $(VERBOSE),-v)
+endef
+
+define HOST_GO_CF_INSTALL_CMDS
+	$(INSTALL) -D -m 0755 $(@D)/bin/go $(HOST_GO_CF_ROOT)/bin/go
+	$(INSTALL) -D -m 0755 $(@D)/bin/gofmt $(HOST_GO_CF_ROOT)/bin/gofmt
+
+	mkdir -p $(HOST_DIR)/bin
+	ln -sf ../lib/go-cf/bin/go $(HOST_DIR)/bin/go-cf
+
+	cp -a $(@D)/lib $(HOST_GO_CF_ROOT)/
+
+	mkdir -p $(HOST_GO_CF_ROOT)/pkg
+	cp -a $(@D)/pkg/include $(HOST_GO_CF_ROOT)/pkg/
+	cp -a $(@D)/pkg/tool $(HOST_GO_CF_ROOT)/pkg/
+
+	# The Go sources must be installed to the host/ tree for the Go stdlib.
+	cp -a $(@D)/src $(HOST_GO_CF_ROOT)/
+
+	# Set file timestamps to prevent the Go compiler from rebuilding the stdlib
+	# when compiling other programs.
+	find $(HOST_GO_CF_ROOT) -type f -exec touch -r $(@D)/bin/go {} \;
+endef
+
+$(eval $(host-generic-package))
diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
index 8e27602d41..2d19c044ee 100644
--- a/package/pkg-golang.mk
+++ b/package/pkg-golang.mk
@@ -21,7 +21,6 @@
 #
 ################################################################################
 
-GO_BIN = $(HOST_DIR)/bin/go
 
 ################################################################################
 # inner-golang-package -- defines how the configuration, compilation and
@@ -40,6 +39,14 @@ GO_BIN = $(HOST_DIR)/bin/go
 
 define inner-golang-package
 
+ifndef $(2)_GO_BIN
+	ifdef $(3)_GO_BIN
+		$(2)_GO_BIN = $$($(3)_GO_BIN)
+	else
+		$(2)_GO_BIN ?= $$(HOST_DIR)/bin/go
+	endif
+endif
+
 $(2)_BUILD_OPTS += \
 	-ldflags "$$($(2)_LDFLAGS)" \
 	-modcacherw \
@@ -124,7 +131,7 @@ define $(2)_BUILD_CMDS
 		cd $$(@D); \
 		$$(HOST_GO_TARGET_ENV) \
 			$$($(2)_GO_ENV) \
-			$$(GO_BIN) build -v $$($(2)_BUILD_OPTS) \
+			$$($(2)_GO_BIN) build -v $$($(2)_BUILD_OPTS) \
 			-o $$(@D)/bin/$$(or $$($(2)_BIN_NAME),$$(notdir $$(d))) \
 			$$($(2)_GOMOD)/$$(d)
 	)
@@ -136,7 +143,7 @@ define $(2)_BUILD_CMDS
 		cd $$(@D); \
 		$$(HOST_GO_HOST_ENV) \
 			$$($(2)_GO_ENV) \
-			$$(GO_BIN) build -v $$($(2)_BUILD_OPTS) \
+			$$($(2)_GO_BIN) build -v $$($(2)_BUILD_OPTS) \
 			-o $$(@D)/bin/$$(or $$($(2)_BIN_NAME),$$(notdir $$(d))) \
 			$$($(2)_GOMOD)/$$(d)
 	)
-- 
2.34.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 2/2] package/cloudflared: use go-cf toolchain

[James Hilliard]

Cloudflared needs a custom fork of golang to function properly.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
 package/cloudflared/cloudflared.mk | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/package/cloudflared/cloudflared.mk b/package/cloudflared/cloudflared.mk
index 68849e9b82..6820fe330f 100644
--- a/package/cloudflared/cloudflared.mk
+++ b/package/cloudflared/cloudflared.mk
@@ -11,5 +11,8 @@ CLOUDFLARED_LICENSE_FILES = LICENSE
 CLOUDFLARED_CPE_ID_VENDOR = cloudflare
 CLOUDFLARED_BUILD_TARGETS = cmd/cloudflared
 CLOUDFLARED_LDFLAGS = -X main.Version=$(CLOUDFLARED_VERSION)
+CLOUDFLARED_DEPENDENCIES = host-go-cf
+CLOUDFLARED_GO_BIN = $(HOST_DIR)/bin/go-cf
+CLOUDFLARED_GO_ENV = $(HOST_GO_CF_TARGET_ENV)
 
 $(eval $(golang-package))
-- 
2.34.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/2] package/go/go-cf: new package

[Thomas Petazzoni via buildroot]

On Mon, 28 Oct 2024 16:10:44 -0600
James Hilliard <james.hilliard1@gmail.com> wrote:

> This is a fork of go needed for cloudflared to work properly.
> 
> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>

This is really nuts/crazy. Are we going to have to package a different
Go compiler for each and every project?

(Note: your commit message needs a lot more explanation on how it
integrates in Buildroot, what happens if one uses both normal Go and
go-cf, whether they will conflict, or not, etc.).

Thomas
-- 
Thomas Petazzoni, co-owner and CEO, Bootlin
Embedded Linux and Kernel engineering and training
https://bootlin.com
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/2] package/go/go-cf: new package

[Christian Stewart via buildroot]

Hi Thomas, James,

On Mon, Oct 28, 2024 at 3:15 PM Thomas Petazzoni
<thomas.petazzoni@bootlin.com> wrote:
>
> On Mon, 28 Oct 2024 16:10:44 -0600
> James Hilliard <james.hilliard1@gmail.com> wrote:
>
> > This is a fork of go needed for cloudflared to work properly.
> >
> > Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
>
> This is really nuts/crazy. Are we going to have to package a different
> Go compiler for each and every project?

The maintenance overhead of adding multiple Go compilers seems not worth it.

I see from the Cloudflare fork of Go that it has these changes to TLS:

- Encrypted ClientHello (ECH)
- Post-quantum key agreement
- Delegated Credentials
- Post-quantum certificates.
- Configuraton of keyshares sent in ClientHello with
tls.Config.ClientCurveGuess.

The concept of having a forked Go in itself is not that crazy.

What makes this difficult / not worth it is how complex the Go
infrastructure is.

Just look at this patch and all the lines added: TARGET_ENV, etc. Do
we have to keep updating these every time we change the Go
infrastructure?

Is there some way we can reuse most of the code from the Go infra for this?

Best regards,
Christian Stewart
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/2] package/go/go-cf: new package

[James Hilliard]

On Mon, Oct 28, 2024 at 4:15 PM Thomas Petazzoni
<thomas.petazzoni@bootlin.com> wrote:
>
> On Mon, 28 Oct 2024 16:10:44 -0600
> James Hilliard <james.hilliard1@gmail.com> wrote:
>
> > This is a fork of go needed for cloudflared to work properly.
> >
> > Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
>
> This is really nuts/crazy. Are we going to have to package a different
> Go compiler for each and every project?

Yeah, I agree it's a bit crazy, not sure if there's a better way to handle
this however. I tried to model this on how we currently handle multiple
go toolchains for bootstrapping purposes with this simply depending
on the latest go toolchain for bootstrapping go-cf.

I sure hope not, however for cloudflared they seem to be depending
on a bunch of experimental features that are not in upstream golang.

>
> (Note: your commit message needs a lot more explanation on how it
> integrates in Buildroot, what happens if one uses both normal Go and
> go-cf, whether they will conflict, or not, etc.).

Yeah, it shouldn't conflict, I did verify I can build packages that require
golang 1.23 at the same time as cloudflared on their 1.22 based fork.

>
> Thomas
> --
> Thomas Petazzoni, co-owner and CEO, Bootlin
> Embedded Linux and Kernel engineering and training
> https://bootlin.com
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/2] package/go/go-cf: new package

[James Hilliard]

On Mon, Oct 28, 2024 at 4:24 PM Christian Stewart <christian@aperture.us> wrote:
>
> Hi Thomas, James,
>
> On Mon, Oct 28, 2024 at 3:15 PM Thomas Petazzoni
> <thomas.petazzoni@bootlin.com> wrote:
> >
> > On Mon, 28 Oct 2024 16:10:44 -0600
> > James Hilliard <james.hilliard1@gmail.com> wrote:
> >
> > > This is a fork of go needed for cloudflared to work properly.
> > >
> > > Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
> >
> > This is really nuts/crazy. Are we going to have to package a different
> > Go compiler for each and every project?
>
> The maintenance overhead of adding multiple Go compilers seems not worth it.

Issue here is cloudflared doesn't really work otherwise so not sure
what other approaches would work.

>
> I see from the Cloudflare fork of Go that it has these changes to TLS:
>
> - Encrypted ClientHello (ECH)
> - Post-quantum key agreement
> - Delegated Credentials
> - Post-quantum certificates.
> - Configuraton of keyshares sent in ClientHello with
> tls.Config.ClientCurveGuess.
>
> The concept of having a forked Go in itself is not that crazy.
>
> What makes this difficult / not worth it is how complex the Go
> infrastructure is.

Not sure how else to make cloudflared work properly since there are
runtime issues without these features.

>
> Just look at this patch and all the lines added: TARGET_ENV, etc. Do
> we have to keep updating these every time we change the Go
> infrastructure?

Well we have a bunch of these duplicated for go bootstrapping already, I
kind of just followed that existing pattern.

>
> Is there some way we can reuse most of the code from the Go infra for this?

Hmm, maybe there's a way to template the directories better? Might help
with the bootstrapping stages as well.

>
> Best regards,
> Christian Stewart
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot