[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Sonic Zhang]

Current DOWNLOAD_SVN and DOWNLOAD_GIT scripts only retrieve source code of a
given version from the repository. It is more complicated to generate patches
against the source after fixing bugs and developing in buildroot directly.
This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
source with version control information. This option is disabled by default.

Signed-off-by: Sonic Zhang <sonic.zhang@analog.com>
---
 Config.in                   |    7 +++++++
 package/Makefile.package.in |   20 ++++++++++++++++----
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/Config.in b/Config.in
index c90de77..7fa2aba 100644
--- a/Config.in
+++ b/Config.in
@@ -154,6 +154,13 @@ config BR2_DEBUG_MUDFLAP
 	  Your GCC must be new enough to support this option.
 endif
 
+config BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT
+	bool "check out source with git or svn revision information"
+	default n
+	help
+	  If y, check out the source from git or svn repository with
+	  revision information and store in the source tar ball.
+
 choice
 	prompt "strip"
 	default BR2_STRIP_strip
diff --git a/package/Makefile.package.in b/package/Makefile.package.in
index 9f876b9..78b2cda 100644
--- a/package/Makefile.package.in
+++ b/package/Makefile.package.in
@@ -100,15 +100,22 @@ endif
 # source repositories, producing a list of all the "external dependencies" of
 # a given build configuration.
 ################################################################################
-
 define DOWNLOAD_GIT
+	if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
+		GITCMD="$(GIT) checkout -b $($(PKG)_BRANCH) $($(PKG)_DL_VERSION)" ; \
+		TARCMD="$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/" ; \
+	else \
+		GITCMD="$(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
+			gzip -c > $(DL_DIR)/$($(PKG)_SOURCE)" ; \
+		TARCMD=test ; \
+	fi ; \
 	test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
 	(pushd $(DL_DIR) > /dev/null && \
 	$(GIT) clone $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \
 	pushd $($(PKG)_BASE_NAME) > /dev/null && \
-	$(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
-		gzip -c > $(DL_DIR)/$($(PKG)_SOURCE) && \
+	eval $$GITCMD && \
 	popd > /dev/null && \
+	eval $$TARCMD && \
 	rm -rf $($(PKG)_DL_DIR) && \
 	popd > /dev/null)
 endef
@@ -125,9 +132,14 @@ endef
 
 
 define DOWNLOAD_SVN
+	if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
+		SVNCMD=checkout ; \
+	else \
+		SVNCMD=export ; \
+	fi ; \
 	test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
 	(pushd $(DL_DIR) > /dev/null && \
-	$(SVN) export -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
+	$(SVN) $$SVNCMD -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
 	$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/ && \
 	rm -rf $($(PKG)_DL_DIR) && \
 	popd > /dev/null)
-- 
1.6.0

[Buildroot] [PATCH v2] buildroot:linux: Add options to checkout Linux kernel source from SVN or GIT repository.

[Sonic Zhang]

Current buildroot Config.in and linux.mk don't support kernel source in SVN
or GIT repository. A pre-checkouted local SVN or GIT kernel tree set in
KERNEL_CUSTOM_TREE is the only option. Because the local path can be anywhere,
no buildroot default config file can be defined in sub folder
target/device/company/boards to run quick building. It is inconvient to run
regression testing. Since buildroot has already support SVN and GIT repository
for application packages, it is reasonable to have it for linux kernel as well.

Signed-off-by: Sonic Zhang <sonic.zhang@analog.com>
---
 linux/Config.in |   19 ++++++++++++++++++-
 linux/linux.mk  |   31 ++++++++++++++++++++++++++++++-
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/linux/Config.in b/linux/Config.in
index 480adca..07b08e2 100644
--- a/linux/Config.in
+++ b/linux/Config.in
@@ -52,6 +52,10 @@ config BR2_LINUX_KERNEL_CUSTOM_TREE
 	help
 	  This option allows use of an already unpacked linux tree.
 
+config BR2_LINUX_KERNEL_CUSTOM_REPOSITORY
+	bool "Custom source repository"
+	help
+	  This option allows to specify the svn or git repository.
 endchoice
 
 config BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE
@@ -67,12 +71,25 @@ config BR2_LINUX_KERNEL_CUSTOM_PATH
 	string "PATH of custom kernel tree"
 	depends on BR2_LINUX_KERNEL_CUSTOM_TREE
 
+config BR2_LINUX_KERNEL_CUSTOM_REPOSITORY_LOCATION
+	string "URL of custom kernel repository"
+	depends on BR2_LINUX_KERNEL_CUSTOM_REPOSITORY
+	help
+	  This is either the clone URL of GIT or trunk/tag/branch URL of SVN. 
+
+config BR2_LINUX_KERNEL_CUSTOM_REPOSITORY_VERSION
+	string "revision(svn/git), tag(git) or branch(git) of custom kernel source"
+	depends on BR2_LINUX_KERNEL_CUSTOM_REPOSITORY
+	default "HEAD"
+	help
+	  Tag or branch of GIT repository should be filled in here.
+
 config BR2_LINUX_KERNEL_VERSION
 	string
 	default "2.6.36" if BR2_LINUX_KERNEL_2_6_36
 	default BR2_DEFAULT_KERNEL_HEADERS if BR2_LINUX_KERNEL_SAME_AS_HEADERS
 	default BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE if BR2_LINUX_KERNEL_CUSTOM_VERSION
-	default "custom" if BR2_LINUX_KERNEL_CUSTOM_TARBALL || BR2_LINUX_KERNEL_CUSTOM_TREE
+	default "custom" if BR2_LINUX_KERNEL_CUSTOM_TARBALL || BR2_LINUX_KERNEL_CUSTOM_TREE || BR2_LINUX_KERNEL_CUSTOM_REPOSITORY
 
 #
 # Patch selection
diff --git a/linux/linux.mk b/linux/linux.mk
index 4b680fd..a6306c9 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -7,11 +7,32 @@ LINUX26_VERSION=$(call qstrip,$(BR2_LINUX_KERNEL_VERSION))
 
 # Compute LINUX26_SOURCE and LINUX26_SITE from the configuration
 ifeq ($(LINUX26_VERSION),custom)
+
+ifeq ($(BR2_LINUX_KERNEL_CUSTOM_REPOSITORY),y)
+LINUX26_SITE:=$(call qstrip, $(BR2_LINUX_KERNEL_CUSTOM_REPOSITORY_LOCATION))
+LINUX26_DL_VERSION:=$(call qstrip, $(BR2_LINUX_KERNEL_CUSTOM_REPOSITORY_VERSION))
+
+ifeq ($(findstring svn://,$(LINUX26_SITE)),svn://)
+LINUX26_SITE_METHOD:=svn
+LINUX26_DL_DIR:=linux-$(notdir $(LINUX26_SITE))
+else
+LINUX26_SITE_METHOD:=git
+LINUX26_DL_DIR:=linux-$(call qstrip, $(BR2_LINUX_KERNEL_CUSTOM_REPOSITORY_VERSION))
+LINUX26_BRANCH:=$(LINUX26_DL_VERSION)
+endif
+
+LINUX26_BASE_NAME:=$(LINUX26_DL_DIR)
+LINUX26_SOURCE:=$(LINUX26_DL_DIR).tar.gz
+else
+
 ifneq ($(BR2_LINUX_KERNEL_CUSTOM_TREE),y)
 LINUX26_TARBALL:=$(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION))
 LINUX26_SITE:=$(dir $(LINUX26_TARBALL))
-LINUX26_SOURCE:=$(notdir $(LINUX26_TARBALL))
+LINUX26_SOURCE:=linux-$(notdir $(LINUX26_TARBALL))
+endif
+
 endif
+
 else
 LINUX26_SOURCE:=linux-$(LINUX26_VERSION).tar.bz2
 LINUX26_SITE:=$(BR2_KERNEL_MIRROR)/linux/kernel/v2.6/
@@ -155,6 +176,14 @@ $(LINUX26_BUILD_DIR)/.stamp_installed: $(LINUX26_BUILD_DIR)/.stamp_compiled
 	fi
 	$(Q)touch $@
 
+
+$(LINUX26_BUILD_DIR)/.stamp_downloaded: PKG=LINUX26
+$(LINUX26_BUILD_DIR)/.stamp_extracted: PKG=LINUX26
+$(LINUX26_BUILD_DIR)/.stamp_patched: PKG=LINUX26
+$(LINUX26_BUILD_DIR)/.stamp_configured: PKG=LINUX26
+$(LINUX26_BUILD_DIR)/.stamp_compiled: PKG=LINUX26
+$(LINUX26_BUILD_DIR)/.stamp_installed: PKG=LINUX26
+
 linux26: host-module-init-tools $(LINUX26_DEPENDENCIES) $(LINUX26_BUILD_DIR)/.stamp_installed
 
 linux26-menuconfig linux26-xconfig linux26-gconfig: dirs $(LINUX26_BUILD_DIR)/.stamp_configured
-- 
1.6.0

[Buildroot] [PATCH v2] buildroot:linux: Add options to checkout Linux kernel source from SVN or GIT repository.

[Thomas Petazzoni]

Hello,

On Thu, 16 Dec 2010 17:49:52 +0800
Sonic Zhang <sonic.adi@gmail.com> wrote:

> Current buildroot Config.in and linux.mk don't support kernel source in SVN
> or GIT repository. A pre-checkouted local SVN or GIT kernel tree set in
> KERNEL_CUSTOM_TREE is the only option.

The KERNEL_CUSTOM_TREE option is not in mainline Buildroot. And as per
my answer to the previous patch, I'd very much prefer a more generic
solution than a solution specific to the Linux kernel build process.

> Because the local path can be anywhere, no buildroot default config file can be defined in sub folder
> target/device/company/boards to run quick building. It is inconvient to run
> regression testing. Since buildroot has already support SVN and GIT repository
> for application packages, it is reasonable to have it for linux kernel as well.

I have a similar comment here: I agree that we must have GIT and SVN
support for the Linux build process, but we should also have it for all
the bootloaders (see the recent patch to introduce X-Loader).
Therefore, instead of solving it in a Linux-specific way, I'd prefer if
we could have a generic infrastructure (in the spirit of the package
infrastructure) that would satisfy the need of bootloaders and Linux
kernel in terms of build process.

Yes, it's a lot more work than just making the change on the Linux
kernel build process only, but on the long term, it's much more
interesting.

Regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Quotient Remainder]

Hello Sonic,

I've done a bit of work in this area, but it's now so outdated (in
before Maxime's changes, although vaguely based on the same approach)
that the patches will not apply to the current HEAD.  I will certainly
share these if anyone's interested.  They'd probably apply to 2010.08.

I have a few observations and questions based on the problems I
encountered.

Ar D?ar, 2010-12-16 ag 17:49 +0800, scr?obh Sonic Zhang:
> This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
> source with version control information. This option is disabled by default.
> 

What if you only want to have working copies of some of the
repositories, not all?

> +	if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
> +		GITCMD="$(GIT) checkout -b $($(PKG)_BRANCH) $($(PKG)_DL_VERSION)" ; \
> +		TARCMD="$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/" ; \

Where does $($(PKG)_BRANCH) get its value?


An finally, when you make a change in your repository working copy, how
does this change get into $(BUILD_DIR) so it's part of the build?


As I say, I have found a way to do these but it's not elegant and will
probably not be suitable for inclusion in mainline without serious
restructuring.

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Sonic Zhang]

On Thu, Dec 16, 2010 at 10:19 PM, Quotient Remainder
<quotientvremainder@gmail.com> wrote:
> Hello Sonic,
>
> I've done a bit of work in this area, but it's now so outdated (in
> before Maxime's changes, although vaguely based on the same approach)
> that the patches will not apply to the current HEAD. ?I will certainly
> share these if anyone's interested. ?They'd probably apply to 2010.08.
>
> I have a few observations and questions based on the problems I
> encountered.
>
> Ar D?ar, 2010-12-16 ag 17:49 +0800, scr?obh Sonic Zhang:
>> This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
>> source with version control information. This option is disabled by default.
>>
>
> What if you only want to have working copies of some of the
> repositories, not all?
>

This option only controls if the revision database is fetched. You can
indicate working copy version or URL in the package's config file.
Fetching revision database for all SVN/GIT packages is OK.


>> + ? ? if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
>> + ? ? ? ? ? ? GITCMD="$(GIT) checkout -b $($(PKG)_BRANCH) $($(PKG)_DL_VERSION)" ; \
>> + ? ? ? ? ? ? TARCMD="$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/" ; \
>
> Where does $($(PKG)_BRANCH) get its value?
>

The package's Config.in should include this option if it points to a
GIT repository.

>
> An finally, when you make a change in your repository working copy, how
> does this change get into $(BUILD_DIR) so it's part of the build?
>

By now, you have to either run "svn update" and "git pull/checkout"
manually under output/build/package_name/ or make clean and remove the
tarball under download folder.

Sonic

>
> As I say, I have found a way to do these but it's not elegant and will
> probably not be suitable for inclusion in mainline without serious
> restructuring.
>
>

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Lionel Landwerlin]

Hello Sonic,

I'm very pleased to see this BR2_CHECKOUT_SOURCE_FOR_DEV option. But I
have a slight point to make, why make a tarball of the checkout tree ?
It seems quite slow to me, because it stores the source code twice (once
in the .svn/.git directory, and once in the working tree).

I've the attached patch which only makes the checkout and do not
generate a tarball (but it might be frustrating when you're offline
after a <pkg>-dirclean...).

Wouldn't it be possible to find a solution to just store .svn/.git
directories and trigger a git-checkout -f $(BRANCH)/svn update --force
after extraction?

Best regards,

--
Lionel Landwerlin

Le jeudi 16 d?cembre 2010 ? 17:49 +0800, Sonic Zhang a ?crit :
> Current DOWNLOAD_SVN and DOWNLOAD_GIT scripts only retrieve source code of a
> given version from the repository. It is more complicated to generate patches
> against the source after fixing bugs and developing in buildroot directly.
> This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
> source with version control information. This option is disabled by default.
> 
> Signed-off-by: Sonic Zhang <sonic.zhang@analog.com>
> ---
>  Config.in                   |    7 +++++++
>  package/Makefile.package.in |   20 ++++++++++++++++----
>  2 files changed, 23 insertions(+), 4 deletions(-)
> 
> diff --git a/Config.in b/Config.in
> index c90de77..7fa2aba 100644
> --- a/Config.in
> +++ b/Config.in
> @@ -154,6 +154,13 @@ config BR2_DEBUG_MUDFLAP
>  	  Your GCC must be new enough to support this option.
>  endif
>  
> +config BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT
> +	bool "check out source with git or svn revision information"
> +	default n
> +	help
> +	  If y, check out the source from git or svn repository with
> +	  revision information and store in the source tar ball.
> +
>  choice
>  	prompt "strip"
>  	default BR2_STRIP_strip
> diff --git a/package/Makefile.package.in b/package/Makefile.package.in
> index 9f876b9..78b2cda 100644
> --- a/package/Makefile.package.in
> +++ b/package/Makefile.package.in
> @@ -100,15 +100,22 @@ endif
>  # source repositories, producing a list of all the "external dependencies" of
>  # a given build configuration.
>  ################################################################################
> -
>  define DOWNLOAD_GIT
> +	if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
> +		GITCMD="$(GIT) checkout -b $($(PKG)_BRANCH) $($(PKG)_DL_VERSION)" ; \
> +		TARCMD="$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/" ; \
> +	else \
> +		GITCMD="$(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
> +			gzip -c > $(DL_DIR)/$($(PKG)_SOURCE)" ; \
> +		TARCMD=test ; \
> +	fi ; \
>  	test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
>  	(pushd $(DL_DIR) > /dev/null && \
>  	$(GIT) clone $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \
>  	pushd $($(PKG)_BASE_NAME) > /dev/null && \
> -	$(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
> -		gzip -c > $(DL_DIR)/$($(PKG)_SOURCE) && \
> +	eval $$GITCMD && \
>  	popd > /dev/null && \
> +	eval $$TARCMD && \
>  	rm -rf $($(PKG)_DL_DIR) && \
>  	popd > /dev/null)
>  endef
> @@ -125,9 +132,14 @@ endef
>  
> 
>  define DOWNLOAD_SVN
> +	if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
> +		SVNCMD=checkout ; \
> +	else \
> +		SVNCMD=export ; \
> +	fi ; \
>  	test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
>  	(pushd $(DL_DIR) > /dev/null && \
> -	$(SVN) export -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
> +	$(SVN) $$SVNCMD -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
>  	$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/ && \
>  	rm -rf $($(PKG)_DL_DIR) && \
>  	popd > /dev/null)

-------------- next part --------------
A non-text attachment was scrubbed...
Name: rework-download.patch
Type: text/x-patch
Size: 2644 bytes
Desc: not available
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20101216/e3ac198e/attachment.bin>

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Sonic Zhang]

On Fri, Dec 17, 2010 at 5:22 AM, Lionel Landwerlin
<llandwerlin@gmail.com> wrote:
> Hello Sonic,
>
> I'm very pleased to see this BR2_CHECKOUT_SOURCE_FOR_DEV option. But I
> have a slight point to make, why make a tarball of the checkout tree ?
> It seems quite slow to me, because it stores the source code twice (once
> in the .svn/.git directory, and once in the working tree).
>

I just avoid changes to buildroot building process in this patch. I
can take a look.

> I've the attached patch which only makes the checkout and do not
> generate a tarball (but it might be frustrating when you're offline
> after a <pkg>-dirclean...).
>
> Wouldn't it be possible to find a solution to just store .svn/.git
> directories and trigger a git-checkout -f $(BRANCH)/svn update --force
> after extraction?
>
> Best regards,
>
> --
> Lionel Landwerlin
>
> Le jeudi 16 d?cembre 2010 ? 17:49 +0800, Sonic Zhang a ?crit :
>> Current DOWNLOAD_SVN and DOWNLOAD_GIT scripts only retrieve source code of a
>> given version from the repository. It is more complicated to generate patches
>> against the source after fixing bugs and developing in buildroot directly.
>> This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
>> source with version control information. This option is disabled by default.
>>
>> Signed-off-by: Sonic Zhang <sonic.zhang@analog.com>
>> ---
>> ?Config.in ? ? ? ? ? ? ? ? ? | ? ?7 +++++++
>> ?package/Makefile.package.in | ? 20 ++++++++++++++++----
>> ?2 files changed, 23 insertions(+), 4 deletions(-)
>>
>> diff --git a/Config.in b/Config.in
>> index c90de77..7fa2aba 100644
>> --- a/Config.in
>> +++ b/Config.in
>> @@ -154,6 +154,13 @@ config BR2_DEBUG_MUDFLAP
>> ? ? ? ? Your GCC must be new enough to support this option.
>> ?endif
>>
>> +config BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT
>> + ? ? bool "check out source with git or svn revision information"
>> + ? ? default n
>> + ? ? help
>> + ? ? ? If y, check out the source from git or svn repository with
>> + ? ? ? revision information and store in the source tar ball.
>> +
>> ?choice
>> ? ? ? prompt "strip"
>> ? ? ? default BR2_STRIP_strip
>> diff --git a/package/Makefile.package.in b/package/Makefile.package.in
>> index 9f876b9..78b2cda 100644
>> --- a/package/Makefile.package.in
>> +++ b/package/Makefile.package.in
>> @@ -100,15 +100,22 @@ endif
>> ?# source repositories, producing a list of all the "external dependencies" of
>> ?# a given build configuration.
>> ?################################################################################
>> -
>> ?define DOWNLOAD_GIT
>> + ? ? if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
>> + ? ? ? ? ? ? GITCMD="$(GIT) checkout -b $($(PKG)_BRANCH) $($(PKG)_DL_VERSION)" ; \
>> + ? ? ? ? ? ? TARCMD="$(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/" ; \
>> + ? ? else \
>> + ? ? ? ? ? ? GITCMD="$(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
>> + ? ? ? ? ? ? ? ? ? ? gzip -c > $(DL_DIR)/$($(PKG)_SOURCE)" ; \
>> + ? ? ? ? ? ? TARCMD=test ; \
>> + ? ? fi ; \
>> ? ? ? test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
>> ? ? ? (pushd $(DL_DIR) > /dev/null && \
>> ? ? ? $(GIT) clone $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \
>> ? ? ? pushd $($(PKG)_BASE_NAME) > /dev/null && \
>> - ? ? $(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
>> - ? ? ? ? ? ? gzip -c > $(DL_DIR)/$($(PKG)_SOURCE) && \
>> + ? ? eval $$GITCMD && \
>> ? ? ? popd > /dev/null && \
>> + ? ? eval $$TARCMD && \
>> ? ? ? rm -rf $($(PKG)_DL_DIR) && \
>> ? ? ? popd > /dev/null)
>> ?endef
>> @@ -125,9 +132,14 @@ endef
>>
>>
>> ?define DOWNLOAD_SVN
>> + ? ? if [ "$(BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT)" = "y" ] ; then \
>> + ? ? ? ? ? ? SVNCMD=checkout ; \
>> + ? ? else \
>> + ? ? ? ? ? ? SVNCMD=export ; \
>> + ? ? fi ; \
>> ? ? ? test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
>> ? ? ? (pushd $(DL_DIR) > /dev/null && \
>> - ? ? $(SVN) export -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
>> + ? ? $(SVN) $$SVNCMD -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
>> ? ? ? $(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/ && \
>> ? ? ? rm -rf $($(PKG)_DL_DIR) && \
>> ? ? ? popd > /dev/null)
>
>
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot
>

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Thomas Petazzoni]

Hello Sonic,

On Thu, 16 Dec 2010 17:49:51 +0800
Sonic Zhang <sonic.adi@gmail.com> wrote:

> Current DOWNLOAD_SVN and DOWNLOAD_GIT scripts only retrieve source code of a
> given version from the repository. It is more complicated to generate patches
> against the source after fixing bugs and developing in buildroot directly.
> This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
> source with version control information. This option is disabled by default.

I understand the need for this, and fully agree with it, but I don't
think that's the right way to solve it.

Currently, Buildroot takes care of downloading, extracting, patching,
configuring, building and installing packages. But for some packages,
we would like Buildroot to not take care of the download/extract/patch
steps, and let the user point Buildroot to a particular location where
the source tree of the component is already available. The use case is
that Buildroot users want to hack on the kernel, on a particular
library or application: they want to have the source tree of this
component is whatever location they want, to manage this source code
with whatever version control system they want, and they don't want
Buildroot to remove the source code when a global "make clean" is not
in Buildroot.

The need solved by your proposal is, I think, solved by this general
use case. Unfortunately, your solution does not solve the entire
problem:

 * It is only valid for components for which sourcing from git/svn is
   done.

 * It still keeps the source tree in Buildroot's build directory, which
   gets removed at "make clean".

I haven't implemented yet an alternative solution, but through a few
discussions on IRC with other Buildroot contributors, the rough scheme
we came up with is the following.

 Allow the user to write a "local.mk" file that gets included before
 any package .mk file. In this "local.mk" file, the user can define
 variables such as DIRECTFB_OVERRIDE_SOURCE_DIR=/home/foobar/directfb/.
 The package infrastructure would of course be modified to take into
 account this new variable, and use the given directory as the source
 tree to compile the DirectFB package.

Again, that's a rough scheme, but at least it should solve your use
case, but also the use case brought by Mike who wanted to specify a
custom source tree for the kernel.

Does that make sense ? Would it solve your use case ?

Of course, I can't guarantee any date for the implementation of this.
Buildroot is, IMO, still in a very large cleanup phase started almost 2
years ago, and I'm currently focusing on finalizing that cleanup phase
before extending Buildroot with radically new use cases. But of course,
if there are contributions going into this direction, they'll be very
welcome.

Regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Quotient Remainder]

Hello Thomas,

I'm glad this is being debated again.  The itch that just won't go
away... :-)

Ar Aoine, 2010-12-17 ag 11:34 +0100, scr?obh Thomas Petazzoni:

> But for some packages,
> we would like Buildroot to not take care of the download/extract/patch
> steps, and let the user point Buildroot to a particular location where
> the source tree of the component is already available. 

There are many great features in Buildroot and one of them is that you
can just give it a defconfig and off it goes to retrieve all the
required components.  It would be a pity to leave this out of the
"working tree" support structure.
I had the idea (not implemented yet) that Buildroot should try to check
out $(PKG)_DL_VERSION and if it fails, get an update from upstream and
try to checkout again, failing only at this point.  Of course changes to
the working copy would have to be cautious so that changes are not
overwritten unintentionally.  OK the thinking is a bit GIT-centric but
can be expanded to other VCS also.  Doesn't everything start by
supporting a limited set before aiming for world domination...

> The use case is
> that Buildroot users want to hack on the kernel, on a particular
> library or application: they want to have the source tree of this
> component is whatever location they want, to manage this source code
> with whatever version control system they want, and they don't want
> Buildroot to remove the source code when a global "make clean" is not
> in Buildroot.

The way I see it Buildroot is used both by developers ("I want to change
source code") and by those who just want a repeatable way to make a
bootable system ("just give me a burnable image").  It would be ideal if
the same system could be used by both camps.
Keeping the source code outside BUILD_DIR through your OVERRIDE_SRC_DIR
seems like a big step towards this.  "make clean" just deletes
BUILD_DIR, not the OVERRIDE dirs.

>  * It is only valid for components for which sourcing from git/svn is
>    done.

But didn't someone post a BZR patch recently and can't it be extended to
any other VCS with an equivalent DOWNLOAD_$(VCS) macro?  Or am I missing
your point?

>  * It still keeps the source tree in Buildroot's build directory, which
>    gets removed at "make clean".
> 
> I haven't implemented yet an alternative solution, but through a few
> discussions on IRC with other Buildroot contributors, the rough scheme
> we came up with is the following.

I did, but it's really dirty, invloving a cautious rsync at build-time
between the working copy and BUILD_DIR.  Your OVERRIDE_DIR is vastly
superior and glaringly obvious now that it's mentioned.
One nice feature of my approach was that it needed no modifications in
the package makefiles, if a VCS method was specified (in a file similar
to your "local.mk") the location and variables there override the
package makefile.

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Thomas Petazzoni]

Hello,

On Fri, 17 Dec 2010 13:11:41 +0000
Quotient Remainder <quotientvremainder@gmail.com> wrote:

> There are many great features in Buildroot and one of them is that you
> can just give it a defconfig and off it goes to retrieve all the
> required components.  It would be a pity to leave this out of the
> "working tree" support structure.
> I had the idea (not implemented yet) that Buildroot should try to check
> out $(PKG)_DL_VERSION and if it fails, get an update from upstream and
> try to checkout again, failing only at this point.  Of course changes to
> the working copy would have to be cautious so that changes are not
> overwritten unintentionally.  OK the thinking is a bit GIT-centric but
> can be expanded to other VCS also.  Doesn't everything start by
> supporting a limited set before aiming for world domination...

Sorry, I didn't follow you here. Could you elaborate ?

> The way I see it Buildroot is used both by developers ("I want to change
> source code") and by those who just want a repeatable way to make a
> bootable system ("just give me a burnable image").  It would be ideal if
> the same system could be used by both camps.
> Keeping the source code outside BUILD_DIR through your OVERRIDE_SRC_DIR
> seems like a big step towards this.  "make clean" just deletes
> BUILD_DIR, not the OVERRIDE dirs.

Yes, that's the plan.

> >  * It is only valid for components for which sourcing from git/svn is
> >    done.
> 
> But didn't someone post a BZR patch recently and can't it be extended to
> any other VCS with an equivalent DOWNLOAD_$(VCS) macro?  Or am I missing
> your point?

What I meant here is that components that are fetched from tarballs are
not being handled by Sonic's approach.

> I did, but it's really dirty, invloving a cautious rsync at build-time
> between the working copy and BUILD_DIR.  Your OVERRIDE_DIR is vastly
> superior and glaringly obvious now that it's mentioned.
> One nice feature of my approach was that it needed no modifications in
> the package makefiles, if a VCS method was specified (in a file similar
> to your "local.mk") the location and variables there override the
> package makefile.

The approach I propose also shouldn't require any modification to the
package makefiles. It should only require modifications at the package
infrastructure level.

Regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Quotient Remainder]

Hello Thomas

Ar Luan, 2010-12-20 ag 15:05 +0100, scr?obh Thomas Petazzoni:

> Sorry, I didn't follow you here. Could you elaborate ?

Lets say that you've cloned a GIT repository for PKG_X a few weeks ago
and done a build, now you get a new configuration (probably by changing
"local.mk" in my understanding of your scheme) and the version string
for PKG_X now contains a value that is not present in the local clone.
So, you type "make" and Buildroot tries to check out the specified new
version, which will fail since the reference is unknown.  At this stage,
I propose that Buildroot should attempt to get an update from upstream
(git fetch origin) and then attempt to check out the version again,
failing if this isn't possible.  It just makes things easier when
there's distributed development going on that an individual wouldn't
have to manually update every repository that is used (in my current
project I have in excess of 40 and this will probably reach 100 before
the project is complete).
A similar principle applies more directly for SVN repository versions,
since version checkouts are on-line.

> "make clean" just deletes
> > BUILD_DIR, not the OVERRIDE dirs.
> 
> Yes, that's the plan.

Once possible problem I envisage with this is that the built objects in
these overridden directories will stick around.  This may lead to
problems.  I suppose the clean/distclean/mrproper target for these could
be called but it's hardly the most reliable approach.  You may have a
cleaner solution in mind.

> What I meant here is that components that are fetched from tarballs are
> not being handled by Sonic's approach.

Aha, I think you mean that the OVERRIDE_DIR applies regardless of where
the package comes from.  Right?

> The approach I propose also shouldn't require any modification to the
> package makefiles. It should only require modifications at the package
> infrastructure level.

Excellent!

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Thomas Petazzoni]

On Mon, 20 Dec 2010 16:04:09 +0000
Quotient Remainder <quotientvremainder@gmail.com> wrote:

> Lets say that you've cloned a GIT repository for PKG_X a few weeks ago
> and done a build, now you get a new configuration (probably by changing
> "local.mk" in my understanding of your scheme) and the version string
> for PKG_X now contains a value that is not present in the local clone.
> So, you type "make" and Buildroot tries to check out the specified new
> version, which will fail since the reference is unknown.  At this stage,
> I propose that Buildroot should attempt to get an update from upstream
> (git fetch origin) and then attempt to check out the version again,
> failing if this isn't possible.  It just makes things easier when
> there's distributed development going on that an individual wouldn't
> have to manually update every repository that is used (in my current
> project I have in excess of 40 and this will probably reach 100 before
> the project is complete).
> A similar principle applies more directly for SVN repository versions,
> since version checkouts are on-line.

The idea is that if you override the source directory of a package with
<pkg>_OVERRIDE_DIR, Buildroot wouldn't look at <pkg>_VERSION anymore.
It's completely up to the developer to use <pkg>_OVERRIDE_DIR to point
to a directory that contains the correct version for the sources.

> Once possible problem I envisage with this is that the built objects in
> these overridden directories will stick around.  This may lead to
> problems.  I suppose the clean/distclean/mrproper target for these could
> be called but it's hardly the most reliable approach.  You may have a
> cleaner solution in mind.

I haven't thought too much, but there are clearly issues that needs to
be solved. For example when a given package is compiled once for the
target and once for the host.

> > What I meant here is that components that are fetched from tarballs are
> > not being handled by Sonic's approach.
> 
> Aha, I think you mean that the OVERRIDE_DIR applies regardless of where
> the package comes from.  Right?

Yes.

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Quotient Remainder]

Ar Luan, 2010-12-20 ag 17:08 +0100, scr?obh Thomas Petazzoni:

> The idea is that if you override the source directory of a package with
> <pkg>_OVERRIDE_DIR, Buildroot wouldn't look at <pkg>_VERSION anymore.
> It's completely up to the developer to use <pkg>_OVERRIDE_DIR to point
> to a directory that contains the correct version for the sources.

But going back to what I said eariler, it'd be a pity if Buildroot
didn't help out with this.  Couldn't the VCS feature *use* the override
feature as well as adding some support for actually managing these
repositories?

> I haven't thought too much, but there are clearly issues that needs to
> be solved. For example when a given package is compiled once for the
> target and once for the host.

Yes, and this is another reason simple symbolic links won't work,
unfortunately.

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Thomas Petazzoni]

On Mon, 20 Dec 2010 16:37:00 +0000
Quotient Remainder <quotientvremainder@gmail.com> wrote:

> > The idea is that if you override the source directory of a package with
> > <pkg>_OVERRIDE_DIR, Buildroot wouldn't look at <pkg>_VERSION anymore.
> > It's completely up to the developer to use <pkg>_OVERRIDE_DIR to point
> > to a directory that contains the correct version for the sources.
> 
> But going back to what I said eariler, it'd be a pity if Buildroot
> didn't help out with this.  Couldn't the VCS feature *use* the override
> feature as well as adding some support for actually managing these
> repositories?

I don't see how that would work, while keeping a sane semantic that
users can understand.

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Sonic Zhang]

Hi Tom,

On Fri, Dec 17, 2010 at 6:34 PM, Thomas Petazzoni
<thomas.petazzoni@free-electrons.com> wrote:
> Hello Sonic,
>
> On Thu, 16 Dec 2010 17:49:51 +0800
> Sonic Zhang <sonic.adi@gmail.com> wrote:
>
>> Current DOWNLOAD_SVN and DOWNLOAD_GIT scripts only retrieve source code of a
>> given version from the repository. It is more complicated to generate patches
>> against the source after fixing bugs and developing in buildroot directly.
>> This patch adds an option BR2_CHECKOUT_SOURCE_FOR_DEVELOPMENT to checkout the
>> source with version control information. This option is disabled by default.
>
> I understand the need for this, and fully agree with it, but I don't
> think that's the right way to solve it.
>
> Currently, Buildroot takes care of downloading, extracting, patching,
> configuring, building and installing packages. But for some packages,
> we would like Buildroot to not take care of the download/extract/patch
> steps, and let the user point Buildroot to a particular location where
> the source tree of the component is already available. The use case is
> that Buildroot users want to hack on the kernel, on a particular
> library or application: they want to have the source tree of this
> component is whatever location they want, to manage this source code
> with whatever version control system they want, and they don't want
> Buildroot to remove the source code when a global "make clean" is not
> in Buildroot.
>
> The need solved by your proposal is, I think, solved by this general
> use case. Unfortunately, your solution does not solve the entire
> problem:
>
> ?* It is only valid for components for which sourcing from git/svn is
> ? done.
>

Yes, my patch doesn't take care of other VCS, because current package
makefile support git/svn only as well.


> ?* It still keeps the source tree in Buildroot's build directory, which
> ? gets removed at "make clean".
>

This problem can be solved by 1) store the git/svn tree in download
folder other than converting to tarballs, 2) create a symbol link in
output/build folder to the git/svn tree in download folder. When make
clean, only the symbol link is removed.


> I haven't implemented yet an alternative solution, but through a few
> discussions on IRC with other Buildroot contributors, the rough scheme
> we came up with is the following.
>
> ?Allow the user to write a "local.mk" file that gets included before
> ?any package .mk file. In this "local.mk" file, the user can define
> ?variables such as DIRECTFB_OVERRIDE_SOURCE_DIR=/home/foobar/directfb/.
> ?The package infrastructure would of course be modified to take into
> ?account this new variable, and use the given directory as the source
> ?tree to compile the DirectFB package.
>
> Again, that's a rough scheme, but at least it should solve your use
> case, but also the use case brought by Mike who wanted to specify a
> custom source tree for the kernel.
>
> Does that make sense ? Would it solve your use case ?

Yes, this helps developers who has some separate local source trees.
But it is not as convenient as automatically storing svn/git tree in
download folder mentioned above, when you want to check out complete
sources trees in a distribution at once. If this is something
buildroot Makefile can help?


Sonic

>
> Of course, I can't guarantee any date for the implementation of this.
> Buildroot is, IMO, still in a very large cleanup phase started almost 2
> years ago, and I'm currently focusing on finalizing that cleanup phase
> before extending Buildroot with radically new use cases. But of course,
> if there are contributions going into this direction, they'll be very
> welcome.
>
> Regards,
>
> Thomas
> --
> Thomas Petazzoni, Free Electrons
> Kernel, drivers, real-time and embedded Linux
> development, consulting, training and support.
> http://free-electrons.com
>

[Buildroot] [PATCH v2] buildroot:download: Add option to download SVN or GIT repository with version control information.

[Daniel Nyström]

>> ?Allow the user to write a "local.mk" file that gets included before
>> ?any package .mk file. In this "local.mk" file, the user can define
>> ?variables such as DIRECTFB_OVERRIDE_SOURCE_DIR=/home/foobar/directfb/.
>> ?The package infrastructure would of course be modified to take into
>> ?account this new variable, and use the given directory as the source
>> ?tree to compile the DirectFB package.
>>
>> Again, that's a rough scheme, but at least it should solve your use
>> case, but also the use case brought by Mike who wanted to specify a
>> custom source tree for the kernel.
>>
>> Does that make sense ? Would it solve your use case ?
>
> Yes, this helps developers who has some separate local source trees.
> But it is not as convenient as automatically storing svn/git tree in
> download folder mentioned above, when you want to check out complete
> sources trees in a distribution at once. If this is something
> buildroot Makefile can help?

Another great benefit of setting path to local source trees, is that
you may want to do some trial and error without flooding your commit
log.