* [RFC/PATCH 1/8] Add initial external odb support
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 2/8] external odb foreach Christian Couder
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
From: Jeff King <peff@peff.net>
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
Makefile | 2 +
cache.h | 9 ++
external-odb.c | 115 +++++++++++++++++++++++
external-odb.h | 8 ++
odb-helper.c | 239 ++++++++++++++++++++++++++++++++++++++++++++++++
odb-helper.h | 25 +++++
sha1_file.c | 64 ++++++++++---
t/t0400-external-odb.sh | 48 ++++++++++
8 files changed, 495 insertions(+), 15 deletions(-)
create mode 100644 external-odb.c
create mode 100644 external-odb.h
create mode 100644 odb-helper.c
create mode 100644 odb-helper.h
create mode 100755 t/t0400-external-odb.sh
diff --git a/Makefile b/Makefile
index de5a030..8641318 100644
--- a/Makefile
+++ b/Makefile
@@ -726,6 +726,7 @@ LIB_OBJS += ewah/ewah_bitmap.o
LIB_OBJS += ewah/ewah_io.o
LIB_OBJS += ewah/ewah_rlw.o
LIB_OBJS += exec_cmd.o
+LIB_OBJS += external-odb.o
LIB_OBJS += fetch-pack.o
LIB_OBJS += fsck.o
LIB_OBJS += gettext.o
@@ -757,6 +758,7 @@ LIB_OBJS += notes-cache.o
LIB_OBJS += notes-merge.o
LIB_OBJS += notes-utils.o
LIB_OBJS += object.o
+LIB_OBJS += odb-helper.o
LIB_OBJS += pack-bitmap.o
LIB_OBJS += pack-bitmap-write.o
LIB_OBJS += pack-check.o
diff --git a/cache.h b/cache.h
index 6049f86..cc0a934 100644
--- a/cache.h
+++ b/cache.h
@@ -864,6 +864,12 @@ const char *git_path_shallow(void);
*/
extern const char *sha1_file_name(const unsigned char *sha1);
+/*
+ * Like sha1_file_name, but return the filename within a specific alternate
+ * object directory. Shares the same static buffer with sha1_file_name.
+ */
+extern const char *sha1_file_name_alt(const char *objdir, const unsigned char *sha1);
+
/*
* Return the name of the (local) packfile with the specified sha1 in
* its name. The return value is a pointer to memory that is
@@ -1093,6 +1099,8 @@ extern int do_check_packed_object_crc;
extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type);
+extern int create_object_tmpfile(struct strbuf *tmp, const char *filename);
+extern void close_sha1_file(int fd);
extern int finalize_object_file(const char *tmpfile, const char *filename);
extern int has_sha1_pack(const unsigned char *sha1);
@@ -1338,6 +1346,7 @@ extern void read_info_alternates(const char * relative_base, int depth);
extern void add_to_alternates_file(const char *reference);
typedef int alt_odb_fn(struct alternate_object_database *, void *);
extern int foreach_alt_odb(alt_odb_fn, void*);
+extern void prepare_external_alt_odb(void);
struct pack_window {
struct pack_window *next;
diff --git a/external-odb.c b/external-odb.c
new file mode 100644
index 0000000..1ccfa99
--- /dev/null
+++ b/external-odb.c
@@ -0,0 +1,115 @@
+#include "cache.h"
+#include "external-odb.h"
+#include "odb-helper.h"
+
+static struct odb_helper *helpers;
+static struct odb_helper **helpers_tail = &helpers;
+
+static struct odb_helper *find_or_create_helper(const char *name, int len)
+{
+ struct odb_helper *o;
+
+ for (o = helpers; o; o = o->next)
+ if (!strncmp(o->name, name, len) && !o->name[len])
+ return o;
+
+ o = odb_helper_new(name, len);
+ *helpers_tail = o;
+ helpers_tail = &o->next;
+
+ return o;
+}
+
+static int external_odb_config(const char *var, const char *value, void *data)
+{
+ struct odb_helper *o;
+ const char *key, *dot;
+
+ if (!skip_prefix(var, "odb.", &key))
+ return 0;
+ dot = strrchr(key, '.');
+ if (!dot)
+ return 0;
+
+ o = find_or_create_helper(key, dot - key);
+ key = dot + 1;
+
+ if (!strcmp(key, "command"))
+ return git_config_string(&o->cmd, var, value);
+
+ return 0;
+}
+
+static void external_odb_init(void)
+{
+ static int initialized;
+
+ if (initialized)
+ return;
+ initialized = 1;
+
+ git_config(external_odb_config, NULL);
+}
+
+const char *external_odb_root(void)
+{
+ static const char *root;
+ if (!root)
+ root = git_pathdup("objects/external");
+ return root;
+}
+
+int external_odb_has_object(const unsigned char *sha1)
+{
+ struct odb_helper *o;
+
+ external_odb_init();
+
+ for (o = helpers; o; o = o->next)
+ if (odb_helper_has_object(o, sha1))
+ return 1;
+ return 0;
+}
+
+int external_odb_fetch_object(const unsigned char *sha1)
+{
+ struct odb_helper *o;
+ const char *path;
+
+ if (!external_odb_has_object(sha1))
+ return -1;
+
+ path = sha1_file_name_alt(external_odb_root(), sha1);
+ safe_create_leading_directories_const(path);
+ prepare_external_alt_odb();
+
+ for (o = helpers; o; o = o->next) {
+ struct strbuf tmpfile = STRBUF_INIT;
+ int ret;
+ int fd;
+
+ if (!odb_helper_has_object(o, sha1))
+ continue;
+
+ fd = create_object_tmpfile(&tmpfile, path);
+ if (fd < 0) {
+ strbuf_release(&tmpfile);
+ return -1;
+ }
+
+ if (odb_helper_fetch_object(o, sha1, fd) < 0) {
+ close(fd);
+ unlink(tmpfile.buf);
+ strbuf_release(&tmpfile);
+ continue;
+ }
+
+ close_sha1_file(fd);
+ ret = finalize_object_file(tmpfile.buf, path);
+ strbuf_release(&tmpfile);
+ if (!ret)
+ return 0;
+ }
+
+ return -1;
+}
diff --git a/external-odb.h b/external-odb.h
new file mode 100644
index 0000000..2397477
--- /dev/null
+++ b/external-odb.h
@@ -0,0 +1,8 @@
+#ifndef EXTERNAL_ODB_H
+#define EXTERNAL_ODB_H
+
+const char *external_odb_root(void);
+int external_odb_has_object(const unsigned char *sha1);
+int external_odb_fetch_object(const unsigned char *sha1);
+
+#endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
new file mode 100644
index 0000000..7029a59
--- /dev/null
+++ b/odb-helper.c
@@ -0,0 +1,239 @@
+#include "cache.h"
+#include "object.h"
+#include "argv-array.h"
+#include "odb-helper.h"
+#include "run-command.h"
+#include "sha1-lookup.h"
+
+struct odb_helper *odb_helper_new(const char *name, int namelen)
+{
+ struct odb_helper *o;
+
+ o = xcalloc(1, sizeof(*o));
+ o->name = xmemdupz(name, namelen);
+
+ return o;
+}
+
+struct odb_helper_cmd {
+ struct argv_array argv;
+ struct child_process child;
+};
+
+static void prepare_helper_command(struct argv_array *argv, const char *cmd,
+ const char *fmt, va_list ap)
+{
+ struct strbuf buf = STRBUF_INIT;
+
+ strbuf_addstr(&buf, cmd);
+ strbuf_addch(&buf, ' ');
+ strbuf_vaddf(&buf, fmt, ap);
+
+ argv_array_push(argv, buf.buf);
+ strbuf_release(&buf);
+}
+
+__attribute__((format (printf,3,4)))
+static int odb_helper_start(struct odb_helper *o,
+ struct odb_helper_cmd *cmd,
+ const char *fmt, ...)
+{
+ va_list ap;
+
+ memset(cmd, 0, sizeof(*cmd));
+ argv_array_init(&cmd->argv);
+
+ if (!o->cmd)
+ return -1;
+
+ va_start(ap, fmt);
+ prepare_helper_command(&cmd->argv, o->cmd, fmt, ap);
+ va_end(ap);
+
+ cmd->child.argv = cmd->argv.argv;
+ cmd->child.use_shell = 1;
+ cmd->child.no_stdin = 1;
+ cmd->child.out = -1;
+
+ if (start_command(&cmd->child) < 0) {
+ argv_array_clear(&cmd->argv);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int odb_helper_finish(struct odb_helper *o,
+ struct odb_helper_cmd *cmd)
+{
+ int ret = finish_command(&cmd->child);
+ argv_array_clear(&cmd->argv);
+ if (ret) {
+ warning("odb helper '%s' reported failure", o->name);
+ return -1;
+ }
+ return 0;
+}
+
+static int parse_object_line(struct odb_helper_object *o, const char *line)
+{
+ char *end;
+ if (get_sha1_hex(line, o->sha1) < 0)
+ return -1;
+
+ line += 40;
+ if (*line++ != ' ')
+ return -1;
+
+ o->size = strtoul(line, &end, 10);
+ if (line == end || *end++ != ' ')
+ return -1;
+
+ o->type = type_from_string(end);
+ return 0;
+}
+
+static int odb_helper_object_cmp(const void *va, const void *vb)
+{
+ const struct odb_helper_object *a = va, *b = vb;
+ return hashcmp(a->sha1, b->sha1);
+}
+
+static void odb_helper_load_have(struct odb_helper *o)
+{
+ struct odb_helper_cmd cmd;
+ FILE *fh;
+ struct strbuf line = STRBUF_INIT;
+
+ if (o->have_valid)
+ return;
+ o->have_valid = 1;
+
+ if (odb_helper_start(o, &cmd, "have") < 0)
+ return;
+
+ fh = xfdopen(cmd.child.out, "r");
+ while (strbuf_getline(&line, fh) != EOF) {
+ ALLOC_GROW(o->have, o->have_nr+1, o->have_alloc);
+ if (parse_object_line(&o->have[o->have_nr], line.buf) < 0) {
+ warning("bad 'have' input from odb helper '%s': %s",
+ o->name, line.buf);
+ break;
+ }
+ o->have_nr++;
+ }
+
+ strbuf_release(&line);
+ fclose(fh);
+ odb_helper_finish(o, &cmd);
+
+ qsort(o->have, o->have_nr, sizeof(*o->have), odb_helper_object_cmp);
+}
+
+static struct odb_helper_object *odb_helper_lookup(struct odb_helper *o,
+ const unsigned char *sha1)
+{
+ int idx;
+
+ odb_helper_load_have(o);
+ idx = sha1_entry_pos(o->have, sizeof(*o->have), 0,
+ 0, o->have_nr, o->have_nr,
+ sha1);
+ if (idx < 0)
+ return NULL;
+ return &o->have[idx];
+}
+
+int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1)
+{
+ return !!odb_helper_lookup(o, sha1);
+}
+
+int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
+ int fd)
+{
+ struct odb_helper_object *obj;
+ struct odb_helper_cmd cmd;
+ unsigned long total_got;
+ git_zstream stream;
+ int zret;
+ git_SHA_CTX hash;
+ unsigned char real_sha1[20];
+
+ obj = odb_helper_lookup(o, sha1);
+ if (!obj)
+ return -1;
+
+ if (odb_helper_start(o, &cmd, "get %s", sha1_to_hex(sha1)) < 0)
+ return -1;
+
+ memset(&stream, 0, sizeof(stream));
+ git_inflate_init(&stream);
+ git_SHA1_Init(&hash);
+ total_got = 0;
+
+ for (;;) {
+ unsigned char buf[4096];
+ int r;
+
+ r = xread(cmd.child.out, buf, sizeof(buf));
+ if (r < 0) {
+ error("unable to read from odb helper '%s': %s",
+ o->name, strerror(errno));
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ git_inflate_end(&stream);
+ return -1;
+ }
+ if (r == 0)
+ break;
+
+ write_or_die(fd, buf, r);
+
+ stream.next_in = buf;
+ stream.avail_in = r;
+ do {
+ unsigned char inflated[4096];
+ unsigned long got;
+
+ stream.next_out = inflated;
+ stream.avail_out = sizeof(inflated);
+ zret = git_inflate(&stream, Z_SYNC_FLUSH);
+ got = sizeof(inflated) - stream.avail_out;
+
+ git_SHA1_Update(&hash, inflated, got);
+ /* skip header when counting size */
+ if (!total_got) {
+ const unsigned char *p = memchr(inflated, '\0', got);
+ if (p)
+ got -= p - inflated + 1;
+ else
+ got = 0;
+ }
+ total_got += got;
+ } while (stream.avail_in && zret == Z_OK);
+ }
+
+ close(cmd.child.out);
+ git_inflate_end(&stream);
+ git_SHA1_Final(real_sha1, &hash);
+ if (odb_helper_finish(o, &cmd))
+ return -1;
+ if (zret != Z_STREAM_END) {
+ warning("bad zlib data from odb helper '%s' for %s",
+ o->name, sha1_to_hex(sha1));
+ return -1;
+ }
+ if (total_got != obj->size) {
+ warning("size mismatch from odb helper '%s' for %s (%lu != %lu)",
+ o->name, sha1_to_hex(sha1), total_got, obj->size);
+ return -1;
+ }
+ if (hashcmp(real_sha1, sha1)) {
+ warning("sha1 mismatch from odb helper '%s' for %s (got %s)",
+ o->name, sha1_to_hex(sha1), sha1_to_hex(real_sha1));
+ return -1;
+ }
+
+ return 0;
+}
diff --git a/odb-helper.h b/odb-helper.h
new file mode 100644
index 0000000..0f704f9
--- /dev/null
+++ b/odb-helper.h
@@ -0,0 +1,25 @@
+#ifndef ODB_HELPER_H
+#define ODB_HELPER_H
+
+struct odb_helper {
+ const char *name;
+ const char *cmd;
+
+ struct odb_helper_object {
+ unsigned char sha1[20];
+ unsigned long size;
+ enum object_type type;
+ } *have;
+ int have_nr;
+ int have_alloc;
+ int have_valid;
+
+ struct odb_helper *next;
+};
+
+struct odb_helper *odb_helper_new(const char *name, int namelen);
+int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1);
+int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
+ int fd);
+
+#endif /* ODB_HELPER_H */
diff --git a/sha1_file.c b/sha1_file.c
index d5e1121..a707bc1 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -23,6 +23,7 @@
#include "bulk-checkin.h"
#include "streaming.h"
#include "dir.h"
+#include "external-odb.h"
#ifndef O_NOATIME
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
@@ -183,13 +184,11 @@ static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
}
}
-const char *sha1_file_name(const unsigned char *sha1)
+const char *sha1_file_name_alt(const char *objdir, const unsigned char *sha1)
{
static char buf[PATH_MAX];
- const char *objdir;
int len;
- objdir = get_object_directory();
len = strlen(objdir);
/* '/' + sha1(2) + '/' + sha1(38) + '\0' */
@@ -203,6 +202,11 @@ const char *sha1_file_name(const unsigned char *sha1)
return buf;
}
+const char *sha1_file_name(const unsigned char *sha1)
+{
+ return sha1_file_name_alt(get_object_directory(), sha1);
+}
+
/*
* Return the name of the pack or index file with the specified sha1
* in its filename. *base and *name are scratch space that must be
@@ -439,6 +443,21 @@ int foreach_alt_odb(alt_odb_fn fn, void *cb)
return r;
}
+void prepare_external_alt_odb(void)
+{
+ static int linked_external;
+ const char *path;
+
+ if (linked_external)
+ return;
+
+ path = external_odb_root();
+ if (!access(path, F_OK)) {
+ link_alt_odb_entry(path, NULL, 0, "");
+ linked_external = 1;
+ }
+}
+
void prepare_alt_odb(void)
{
const char *alt;
@@ -453,6 +472,7 @@ void prepare_alt_odb(void)
link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
read_info_alternates(get_object_directory(), 0);
+ prepare_external_alt_odb();
}
/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
@@ -493,7 +513,7 @@ static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
if (check_and_freshen_file(alt->base, freshen))
return 1;
}
- return 0;
+ return external_odb_has_object(sha1);
}
static int check_and_freshen(const unsigned char *sha1, int freshen)
@@ -1523,19 +1543,15 @@ static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
return -1;
}
-static int open_sha1_file(const unsigned char *sha1)
+static int open_sha1_file_alt(const unsigned char *sha1)
{
- int fd;
struct alternate_object_database *alt;
- int most_interesting_errno;
-
- fd = git_open_noatime(sha1_file_name(sha1));
- if (fd >= 0)
- return fd;
- most_interesting_errno = errno;
+ int most_interesting_errno = errno;
prepare_alt_odb();
for (alt = alt_odb_list; alt; alt = alt->next) {
+ int fd;
+
fill_sha1_path(alt->name, sha1);
fd = git_open_noatime(alt->base);
if (fd >= 0)
@@ -1547,6 +1563,24 @@ static int open_sha1_file(const unsigned char *sha1)
return -1;
}
+static int open_sha1_file(const unsigned char *sha1)
+{
+ int fd;
+
+ fd = git_open_noatime(sha1_file_name(sha1));
+ if (fd >= 0)
+ return fd;
+
+ fd = open_sha1_file_alt(sha1);
+ if (fd >= 0)
+ return fd;
+
+ if (!external_odb_fetch_object(sha1))
+ fd = open_sha1_file_alt(sha1);
+
+ return fd;
+}
+
void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
{
void *map;
@@ -3011,7 +3045,7 @@ int hash_sha1_file(const void *buf, unsigned long len, const char *type,
}
/* Finalize a file on disk, and close it. */
-static void close_sha1_file(int fd)
+void close_sha1_file(int fd)
{
if (fsync_object_files)
fsync_or_die(fd, "sha1 file");
@@ -3035,7 +3069,7 @@ static inline int directory_size(const char *filename)
* We want to avoid cross-directory filename renames, because those
* can have problems on various filesystems (FAT, NFS, Coda).
*/
-static int create_tmpfile(struct strbuf *tmp, const char *filename)
+int create_object_tmpfile(struct strbuf *tmp, const char *filename)
{
int fd, dirlen = directory_size(filename);
@@ -3075,7 +3109,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
static struct strbuf tmp_file = STRBUF_INIT;
const char *filename = sha1_file_name(sha1);
- fd = create_tmpfile(&tmp_file, filename);
+ fd = create_object_tmpfile(&tmp_file, filename);
if (fd < 0) {
if (errno == EACCES)
return error("insufficient permission for adding an object to repository database %s", get_object_directory());
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
new file mode 100755
index 0000000..2b01617
--- /dev/null
+++ b/t/t0400-external-odb.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+test_description='basic tests for external object databases'
+
+. ./test-lib.sh
+
+ALT_SOURCE="$PWD/alt-repo/.git"
+export ALT_SOURCE
+write_script odb-helper <<\EOF
+GIT_DIR=$ALT_SOURCE; export GIT_DIR
+case "$1" in
+have)
+ git rev-list --all --objects |
+ cut -d' ' -f1 |
+ git cat-file --batch-check |
+ awk '{print $1 " " $3 " " $2}'
+ ;;
+get)
+ cat "$GIT_DIR"/objects/$(echo $2 | sed 's#..#&/#')
+ ;;
+esac
+EOF
+HELPER="\"$PWD\"/odb-helper"
+
+test_expect_success 'setup alternate repo' '
+ git init alt-repo &&
+ (cd alt-repo &&
+ test_commit one &&
+ test_commit two
+ ) &&
+ alt_head=`cd alt-repo && git rev-parse HEAD`
+'
+
+test_expect_success 'alt objects are missing' '
+ test_must_fail git log --format=%s $alt_head
+'
+
+test_expect_success 'helper can retrieve alt objects' '
+ test_config odb.magic.command "$HELPER" &&
+ cat >expect <<-\EOF &&
+ two
+ one
+ EOF
+ git log --format=%s $alt_head >actual &&
+ test_cmp expect actual
+'
+
+test_done
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 2/8] external odb foreach
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 1/8] Add initial external odb support Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 3/8] t0400: use --batch-all-objects to get all objects Christian Couder
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git; +Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey
From: Jeff King <peff@peff.net>
---
external-odb.c | 14 ++++++++++++++
external-odb.h | 6 ++++++
odb-helper.c | 15 +++++++++++++++
odb-helper.h | 4 ++++
4 files changed, 39 insertions(+)
diff --git a/external-odb.c b/external-odb.c
index 1ccfa99..42978a3 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -113,3 +113,17 @@ int external_odb_fetch_object(const unsigned char *sha1)
return -1;
}
+
+int external_odb_for_each_object(each_external_object_fn fn, void *data)
+{
+ struct odb_helper *o;
+
+ external_odb_init();
+
+ for (o = helpers; o; o = o->next) {
+ int r = odb_helper_for_each_object(o, fn, data);
+ if (r)
+ return r;
+ }
+ return 0;
+}
diff --git a/external-odb.h b/external-odb.h
index 2397477..cea8570 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -5,4 +5,10 @@ const char *external_odb_root(void);
int external_odb_has_object(const unsigned char *sha1);
int external_odb_fetch_object(const unsigned char *sha1);
+typedef int (*each_external_object_fn)(const unsigned char *sha1,
+ enum object_type type,
+ unsigned long size,
+ void *data);
+int external_odb_for_each_object(each_external_object_fn, void *);
+
#endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
index 7029a59..045cf6f 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -235,5 +235,20 @@ int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
return -1;
}
+ return 0;
+}
+
+int odb_helper_for_each_object(struct odb_helper *o,
+ each_external_object_fn fn,
+ void *data)
+{
+ int i;
+ for (i = 0; i < o->have_nr; i++) {
+ struct odb_helper_object *obj = &o->have[i];
+ int r = fn(obj->sha1, obj->type, obj->size, data);
+ if (r)
+ return r;
+ }
+
return 0;
}
diff --git a/odb-helper.h b/odb-helper.h
index 0f704f9..8c3916d 100644
--- a/odb-helper.h
+++ b/odb-helper.h
@@ -1,6 +1,8 @@
#ifndef ODB_HELPER_H
#define ODB_HELPER_H
+#include "external-odb.h"
+
struct odb_helper {
const char *name;
const char *cmd;
@@ -21,5 +23,7 @@ struct odb_helper *odb_helper_new(const char *name, int namelen);
int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1);
int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
int fd);
+int odb_helper_for_each_object(struct odb_helper *o,
+ each_external_object_fn, void *);
#endif /* ODB_HELPER_H */
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 3/8] t0400: use --batch-all-objects to get all objects
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 1/8] Add initial external odb support Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 2/8] external odb foreach Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 4/8] t0400: add 'put' command to odb-helper script Christian Couder
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index 2b01617..fe85413 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -10,9 +10,7 @@ write_script odb-helper <<\EOF
GIT_DIR=$ALT_SOURCE; export GIT_DIR
case "$1" in
have)
- git rev-list --all --objects |
- cut -d' ' -f1 |
- git cat-file --batch-check |
+ git cat-file --batch-check --batch-all-objects |
awk '{print $1 " " $3 " " $2}'
;;
get)
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 4/8] t0400: add 'put' command to odb-helper script
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
` (2 preceding siblings ...)
2016-06-13 8:55 ` [RFC/PATCH 3/8] t0400: use --batch-all-objects to get all objects Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 5/8] t0400: add test for 'put' command Christian Couder
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index fe85413..0f1bb97 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -7,6 +7,10 @@ test_description='basic tests for external object databases'
ALT_SOURCE="$PWD/alt-repo/.git"
export ALT_SOURCE
write_script odb-helper <<\EOF
+die() {
+ printf >&2 "%s\n" "$@"
+ exit 1
+}
GIT_DIR=$ALT_SOURCE; export GIT_DIR
case "$1" in
have)
@@ -16,6 +20,16 @@ have)
get)
cat "$GIT_DIR"/objects/$(echo $2 | sed 's#..#&/#')
;;
+put)
+ sha1="$2"
+ size="$3"
+ kind="$4"
+ writen=$(git hash-object -w -t "$kind" --stdin)
+ test "$writen" = "$sha1" || die "bad sha1 passed '$sha1' vs writen '$writen'"
+ ;;
+*)
+ die "unknown command '$1'"
+ ;;
esac
EOF
HELPER="\"$PWD\"/odb-helper"
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 5/8] t0400: add test for 'put' command
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
` (3 preceding siblings ...)
2016-06-13 8:55 ` [RFC/PATCH 4/8] t0400: add 'put' command to odb-helper script Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 6/8] external odb: add write support Christian Couder
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index 0f1bb97..6c6da5c 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -57,4 +57,13 @@ test_expect_success 'helper can retrieve alt objects' '
test_cmp expect actual
'
+test_expect_success 'helper can add objects to alt repo' '
+ hash=$(echo "Hello odb!" | git hash-object -w -t blob --stdin) &&
+ test -f .git/objects/$(echo $hash | sed "s#..#&/#") &&
+ size=$(git cat-file -s "$hash") &&
+ git cat-file blob "$hash" | ./odb-helper put "$hash" "$size" blob &&
+ alt_size=$(cd alt-repo && git cat-file -s "$hash") &&
+ test "$size" -eq "$alt_size"
+'
+
test_done
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 6/8] external odb: add write support
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
` (4 preceding siblings ...)
2016-06-13 8:55 ` [RFC/PATCH 5/8] t0400: add test for 'put' command Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 7/8] external-odb: accept only blobs for now Christian Couder
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
external-odb.c | 15 +++++++++++++++
external-odb.h | 2 ++
odb-helper.c | 41 +++++++++++++++++++++++++++++++++++++----
odb-helper.h | 3 +++
sha1_file.c | 2 ++
5 files changed, 59 insertions(+), 4 deletions(-)
diff --git a/external-odb.c b/external-odb.c
index 42978a3..bb70fe3 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -127,3 +127,18 @@ int external_odb_for_each_object(each_external_object_fn fn, void *data)
}
return 0;
}
+
+int external_odb_write_object(const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1)
+{
+ struct odb_helper *o;
+
+ external_odb_init();
+
+ for (o = helpers; o; o = o->next) {
+ int r = odb_helper_write_object(o, buf, len, type, sha1);
+ if (r <= 0)
+ return r;
+ }
+ return 1;
+}
diff --git a/external-odb.h b/external-odb.h
index cea8570..55d291d 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -10,5 +10,7 @@ typedef int (*each_external_object_fn)(const unsigned char *sha1,
unsigned long size,
void *data);
int external_odb_for_each_object(each_external_object_fn, void *);
+int external_odb_write_object(const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1);
#endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
index 045cf6f..677a5e7 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -33,9 +33,10 @@ static void prepare_helper_command(struct argv_array *argv, const char *cmd,
strbuf_release(&buf);
}
-__attribute__((format (printf,3,4)))
+__attribute__((format (printf,4,5)))
static int odb_helper_start(struct odb_helper *o,
struct odb_helper_cmd *cmd,
+ int use_stdin,
const char *fmt, ...)
{
va_list ap;
@@ -52,7 +53,10 @@ static int odb_helper_start(struct odb_helper *o,
cmd->child.argv = cmd->argv.argv;
cmd->child.use_shell = 1;
- cmd->child.no_stdin = 1;
+ if (use_stdin)
+ cmd->child.in = -1;
+ else
+ cmd->child.no_stdin = 1;
cmd->child.out = -1;
if (start_command(&cmd->child) < 0) {
@@ -109,7 +113,7 @@ static void odb_helper_load_have(struct odb_helper *o)
return;
o->have_valid = 1;
- if (odb_helper_start(o, &cmd, "have") < 0)
+ if (odb_helper_start(o, &cmd, 0, "have") < 0)
return;
fh = xfdopen(cmd.child.out, "r");
@@ -164,7 +168,7 @@ int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
if (!obj)
return -1;
- if (odb_helper_start(o, &cmd, "get %s", sha1_to_hex(sha1)) < 0)
+ if (odb_helper_start(o, &cmd, 0, "get %s", sha1_to_hex(sha1)) < 0)
return -1;
memset(&stream, 0, sizeof(stream));
@@ -252,3 +256,32 @@ int odb_helper_for_each_object(struct odb_helper *o,
return 0;
}
+
+int odb_helper_write_object(struct odb_helper *o,
+ const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1)
+{
+ struct odb_helper_cmd cmd;
+
+ if (odb_helper_start(o, &cmd, 1, "put %s %lu %s",
+ sha1_to_hex(sha1), len, type) < 0)
+ return -1;
+
+ do {
+ int w = xwrite(cmd.child.in, buf, len);
+ if (w < 0) {
+ error("unable to write to odb helper '%s': %s",
+ o->name, strerror(errno));
+ close(cmd.child.in);
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ return -1;
+ }
+ len -= w;
+ } while (len > 0);
+
+ close(cmd.child.in);
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ return 0;
+}
diff --git a/odb-helper.h b/odb-helper.h
index 8c3916d..af31cc2 100644
--- a/odb-helper.h
+++ b/odb-helper.h
@@ -25,5 +25,8 @@ int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
int fd);
int odb_helper_for_each_object(struct odb_helper *o,
each_external_object_fn, void *);
+int odb_helper_write_object(struct odb_helper *o,
+ const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1);
#endif /* ODB_HELPER_H */
diff --git a/sha1_file.c b/sha1_file.c
index a707bc1..90f19de 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -3192,6 +3192,8 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign
* it out into .git/objects/??/?{38} file.
*/
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
+ if (!external_odb_write_object(buf, len, type, sha1))
+ return 0;
if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
return 0;
return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 7/8] external-odb: accept only blobs for now
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
` (5 preceding siblings ...)
2016-06-13 8:55 ` [RFC/PATCH 6/8] external odb: add write support Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 8:55 ` [RFC/PATCH 8/8] t0400: add test for external odb write support Christian Couder
2016-06-13 10:10 ` [RFC/PATCH 0/8] Add initial experimental external ODB support Duy Nguyen
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
external-odb.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/external-odb.c b/external-odb.c
index bb70fe3..6dd7b25 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -133,6 +133,10 @@ int external_odb_write_object(const void *buf, unsigned long len,
{
struct odb_helper *o;
+ /* For now accept only blobs */
+ if (strcmp(type, "blob"))
+ return 1;
+
external_odb_init();
for (o = helpers; o; o = o->next) {
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [RFC/PATCH 8/8] t0400: add test for external odb write support
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
` (6 preceding siblings ...)
2016-06-13 8:55 ` [RFC/PATCH 7/8] external-odb: accept only blobs for now Christian Couder
@ 2016-06-13 8:55 ` Christian Couder
2016-06-13 10:10 ` [RFC/PATCH 0/8] Add initial experimental external ODB support Duy Nguyen
8 siblings, 0 replies; 10+ messages in thread
From: Christian Couder @ 2016-06-13 8:55 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index 6c6da5c..3c868ca 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -66,4 +66,12 @@ test_expect_success 'helper can add objects to alt repo' '
test "$size" -eq "$alt_size"
'
+test_expect_success 'commit adds objects to alt repo' '
+ test_config odb.magic.command "$HELPER" &&
+ test_commit three &&
+ hash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\ -f3) &&
+ content=$(cd alt-repo && git show "$hash3") &&
+ test "$content" = "three"
+'
+
test_done
--
2.9.0.rc2.362.g3cd93d0
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [RFC/PATCH 0/8] Add initial experimental external ODB support
2016-06-13 8:55 [RFC/PATCH 0/8] Add initial experimental external ODB support Christian Couder
` (7 preceding siblings ...)
2016-06-13 8:55 ` [RFC/PATCH 8/8] t0400: add test for external odb write support Christian Couder
@ 2016-06-13 10:10 ` Duy Nguyen
8 siblings, 0 replies; 10+ messages in thread
From: Duy Nguyen @ 2016-06-13 10:10 UTC (permalink / raw)
To: Christian Couder
Cc: Git Mailing List, Junio C Hamano, Jeff King, Mike Hommey,
Christian Couder
On Mon, Jun 13, 2016 at 3:55 PM, Christian Couder
<christian.couder@gmail.com> wrote:
> Future work
> ~~~~~~~~~~~
>
> From the discussions it appear that using the bundle v3 mechanism to
> tranfer external ODB data could work, but only if the server has access
> to its own external ODB.
>
> Another possible mechanism to transfer external ODB data would be some
> kind of replace refs. This would be slower but the mechanism for the
> transfer already fully exists.
>
> So I think I am going to experiment with some kind of replace refs.
Or we can go "fault-in" all the way.
1) Extend rev-list to support large blob exclusion, e.g. blobs whose
size above a certain limit are excluded, or blobs in certain paths are
excluded (maybe except those at the tip)
2) Add a new command, similar to "shallow", in the git protocol for
the client to tell the server that it does not want the server to send
certain large blobs (and the server probably should acknowledge with
"ok you can get them from this url...")
3) upload-pack passes the relevant rev-list blob filtering arguments
to pack-objects, no big blobs are transferred
4) External odb setting must be done (temporarily) before index-pack
is run (as part of the clone/fetch process). index-pack will notice
that certain blobs are missing from the received pack, but it will
also notice that it already has them via standard has_sha1_file() (we
just need to make sure it does not read blob content back, i think
we're ok here). I'm assuming here that EODB in fact contains _all_
blobs, so no extra setup is required at the server side.
5) If index-pack says "ok" then you can make this new external odb
permanent, else remove it.
Subsequent fetches need to send the same "do not include these blobs"
instructions, of course. Not sure how well it interacts with
pack-bitmap, but I guess when you use this, you probably already use
shallow clone (which disables pack bitmap).
> One interesting thing also would be to use the streaming api when
> reading from or writing to the external ODB.
If EODB is about large blobs only, you probably want to stream them
directly to a pack as you get them and not keep in loose object form.
Infrastructure is already in place (I think it's used by git-add), but
hooking it at sha1_file.c layer might be tricky. This is just cherry
on top, not strictly needed of course.
--
Duy
^ permalink raw reply [flat|nested] 10+ messages in thread