[PATCH v2 0/7] More work supporting objects larger than 4GB on Windows

["Johannes Schindelin via GitGitGadget" <gitgitgadget@gmail.com>]

This patch series tries to address the problems pointed out by the expensive
tests that now run in CI: t5608 and t7508 verify various aspects about
objects larger than 4GB, which Git does not currently handle correctly when
run on a platform where size_t is 64-bit and unsigned long is 32-bit.

Changes vs v1:

 * Rebased onto master, which merged ps/odb-source-loose (with which these
   patches previously conflicted rather badly).
 * Removed superfluous size_t s variables (thanks, Patrick!).

Johannes Schindelin (7):
  compat/msvc: use _chsize_s for ftruncate
  patch-delta: use size_t for sizes
  pack-objects(check_pack_inflate()): use size_t instead of unsigned
    long
  packfile: widen unpack_entry()'s size out-parameter to size_t
  pack-objects: use size_t for in-core object sizes
  packfile,delta: drop the `cast_size_t_to_ulong()` wrappers
  odb: use size_t for object_info.sizep and the size APIs

 apply.c                       |  8 ++--
 archive.c                     |  4 +-
 attr.c                        |  2 +-
 bisect.c                      |  2 +-
 blame.c                       | 15 +++++--
 builtin/cat-file.c            | 61 ++++++++++++++---------------
 builtin/difftool.c            |  2 +-
 builtin/fast-export.c         |  7 +++-
 builtin/fast-import.c         | 29 ++++++++++----
 builtin/fsck.c                |  2 +-
 builtin/grep.c                | 12 +++---
 builtin/index-pack.c          | 10 ++---
 builtin/log.c                 |  2 +-
 builtin/ls-files.c            |  2 +-
 builtin/ls-tree.c             |  4 +-
 builtin/merge-tree.c          |  6 +--
 builtin/mktag.c               |  2 +-
 builtin/notes.c               |  6 +--
 builtin/pack-objects.c        | 73 +++++++++++++++++++++--------------
 builtin/repo.c                |  4 +-
 builtin/tag.c                 |  4 +-
 builtin/unpack-file.c         |  2 +-
 builtin/unpack-objects.c      |  8 ++--
 bundle.c                      |  2 +-
 combine-diff.c                |  4 +-
 commit.c                      | 10 ++---
 compat/msvc-posix.h           | 24 +++++++++++-
 config.c                      |  2 +-
 delta.h                       | 20 +++-------
 diff.c                        |  5 ++-
 dir.c                         |  2 +-
 entry.c                       |  4 +-
 fmt-merge-msg.c               |  4 +-
 fsck.c                        |  2 +-
 grep.c                        |  4 +-
 http-push.c                   |  2 +-
 list-objects-filter.c         |  2 +-
 mailmap.c                     |  2 +-
 match-trees.c                 |  4 +-
 merge-blobs.c                 |  6 +--
 merge-blobs.h                 |  2 +-
 merge-ort.c                   |  2 +-
 notes-cache.c                 |  2 +-
 notes-merge.c                 |  2 +-
 notes.c                       |  8 ++--
 object-file.c                 |  6 +--
 object.c                      |  2 +-
 odb.c                         | 12 +++---
 odb.h                         | 10 ++---
 odb/source-loose.c            | 12 +-----
 odb/streaming.c               | 13 +------
 pack-bitmap.c                 |  4 +-
 pack-check.c                  |  5 +--
 pack-objects.h                |  2 +-
 packfile.c                    | 54 ++++++++++----------------
 packfile.h                    |  5 ++-
 patch-delta.c                 |  8 ++--
 path-walk.c                   |  2 +-
 protocol-caps.c               |  5 ++-
 read-cache.c                  |  6 +--
 ref-filter.c                  |  2 +-
 reflog.c                      |  2 +-
 rerere.c                      |  2 +-
 submodule-config.c            |  2 +-
 t/helper/test-delta.c         | 10 +++--
 t/helper/test-pack-deltas.c   |  3 +-
 t/helper/test-partial-clone.c |  2 +-
 t/unit-tests/u-odb-inmemory.c |  2 +-
 tag.c                         |  4 +-
 tree-walk.c                   | 10 +++--
 tree.c                        |  2 +-
 xdiff-interface.c             |  2 +-
 72 files changed, 300 insertions(+), 271 deletions(-)


base-commit: ea97ad8d017de0c9037451a78008a0fd60abea0c
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-2137%2Fdscho%2Fobjects-larger-than-4gb-on-windows-pt2-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-2137/dscho/objects-larger-than-4gb-on-windows-pt2-v2
Pull-Request: https://github.com/gitgitgadget/git/pull/2137

Range-diff vs v1:

 1:  de9fc5c455 = 1:  531bca775c compat/msvc: use _chsize_s for ftruncate
 2:  1fd7646ca1 = 2:  66a642c39e patch-delta: use size_t for sizes
 3:  ddb75326cd = 3:  271a5299e3 pack-objects(check_pack_inflate()): use size_t instead of unsigned long
 4:  bdebc36f21 = 4:  5c329535df packfile: widen unpack_entry()'s size out-parameter to size_t
 5:  68750ba2d1 = 5:  01b9209b26 pack-objects: use size_t for in-core object sizes
 6:  460d733fee = 6:  12c142f8ab packfile,delta: drop the `cast_size_t_to_ulong()` wrappers
 7:  f3aeae983a ! 7:  37d030d867 odb: use size_t for object_info.sizep and the size APIs
     @@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const
       	struct object_info oi = OBJECT_INFO_INIT;
       	unsigned flags = OBJECT_INFO_LOOKUP_REPLACE;
      @@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
     - 		if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG)) {
     - 			size_t s = size;
     - 			buf = replace_idents_using_mailmap(buf, &s);
     + 		if (odb_read_object_info_extended(the_repository->objects, &oid, &oi, flags) < 0)
     + 			die("git cat-file: could not get object info");
     + 
     +-		if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG)) {
     +-			size_t s = size;
     +-			buf = replace_idents_using_mailmap(buf, &s);
      -			size = cast_size_t_to_ulong(s);
     -+			size = s;
     - 		}
     +-		}
     ++		if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG))
     ++			buf = replace_idents_using_mailmap(buf, &size);
       
       		printf("%"PRIuMAX"\n", (uintmax_t)size);
     + 		ret = 0;
      @@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
       		break;
       
     @@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const
       
       	case 'p':
      @@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
     - 		if (use_mailmap) {
     - 			size_t s = size;
     - 			buf = replace_idents_using_mailmap(buf, &s);
     + 		if (!buf)
     + 			die("Cannot read object %s", obj_name);
     + 
     +-		if (use_mailmap) {
     +-			size_t s = size;
     +-			buf = replace_idents_using_mailmap(buf, &s);
      -			size = cast_size_t_to_ulong(s);
     -+			size = s;
     - 		}
     +-		}
     ++		if (use_mailmap)
     ++			buf = replace_idents_using_mailmap(buf, &size);
       
       		/* otherwise just spit out the data */
     + 		break;
      @@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
     - 		if (use_mailmap) {
     - 			size_t s = size;
     - 			buf = replace_idents_using_mailmap(buf, &s);
     + 		buf = odb_read_object_peeled(the_repository->objects, &oid,
     + 					     exp_type_id, &size, NULL);
     + 
     +-		if (use_mailmap) {
     +-			size_t s = size;
     +-			buf = replace_idents_using_mailmap(buf, &s);
      -			size = cast_size_t_to_ulong(s);
     -+			size = s;
     - 		}
     +-		}
     ++		if (use_mailmap)
     ++			buf = replace_idents_using_mailmap(buf, &size);
       		break;
       	}
     + 	default:
      @@ builtin/cat-file.c: cleanup:
       struct expand_data {
       	struct object_id oid;
     @@ builtin/cat-file.c: static void print_object_or_die(struct batch_options *opt, s
       
       		contents = odb_read_object(the_repository->objects, oid,
      @@ builtin/cat-file.c: static void print_object_or_die(struct batch_options *opt, struct expand_data *d
     - 		if (use_mailmap) {
     - 			size_t s = size;
     - 			contents = replace_idents_using_mailmap(contents, &s);
     + 		if (!contents)
     + 			die("object %s disappeared", oid_to_hex(oid));
     + 
     +-		if (use_mailmap) {
     +-			size_t s = size;
     +-			contents = replace_idents_using_mailmap(contents, &s);
      -			size = cast_size_t_to_ulong(s);
     -+			size = s;
     - 		}
     +-		}
     ++		if (use_mailmap)
     ++			contents = replace_idents_using_mailmap(contents, &size);
       
       		if (type != data->type)
     + 			die("object %s changed type!?", oid_to_hex(oid));
      @@ builtin/cat-file.c: static void batch_object_write(const char *obj_name,
     + 		}
     + 
     + 		if (use_mailmap && (data->type == OBJ_COMMIT || data->type == OBJ_TAG)) {
     +-			size_t s = data->size;
     + 			char *buf = NULL;
     + 
     + 			buf = odb_read_object(the_repository->objects, &data->oid,
     + 					      &data->type, &data->size);
       			if (!buf)
       				die(_("unable to read %s"), oid_to_hex(&data->oid));
     - 			buf = replace_idents_using_mailmap(buf, &s);
     +-			buf = replace_idents_using_mailmap(buf, &s);
      -			data->size = cast_size_t_to_ulong(s);
     -+			data->size = s;
     ++			buf = replace_idents_using_mailmap(buf, &data->size);
       
       			free(buf);
       		}
     @@ builtin/log.c: static int show_blob_object(const struct object_id *oid, struct r
      
       ## builtin/ls-files.c ##
      @@ builtin/ls-files.c: static void expand_objectsize(struct repository *repo, struct strbuf *line,
     - 			      const enum object_type type, unsigned int padded)
     - {
     + 	size_t len;
     + 
       	if (type == OBJ_BLOB) {
      -		unsigned long size;
      +		size_t size;
     @@ builtin/ls-files.c: static void expand_objectsize(struct repository *repo, struc
      
       ## builtin/ls-tree.c ##
      @@ builtin/ls-tree.c: static void expand_objectsize(struct strbuf *line, const struct object_id *oid,
     - 			      const enum object_type type, unsigned int padded)
     - {
     + 	size_t len;
     + 
       	if (type == OBJ_BLOB) {
      -		unsigned long size;
      +		size_t size;
     @@ notes.c: static void format_note(struct notes_tree *t, const struct object_id *o
       	if (!t)
      
       ## object-file.c ##
     -@@ object-file.c: static int parse_loose_header(const char *hdr, struct object_info *oi)
     +@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi)
       	}
       
       	if (oi->sizep)
     @@ object-file.c: static int parse_loose_header(const char *hdr, struct object_info
       
       	/*
       	 * The length must be followed by a zero byte
     -@@ object-file.c: static int read_object_info_from_path(struct odb_source *source,
     - 	void *map = NULL;
     - 	git_zstream stream, *stream_to_end = NULL;
     - 	char hdr[MAX_HEADER_LEN];
     --	unsigned long size_scratch;
     -+	size_t size_scratch;
     - 	enum object_type type_scratch;
     - 	struct stat st;
     - 
      @@ object-file.c: int force_object_loose(struct odb_source *source,
     - {
     + 	struct odb_source_files *files = odb_source_files_downcast(source);
       	const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
       	void *buf;
      -	unsigned long len;
     @@ object-file.c: int read_loose_object(struct repository *repo,
       
       	fd = git_open(path);
       	if (fd >= 0)
     -@@ object-file.c: int odb_source_loose_read_object_stream(struct odb_read_stream **out,
     - 	struct object_info oi = OBJECT_INFO_INIT;
     - 	struct odb_loose_read_stream *st;
     - 	unsigned long mapsize;
     --	unsigned long size_ul;
     - 	void *mapped;
     - 
     - 	mapped = odb_source_loose_map_object(source, oid, &mapsize);
     -@@ object-file.c: int odb_source_loose_read_object_stream(struct odb_read_stream **out,
     - 		goto error;
     - 	}
     - 
     --	/*
     --	 * object_info.sizep is unsigned long* (32-bit on Windows), but
     --	 * st->base.size is size_t (64-bit). Use temporary variable.
     --	 * Note: loose objects >4GB would still truncate here, but such
     --	 * large loose objects are uncommon (they'd normally be packed).
     --	 */
     --	oi.sizep = &size_ul;
     -+	oi.sizep = &st->base.size;
     - 	oi.typep = &st->base.type;
     - 
     - 	if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
     - 		goto error;
     --	st->base.size = size_ul;
     - 
     - 	st->mapped = mapped;
     - 	st->mapsize = mapsize;
      
       ## object.c ##
      @@ object.c: struct object *parse_object_with_flags(struct repository *r,
     @@ odb.h: int odb_read_object_info_extended(struct object_database *odb,
       enum odb_has_object_flags {
       	/* Retry packed storage after checking packed and loose storage */
      
     + ## odb/source-loose.c ##
     +@@ odb/source-loose.c: static int read_object_info_from_path(struct odb_source_loose *loose,
     + 	void *map = NULL;
     + 	git_zstream stream, *stream_to_end = NULL;
     + 	char hdr[MAX_HEADER_LEN];
     +-	unsigned long size_scratch;
     ++	size_t size_scratch;
     + 	enum object_type type_scratch;
     + 	struct stat st;
     + 
     +@@ odb/source-loose.c: static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
     + 	struct object_info oi = OBJECT_INFO_INIT;
     + 	struct odb_loose_read_stream *st;
     + 	unsigned long mapsize;
     +-	unsigned long size_ul;
     + 	void *mapped;
     + 
     + 	mapped = odb_source_loose_map_object(loose, oid, &mapsize);
     +@@ odb/source-loose.c: static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
     + 		goto error;
     + 	}
     + 
     +-	/*
     +-	 * object_info.sizep is unsigned long* (32-bit on Windows), but
     +-	 * st->base.size is size_t (64-bit). Use temporary variable.
     +-	 * Note: loose objects >4GB would still truncate here, but such
     +-	 * large loose objects are uncommon (they'd normally be packed).
     +-	 */
     +-	oi.sizep = &size_ul;
     ++	oi.sizep = &st->base.size;
     + 	oi.typep = &st->base.type;
     + 
     + 	if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
     + 		goto error;
     +-	st->base.size = size_ul;
     + 
     + 	st->mapped = mapped;
     + 	st->mapsize = mapsize;
     +
       ## odb/streaming.c ##
      @@ odb/streaming.c: static int open_istream_incore(struct odb_read_stream **out,
       		.base.read = read_istream_incore,

-- 
gitgitgadget