[PATCH v4 0/5] Reftable support git-core
- From
Han-Wen Nienhuys via GitGitGadget <gitgitgadget@gmail.com>
- Date
- Feb 6, 2020, 22:55 UTC
- Message-ID
- <pull.539.v4.git.1581029756.gitgitgadget@gmail.com>
- In-Reply-To
- <pull.539.v3.git.1580848060.gitgitgadget@gmail.com>
This adds the reftable library, and hooks it up as a ref backend.
At this point, I am mainly interested in feedback on the spots marked with XXX in the Git source code.
v3
* passes gitgitgadget CI.
Han-Wen Nienhuys (5): refs.h: clarify reflog iteration order create .git/refs in files-backend.c refs: document how ref_iterator_advance_fn should handle symrefs Add reftable library Reftable support for git-core
Makefile | 24 +- builtin/init-db.c | 49 +- cache.h | 2 + refs.c | 20 +- refs.h | 5 +- refs/files-backend.c | 6 + refs/refs-internal.h | 6 + refs/reftable-backend.c | 880 +++++++++++++++++++++++++++++++ reftable/LICENSE | 31 ++ reftable/README.md | 19 + reftable/VERSION | 5 + reftable/basics.c | 196 +++++++ reftable/basics.h | 37 ++ reftable/block.c | 401 ++++++++++++++ reftable/block.h | 71 +++ reftable/blocksource.h | 20 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 27 + reftable/dump.c | 97 ++++ reftable/file.c | 97 ++++ reftable/iter.c | 229 ++++++++ reftable/iter.h | 56 ++ reftable/merged.c | 286 ++++++++++ reftable/merged.h | 34 ++ reftable/pq.c | 114 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 708 +++++++++++++++++++++++++ reftable/reader.h | 52 ++ reftable/record.c | 1107 +++++++++++++++++++++++++++++++++++++++ reftable/record.h | 79 +++ reftable/reftable.h | 399 ++++++++++++++ reftable/slice.c | 199 +++++++ reftable/slice.h | 39 ++ reftable/stack.c | 983 ++++++++++++++++++++++++++++++++++ reftable/stack.h | 40 ++ reftable/system.h | 55 ++ reftable/tree.c | 66 +++ reftable/tree.h | 24 + reftable/writer.c | 623 ++++++++++++++++++++++ reftable/writer.h | 46 ++ reftable/zlib-compat.c | 92 ++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- 45 files changed, 7248 insertions(+), 28 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
base-commit: 5b0ca878e008e82f91300091e793427205ce3544 Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v4 Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v4 Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v3:
1: c00403c94d = 1: c00403c94d refs.h: clarify reflog iteration order
2: 57c7342319 < -: ---------- setup.c: enable repo detection for reftable
3: 5b7060cb2f ! 2: 4d6da9bc47 create .git/refs in files-backend.c
@@ -29,9 +29,13 @@
+ files_ref_path(refs, &sb, "refs");
+ safe_create_dir(sb.buf, 1);
-+ /* adjust permissions even if directory already exists. */
++ /* adjust permissions even if directory already exists. */
+ adjust_shared_perm(sb.buf);
+
/*
* Create .git/refs/{heads,tags}
*/
++ strbuf_reset(&sb);
+ files_ref_path(refs, &sb, "refs/heads");
+ safe_create_dir(sb.buf, 1);
+
4: 1b01c735a9 = 3: fbdcdccc88 refs: document how ref_iterator_advance_fn should handle symrefs
5: eb0df10068 ! 4: 02d2ca8b87 Add reftable library
@@ -98,11 +98,11 @@
--- /dev/null
+++ b/reftable/VERSION
@@
-+commit e54326f73d95bfe8b17f264c400f4c365dbd5e5e
++commit e7c3fc3099d9999bc8d895f84027b0e36348d5e6
+Author: Han-Wen Nienhuys <hanwen@google.com>
-+Date: Tue Feb 4 19:48:17 2020 +0100
++Date: Thu Feb 6 20:17:40 2020 +0100
+
-+ C: PRI?MAX use; clang-format.
++ C: use inttypes.h header definitions
diff --git a/reftable/basics.c b/reftable/basics.c
new file mode 100644
@@ -2882,7 +2882,7 @@
+{
+ char hex[SHA256_SIZE + 1] = {};
+
-+ printf("ref{%s(%" PRIdMAX ") ", ref->ref_name, ref->update_index);
++ printf("ref{%s(%" PRIu64 ") ", ref->ref_name, ref->update_index);
+ if (ref->value != NULL) {
+ hex_format(hex, ref->value, hash_size);
+ printf("%s", hex);
@@ -3274,9 +3274,9 @@
+{
+ char hex[SHA256_SIZE + 1] = {};
+
-+ printf("log{%s(%" PRIdMAX ") %s <%s> %" PRIuMAX " %04d\n",
-+ log->ref_name, log->update_index, log->name, log->email,
-+ log->time, log->tz_offset);
++ printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", log->ref_name,
++ log->update_index, log->name, log->email, log->time,
++ log->tz_offset);
+ hex_format(hex, log->old_hash, hash_size);
+ printf("%s => ", hex);
+ hex_format(hex, log->new_hash, hash_size);
@@ -4860,7 +4860,7 @@
+static void format_name(struct slice *dest, uint64_t min, uint64_t max)
+{
+ char buf[100];
-+ snprintf(buf, sizeof(buf), "%012" PRIxMAX "-%012" PRIxMAX, min, max);
++ snprintf(buf, sizeof(buf), "%012" PRIx64 "-%012" PRIx64, min, max);
+ slice_set_string(dest, buf);
+}
+
@@ -5585,6 +5585,7 @@
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
++#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
@@ -5595,9 +5596,6 @@
+#include <unistd.h>
+#include <zlib.h>
+
-+#define PRIuMAX "lu"
-+#define PRIdMAX "ld"
-+#define PRIxMAX "lx"
+#define ARRAY_SIZE(a) sizeof((a)) / sizeof((a)[0])
+#define FREE_AND_NULL(x) \
+ do { \
@@ -5605,14 +5603,14 @@
+ (x) = NULL; \
+ } while (0)
+#define QSORT(arr, n, cmp) qsort(arr, n, sizeof(arr[0]), cmp)
-+#define SWAP(a, b) \
-+ { \
-+ char tmp[sizeof(a)]; \
-+ assert(sizeof(a)==sizeof(b)); \
-+ memcpy(&tmp[0], &a, sizeof(a)); \
-+ memcpy(&a, &b, sizeof(a)); \
-+ memcpy(&b, &tmp[0], sizeof(a)); \
-+ }
++#define SWAP(a, b) \
++ { \
++ char tmp[sizeof(a)]; \
++ assert(sizeof(a) == sizeof(b)); \
++ memcpy(&tmp[0], &a, sizeof(a)); \
++ memcpy(&a, &b, sizeof(a)); \
++ memcpy(&b, &tmp[0], sizeof(a)); \
++ }
+#endif
+
+int uncompress_return_consumed(Bytef *dest, uLongf *destLen,
@@ -5758,6 +5756,7 @@
+ return &w->stats.log_stats;
+ }
+ assert(false);
++ return NULL;
+}
+
+/* write data, queuing the padding for the next write. Returns negative for
@@ -6299,7 +6298,7 @@
+ w->stats.blocks++;
+
+ if (debug) {
-+ fprintf(stderr, "block %c off %" PRIuMAX " sz %d (%d)\n", typ,
++ fprintf(stderr, "block %c off %" PRIu64 " sz %d (%d)\n", typ,
+ w->next, raw_bytes,
+ get_u24(w->block + w->block_writer->header_off + 1));
+ }
6: 0b2a1a81d6 ! 5: 2786a6bf61 Reftable support for git-core
@@ -39,6 +39,7 @@
reftable.LogRecord{RefName:"HEAD", UpdateIndex:0x2, New:[]uint8{0x37, 0x3d, 0x96, 0x97, 0x2f, 0xca, 0x9b, 0x63, 0x59, 0x57, 0x40, 0xbb, 0xa3, 0x89, 0x8a, 0x76, 0x27, 0x78, 0xba, 0x20}, Old:[]uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, Name:"Han-Wen Nienhuys", Email:"hanwen@google.com", Time:0x5e29ef27, TZOffset:100, Message:"commit (initial): x\n"}
Signed-off-by: Han-Wen Nienhuys <hanwen@google.com>
+ Co-authored-by: Jeff King <peff@peff.net>
diff --git a/Makefile b/Makefile
--- a/Makefile
@@ -120,6 +121,31 @@ $^
{
struct stat st1;
struct strbuf buf = STRBUF_INIT;
+@@
+ is_bare_repository_cfg = init_is_bare_repository;
+ if (init_shared_repository != -1)
+ set_shared_repository(init_shared_repository);
++ if (flags & INIT_DB_REFTABLE)
++ the_repository->ref_storage_format = xstrdup("reftable");
+
+ /*
+ * We would have created the above under user's umask -- under
+@@
+ adjust_shared_perm(get_git_dir());
+ }
+
++ /*
++ * Check to see if .git/HEAD exists; this must happen before
++ * initializing the ref db, because we want to see if there is an
++ * existing HEAD.
++ */
++ path = git_path_buf(&buf, "HEAD");
++ reinit = (!access(path, R_OK) ||
++ readlink(path, junk, sizeof(junk) - 1) != -1);
++
+ /*
+ * We need to create a "refs" dir in any case so that older
+ * versions of git can tell that this is a repository.
@@
* Create the default symlink from ".git/HEAD" to the "master"
* branch, if it does not exist yet.
@@ -127,18 +153,14 @@ $^
- path = git_path_buf(&buf, "HEAD");
- reinit = (!access(path, R_OK)
- || readlink(path, junk, sizeof(junk)-1) != -1);
-+ if (flags & INIT_DB_REFTABLE) {
-+ reinit = 0; /* XXX - how do we recognize a reinit,
-+ * and what should we do? */
-+ } else {
-+ path = git_path_buf(&buf, "HEAD");
-+ reinit = (!access(path, R_OK) ||
-+ readlink(path, junk, sizeof(junk) - 1) != -1);
-+ }
-+
if (!reinit) {
if (create_symref("HEAD", "refs/heads/master", NULL) < 0)
exit(1);
++ } else {
++ /*
++ * XXX should check whether our ref backend matches the
++ * original one; if not, either die() or convert
++ */
}
/* This forces creation of new config file */
@@ -251,10 +273,8 @@ $^
- r->refs = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS);
+ r->refs = ref_store_init(r->gitdir,
-+ /* XXX r->ref_storage_format == NULL. Where
-+ * should the config file be parsed out? */
+ r->ref_storage_format ? r->ref_storage_format :
-+ "reftable",
++ "files",
+ REF_STORE_ALL_CAPS);
return r->refs;
}
@@ -410,13 +430,14 @@ $^
+{
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
-+
++ FILE *file;
+ safe_create_dir(refs->reftable_dir, 1);
-+ FILE *f = fopen(refs->table_list_file, "a");
-+ if (f == NULL) {
++
++ file = fopen(refs->table_list_file, "a");
++ if (file == NULL) {
+ return -1;
+ }
-+ fclose(f);
++ fclose(file);
+ return 0;
+}
+
@@ -777,7 +798,6 @@ $^
+ log.old_hash = old_oid.hash;
+ }
+
-+ /* XXX should the resolution be done relative or absolute? */
+ if (refs_resolve_ref_unsafe((struct ref_store *)create->refs,
+ create->target, RESOLVE_REF_READING,
+ &new_oid, NULL) != NULL) {
@@ -1194,9 +1214,7 @@ $^
if (worktree)
repo_set_worktree(repo, worktree);
-+ repo->ref_storage_format = format.ref_storage != NULL ?
-+ xstrdup(format.ref_storage) :
-+ "files"; /* XXX */
++ repo->ref_storage_format = xstrdup_or_null(format.ref_storage);
+
clear_repository_format(&format);
return 0;
@@ -1227,7 +1245,7 @@ $^
+ } else if (!strcmp(ext, "worktreeconfig")) {
data->worktree_config = git_config_bool(var, value);
- else
-+ } else if (!strcmp(ext, "refStorage")) {
++ } else if (!strcmp(ext, "refstorage")) {
+ data->ref_storage = xstrdup(value);
+ } else
string_list_append(&data->unknown_extensions, ext);
@@ -1241,3 +1259,16 @@ $^
init_repository_format(format);
}
+@@
+ gitdir = DEFAULT_GIT_DIR_ENVIRONMENT;
+ setup_git_env(gitdir);
+ }
+- if (startup_info->have_repository)
++ if (startup_info->have_repository) {
+ repo_set_hash_algo(the_repository, repo_fmt.hash_algo);
++ the_repository->ref_storage_format =
++ xstrdup_or_null(repo_fmt.ref_storage);
++ }
+ }
+
+ strbuf_release(&dir);-- gitgitgadget