threads / patch / 36850

patch, 2 partsrefs.c: write updates to packed refs when a transaction has more than one ref

Subject: [PATCH 1/2] refs.c: write updates to packed refs when a transaction has more than one ref

## tl;dr

5 messages between Jun 5, 2014 and Jun 9, 2014. Diffs are folded; open one to read it.

replies: 4people: 2as markdown or json

Ronnie Sahlberg· Jun 5, 2014, 23:26 UTC · lore

[PATCH 0/2] Write multi-ref updates directly to packed refs

This patch adds support to the transaction system to update all non-refs as packed refs instead of loose refs.

This series can also be found at https://github.com/rsahlberg/git/tree/ref-transactions-req-packed-refs

This series adds support to have transaction write updates to packed refs instead of as loose refs. If the transaction only contains a single update to a single ref, such as the commit case, we continue to update this as locking and writing to the loose ref file. But If on the other hand the transaction will update more than one ref, then we perform all these updates inside the packed refs file and remove all the affected loose refs files. This allows us to both perform deletes (added previously) as well as updates-of-more-than-one-ref as one single atomic rename operation.

Since the transactions will now by default write directly to the packed refs file for updates to more than one ref, we no longer need to provide a pacekd refs api to builtin/clone.c any more. So lets change clone.c to use a transaction. This removes 4 fucntions from the refs.c API we can now unexport and make static.

This change means that clone will now apear as one single atomic operation to any external observer where all refs appear at once instead of one ref becoming visible at a time.

Ronnie Sahlberg (2):
  refs.c: write updates to packed refs when a transaction has more than
    one ref
  refs.c: make the *_packed_refs functions static
 builtin/clone.c | 16 ++++++++---
 refs.c          | 82 ++++++++++++++++++++++++++++++++++++++++-----------------
 refs.h          | 28 --------------------
 3 files changed, 70 insertions(+), 56 deletions(-)
-- 
2.0.0.583.g402232d
Ronnie Sahlberg· Jun 5, 2014, 23:26 UTC · re: Ronnie Sahlberg · lore

When we are updating more than one single ref, i.e. not a commit, then write the updated refs directly to the packed refs file instead of writing them as loose refs.

Change clone to use a transaction instead of using the pacekd refs api.
Signed-off-by: Ronnie Sahlberg <sahlberg@google.com>
---
 builtin/clone.c | 16 +++++++++----
 refs.c          | 74 +++++++++++++++++++++++++++++++++++++++++----------------
 2 files changed, 66 insertions(+), 24 deletions(-)
Show changes to 2 files +66 −24

builtin/clone.c, refs.c

diff --git a/builtin/clone.c b/builtin/clone.c
index b12989d..000a236 100644
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -497,17 +497,25 @@ static struct ref *wanted_peer_refs(const struct ref *refs,
 static void write_remote_refs(const struct ref *local_refs)
 {
 	const struct ref *r;
+	struct ref_transaction *transaction;
+	struct strbuf err = STRBUF_INIT;
 
-	lock_packed_refs(LOCK_DIE_ON_ERROR);
+	transaction = transaction_begin(&err);
+	if (!transaction)
+		die("%s", err.buf);
 
 	for (r = local_refs; r; r = r->next) {
 		if (!r->peer_ref)
 			continue;
-		add_packed_ref(r->peer_ref->name, r->old_sha1);
+		if (transaction_update_sha1(transaction, r->peer_ref->name,
+					    r->old_sha1, NULL, 0, 0, NULL,
+					    &err))
+			die("%s", err.buf);
 	}
 
-	if (commit_packed_refs())
-		die_errno("unable to overwrite old ref-pack file");
+	if (transaction_commit(transaction, &err))
+		die("%s", err.buf);
+	transaction_free(transaction);
 }
 
 static void write_followtags(const struct ref *refs, const char *msg)
diff --git a/refs.c b/refs.c
index 4281011..6aa64dd 100644
--- a/refs.c
+++ b/refs.c
@@ -2481,31 +2481,18 @@ static int repack_without_refs(const char **refnames, int n, struct strbuf *err)
 	struct ref_dir *packed;
 	struct string_list refs_to_delete = STRING_LIST_INIT_DUP;
 	struct string_list_item *ref_to_delete;
-	int i, ret, removed = 0;
+	int i, ret;
 
 	/* Look for a packed ref */
 	for (i = 0; i < n; i++)
 		if (get_packed_ref(refnames[i]))
 			break;
 
-	/* Avoid processing if we have nothing to do */
-	if (i == n)
-		return 0; /* no refname exists in packed refs */
-
 	packed = get_packed_refs(&ref_cache);
 
 	/* Remove refnames from the cache */
 	for (i = 0; i < n; i++)
-		if (remove_entry(packed, refnames[i]) != -1)
-			removed = 1;
-	if (!removed) {
-		/*
-		 * All packed entries disappeared while we were
-		 * acquiring the lock.
-		 */
-		rollback_packed_refs();
-		return 0;
-	}
+		remove_entry(packed, refnames[i]);
 
 	/* Remove any other accumulated cruft */
 	do_for_each_entry_in_dir(packed, 0, curate_packed_ref_fn, &refs_to_delete);
@@ -3614,6 +3601,7 @@ int transaction_commit(struct ref_transaction *transaction,
 			   struct strbuf *err)
 {
 	int ret = 0, delnum = 0, i, df_conflict = 0, need_repack = 0;
+	int num_updates = 0;
 	const char **delnames;
 	int n = transaction->nr;
 	struct packed_ref_cache *packed_ref_cache;
@@ -3647,18 +3635,38 @@ int transaction_commit(struct ref_transaction *transaction,
 		goto cleanup;
 	}
 
-	/* any loose refs are to be deleted are first copied to packed refs */
+	/* count how many refs we are updating (not deleting) */
 	for (i = 0; i < n; i++) {
 		struct ref_update *update = updates[i];
 		unsigned char sha1[20];
 
 		if (update->update_type != UPDATE_SHA1)
 			continue;
-		if (!is_null_sha1(update->new_sha1))
+		if (is_null_sha1(update->new_sha1))
+			continue;
+
+		num_updates++;
+	}
+
+	/*
+	 * Always copy loose refs that are to be deleted to the packed refs.
+	 * If we are updating multiple refs then copy all non symref refs
+	 * to the packed refs too.
+	 */
+	for (i = 0; i < n; i++) {
+		struct ref_update *update = updates[i];
+		unsigned char sha1[20];
+		int flag;
+
+		if (update->update_type != UPDATE_SHA1)
+			continue;
+		if (num_updates < 2 && !is_null_sha1(update->new_sha1))
 			continue;
 		if (get_packed_ref(update->refname))
 			continue;
-		if (!resolve_ref_unsafe(update->refname, sha1, 1, NULL))
+		if (!resolve_ref_unsafe(update->refname, sha1, 1, &flag))
+			continue;
+		if (flag & REF_ISSYMREF)
 			continue;
 
 		add_packed_ref(update->refname, sha1);
@@ -3682,11 +3690,14 @@ int transaction_commit(struct ref_transaction *transaction,
 			goto cleanup;
 		}
 	}
+	need_repack = 0;
 
 	/*
 	 * At this stage any refs that are to be deleted have been moved to the
 	 * packed refs file anf the packed refs file is deleted. We can now
 	 * safely delete these loose refs.
+	 * If we are updating multiple normal refs, then those will also be in
+	 * the packed refs file so we can delete them too.
 	 */
 
 	/* Unlink any loose refs scheduled for deletion */
@@ -3724,7 +3735,10 @@ int transaction_commit(struct ref_transaction *transaction,
 		update->lock = NULL;
 	}
 
-	/* Acquire all ref locks for updates while verifying old values */
+	/*
+	 * Acquire all ref locks for updates while verifying old values.
+	 * If we are multi-updating then update them in packed refs.
+	 */
 	for (i = 0; i < n; i++) {
 		struct ref_update *update = updates[i];
 
@@ -3748,6 +3762,22 @@ int transaction_commit(struct ref_transaction *transaction,
 			ret = -1;
 			goto cleanup;
 		}
+		if (num_updates < 2 || update->type & REF_ISSYMREF)
+			continue;
+
+		if (delete_ref_loose(update->lock, update->type, err)) {
+			ret = -1;
+			goto cleanup;
+		}
+		unlock_ref(update->lock);
+		update->lock = NULL;
+
+		packed = get_packed_refs(&ref_cache);;
+		remove_entry(packed, update->refname);
+		add_packed_ref(update->refname, update->new_sha1);
+		need_repack = 1;
+
+		try_remove_empty_parents((char *)update->refname);
 	}
 
 	/* delete reflog for all deleted refs */
@@ -3795,7 +3825,7 @@ int transaction_commit(struct ref_transaction *transaction,
 
 		if (update->update_type != UPDATE_SHA1)
 			continue;
-		if (!is_null_sha1(update->new_sha1)) {
+		if (update->lock && !is_null_sha1(update->new_sha1)) {
 			ret = write_ref_sha1(update->lock, update->new_sha1,
 					     update->msg);
 			update->lock = NULL; /* freed by write_ref_sha1 */
@@ -3860,6 +3890,10 @@ int transaction_commit(struct ref_transaction *transaction,
 		}
 	}
 
+	if (need_repack) {
+		packed = get_packed_refs(&ref_cache);
+		sort_ref_dir(packed);
+	}
 	if (repack_without_refs(delnames, delnum, err))
 		ret = -1;
 	clear_loose_ref_cache(&ref_cache);
-- 
2.0.0.583.g402232d
Eric Sunshine· Jun 8, 2014, 09:03 UTC · re: Ronnie Sahlberg · lore

Re: [PATCH 1/2] refs.c: write updates to packed refs when a transaction has more than one ref

On Thu, Jun 5, 2014 at 7:26 PM, Ronnie Sahlberg <sahlberg@google.com> wrote:
Show 5 quoted lines
> When we are updating more than one single ref, i.e. not a commit, then
> write the updated refs directly to the packed refs file instead of writing
> them as loose refs.
>
> Change clone to use a transaction instead of using the pacekd refs api.
s/pacekd/packed/
> Signed-off-by: Ronnie Sahlberg <sahlberg@google.com>
Ronnie Sahlberg· Jun 9, 2014, 16:20 UTC · re: Eric Sunshine · lore

Re: [PATCH 1/2] refs.c: write updates to packed refs when a transaction has more than one ref

thanks
On Sun, Jun 8, 2014 at 2:03 AM, Eric Sunshine <sunshine@sunshineco.com> wrote:
Show 10 quoted lines
> On Thu, Jun 5, 2014 at 7:26 PM, Ronnie Sahlberg <sahlberg@google.com> wrote:
>> When we are updating more than one single ref, i.e. not a commit, then
>> write the updated refs directly to the packed refs file instead of writing
>> them as loose refs.
>>
>> Change clone to use a transaction instead of using the pacekd refs api.
>
> s/pacekd/packed/
>
>> Signed-off-by: Ronnie Sahlberg <sahlberg@google.com>
Ronnie Sahlberg· Jun 5, 2014, 23:26 UTC · re: Ronnie Sahlberg · lore

[PATCH 2/2] refs.c: make the *_packed_refs functions static

We no longer need to expose the lock/add/commit/rollback functions for packed refs anymore so make them static and remove them from the api.

Signed-off-by: Ronnie Sahlberg <sahlberg@google.com>
---
 refs.c |  8 ++++----
 refs.h | 28 ----------------------------
 2 files changed, 4 insertions(+), 32 deletions(-)
Show changes to 2 files +4 −32

refs.c, refs.h

diff --git a/refs.c b/refs.c
index 6aa64dd..530947e 100644
--- a/refs.c
+++ b/refs.c
@@ -1123,7 +1123,7 @@ static struct ref_dir *get_packed_refs(struct ref_cache *refs)
 	return get_packed_ref_dir(get_packed_ref_cache(refs));
 }
 
-void add_packed_ref(const char *refname, const unsigned char *sha1)
+static void add_packed_ref(const char *refname, const unsigned char *sha1)
 {
 	struct packed_ref_cache *packed_ref_cache =
 		get_packed_ref_cache(&ref_cache);
@@ -2211,7 +2211,7 @@ static int write_packed_entry_fn(struct ref_entry *entry, void *cb_data)
 	return 0;
 }
 
-int lock_packed_refs(int flags)
+static int lock_packed_refs(int flags)
 {
 	struct packed_ref_cache *packed_ref_cache;
 
@@ -2234,7 +2234,7 @@ int lock_packed_refs(int flags)
  * Commit the packed refs changes.
  * On error we must make sure that errno contains a meaningful value.
  */
-int commit_packed_refs(void)
+static int commit_packed_refs(void)
 {
 	struct packed_ref_cache *packed_ref_cache =
 		get_packed_ref_cache(&ref_cache);
@@ -2259,7 +2259,7 @@ int commit_packed_refs(void)
 	return error;
 }
 
-void rollback_packed_refs(void)
+static void rollback_packed_refs(void)
 {
 	struct packed_ref_cache *packed_ref_cache =
 		get_packed_ref_cache(&ref_cache);
diff --git a/refs.h b/refs.h
index 3c99619..e92cb92 100644
--- a/refs.h
+++ b/refs.h
@@ -118,34 +118,6 @@ extern int for_each_rawref(each_ref_fn, void *);
 extern void warn_dangling_symref(FILE *fp, const char *msg_fmt, const char *refname);
 
 /*
- * Lock the packed-refs file for writing.  Flags is passed to
- * hold_lock_file_for_update().  Return 0 on success.
- */
-extern int lock_packed_refs(int flags);
-
-/*
- * Add a reference to the in-memory packed reference cache.  This may
- * only be called while the packed-refs file is locked (see
- * lock_packed_refs()).  To actually write the packed-refs file, call
- * commit_packed_refs().
- */
-extern void add_packed_ref(const char *refname, const unsigned char *sha1);
-
-/*
- * Write the current version of the packed refs cache from memory to
- * disk.  The packed-refs file must already be locked for writing (see
- * lock_packed_refs()).  Return zero on success.
- */
-extern int commit_packed_refs(void);
-
-/*
- * Rollback the lockfile for the packed-refs file, and discard the
- * in-memory packed reference cache.  (The packed-refs file will be
- * read anew if it is needed again after this function is called.)
- */
-extern void rollback_packed_refs(void);
-
-/*
  * Flags for controlling behaviour of pack_refs()
  * PACK_REFS_PRUNE: Prune loose refs after packing
  * PACK_REFS_ALL:   Pack _all_ refs, not just tags and already packed refs
-- 
2.0.0.583.g402232d

← back to recent threads