From c45841fff8f444cc61cecd42e5d2032844ecbe24 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 10 Jul 2023 17:12:33 -0400 Subject: revision.h: store hidden refs in a `strvec` In subsequent commits, it will be convenient to have a 'const char **' of hidden refs (matching `transfer.hiderefs`, `uploadpack.hideRefs`, etc.), instead of a `string_list`. Convert spots throughout the tree that store the list of hidden refs from a `string_list` to a `strvec`. Note that in `parse_hide_refs_config()` there is an ugly const-cast used to avoid an extra copy of each value before trimming any trailing slash characters. This could instead be written as: ref = xstrdup(value); len = strlen(ref); while (len && ref[len - 1] == '/') ref[--len] = '\0'; strvec_push(hide_refs, ref); free(ref); but the double-copy (once when calling `xstrdup()`, and another via `strvec_push()`) is wasteful. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- upload-pack.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index d3312006a3..1a213ed775 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -69,7 +69,7 @@ struct upload_pack_data { struct object_array have_obj; struct oid_array haves; /* v2 only */ struct string_list wanted_refs; /* v2 only */ - struct string_list hidden_refs; + struct strvec hidden_refs; struct object_array shallows; struct string_list deepen_not; @@ -127,7 +127,7 @@ static void upload_pack_data_init(struct upload_pack_data *data) { struct string_list symref = STRING_LIST_INIT_DUP; struct string_list wanted_refs = STRING_LIST_INIT_DUP; - struct string_list hidden_refs = STRING_LIST_INIT_DUP; + struct strvec hidden_refs = STRVEC_INIT; struct object_array want_obj = OBJECT_ARRAY_INIT; struct object_array have_obj = OBJECT_ARRAY_INIT; struct oid_array haves = OID_ARRAY_INIT; @@ -162,7 +162,7 @@ static void upload_pack_data_clear(struct upload_pack_data *data) { string_list_clear(&data->symref, 1); string_list_clear(&data->wanted_refs, 1); - string_list_clear(&data->hidden_refs, 0); + strvec_clear(&data->hidden_refs); object_array_clear(&data->want_obj); object_array_clear(&data->have_obj); oid_array_clear(&data->haves); @@ -1170,7 +1170,7 @@ static void receive_needs(struct upload_pack_data *data, /* return non-zero if the ref is hidden, otherwise 0 */ static int mark_our_ref(const char *refname, const char *refname_full, - const struct object_id *oid, const struct string_list *hidden_refs) + const struct object_id *oid, const struct strvec *hidden_refs) { struct object *o = lookup_unknown_object(the_repository, oid); @@ -1465,7 +1465,7 @@ static int parse_want(struct packet_writer *writer, const char *line, static int parse_want_ref(struct packet_writer *writer, const char *line, struct string_list *wanted_refs, - struct string_list *hidden_refs, + struct strvec *hidden_refs, struct object_array *want_obj) { const char *refname_nons; -- cgit v1.2.3 From e6bf24d39a6a85e0a2257f22bffc09b9df749b4a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 10 Jul 2023 17:12:36 -0400 Subject: refs.h: let `for_each_namespaced_ref()` take excluded patterns A future commit will want to call `for_each_namespaced_ref()` with a list of excluded patterns. We could introduce a variant of that function, say, `for_each_namespaced_ref_exclude()` which takes the extra parameter, and reimplement the original function in terms of that. But all but one caller (in `http-backend.c`) will supply the new parameter, so add the new parameter to `for_each_namespaced_ref()` itself instead of introducing a new function. For now, supply NULL for the list of excluded patterns at all callers to avoid changing behavior, which we will do in a future change. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- http-backend.c | 2 +- refs.c | 5 +++-- refs.h | 7 ++++++- upload-pack.c | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) (limited to 'upload-pack.c') diff --git a/http-backend.c b/http-backend.c index ac146d85c5..ad500683c8 100644 --- a/http-backend.c +++ b/http-backend.c @@ -559,7 +559,7 @@ static void get_info_refs(struct strbuf *hdr, char *arg UNUSED) } else { select_getanyfile(hdr); - for_each_namespaced_ref(show_text_ref, &buf); + for_each_namespaced_ref(NULL, show_text_ref, &buf); send_strbuf(hdr, "text/plain", &buf); } strbuf_release(&buf); diff --git a/refs.c b/refs.c index ec4d5b9101..95a7db9563 100644 --- a/refs.c +++ b/refs.c @@ -1660,13 +1660,14 @@ int for_each_replace_ref(struct repository *r, each_repo_ref_fn fn, void *cb_dat DO_FOR_EACH_INCLUDE_BROKEN, cb_data); } -int for_each_namespaced_ref(each_ref_fn fn, void *cb_data) +int for_each_namespaced_ref(const char **exclude_patterns, + each_ref_fn fn, void *cb_data) { struct strbuf buf = STRBUF_INIT; int ret; strbuf_addf(&buf, "%srefs/", get_git_namespace()); ret = do_for_each_ref(get_main_ref_store(the_repository), - buf.buf, NULL, fn, 0, 0, cb_data); + buf.buf, exclude_patterns, fn, 0, 0, cb_data); strbuf_release(&buf); return ret; } diff --git a/refs.h b/refs.h index 012c0d5213..7152e0a09f 100644 --- a/refs.h +++ b/refs.h @@ -376,7 +376,12 @@ int for_each_glob_ref_in(each_ref_fn fn, const char *pattern, const char *prefix, void *cb_data); int head_ref_namespaced(each_ref_fn fn, void *cb_data); -int for_each_namespaced_ref(each_ref_fn fn, void *cb_data); +/* + * references matching any pattern in "exclude_patterns" are omitted from the + * result set on a best-effort basis. + */ +int for_each_namespaced_ref(const char **exclude_patterns, + each_ref_fn fn, void *cb_data); /* can be used to learn about broken ref and symref */ int refs_for_each_rawref(struct ref_store *refs, each_ref_fn fn, void *cb_data); diff --git a/upload-pack.c b/upload-pack.c index 1a213ed775..99d216938c 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -855,7 +855,7 @@ static void deepen(struct upload_pack_data *data, int depth) * marked with OUR_REF. */ head_ref_namespaced(check_ref, data); - for_each_namespaced_ref(check_ref, data); + for_each_namespaced_ref(NULL, check_ref, data); get_reachable_list(data, &reachable_shallows); result = get_shallow_commits(&reachable_shallows, @@ -1386,7 +1386,7 @@ void upload_pack(const int advertise_refs, const int stateless_rpc, if (advertise_refs) data.no_done = 1; head_ref_namespaced(send_ref, &data); - for_each_namespaced_ref(send_ref, &data); + for_each_namespaced_ref(NULL, send_ref, &data); if (!data.sent_capabilities) { const char *refname = "capabilities^{}"; write_v0_ref(&data, refname, refname, null_oid()); @@ -1400,7 +1400,7 @@ void upload_pack(const int advertise_refs, const int stateless_rpc, packet_flush(1); } else { head_ref_namespaced(check_ref, &data); - for_each_namespaced_ref(check_ref, &data); + for_each_namespaced_ref(NULL, check_ref, &data); } if (!advertise_refs) { -- cgit v1.2.3 From 18b6b1b5c57bb4d1b2730b664fb1660b006ef7d3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 10 Jul 2023 17:12:45 -0400 Subject: upload-pack.c: avoid enumerating hidden refs where possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a similar fashion as a previous commit, teach `upload-pack` to avoid enumerating hidden references where possible. Note, however, that there are certain cases where cannot avoid enumerating even hidden references, in particular when either of: - `uploadpack.allowTipSHA1InWant`, or - `uploadpack.allowReachableSHA1InWant` are set, corresponding to `ALLOW_TIP_SHA1` and `ALLOW_REACHABLE_SHA1`, respectively. When either of these bits are set, upload-pack's `is_our_ref()` function needs to consider the `HIDDEN_REF` bit of the referent's object flags. So we must visit all references, including the hidden ones, in order to mark their referents with the `HIDDEN_REF` bit. When neither `ALLOW_TIP_SHA1` nor `ALLOW_REACHABLE_SHA1` are set, the `is_our_ref()` function considers only the `OUR_REF` bit, and not the `HIDDEN_REF` one. `OUR_REF` is applied via `mark_our_ref()`, and only to objects at the tips of non-hidden references, so we do not need to visit hidden references in this case. When neither of those bits are set, `upload-pack` can potentially avoid enumerating a large number of references. In the same example as a previous commit (linux.git with one hidden reference per commit, "refs/pull/N"): $ printf 0000 >in $ hyperfine --warmup=1 \ 'git -c transfer.hideRefs=refs/pull upload-pack . Signed-off-by: Junio C Hamano --- upload-pack.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 99d216938c..ed0126de49 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -602,11 +602,36 @@ static int get_common_commits(struct upload_pack_data *data, } } +static int allow_hidden_refs(enum allow_uor allow_uor) +{ + if ((allow_uor & ALLOW_ANY_SHA1) == ALLOW_ANY_SHA1) + return 1; + return !(allow_uor & (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)); +} + +static void for_each_namespaced_ref_1(each_ref_fn fn, + struct upload_pack_data *data) +{ + const char **excludes = NULL; + /* + * If `data->allow_uor` allows fetching hidden refs, we need to + * mark all references (including hidden ones), to check in + * `is_our_ref()` below. + * + * Otherwise, we only care about whether each reference's object + * has the OUR_REF bit set or not, so do not need to visit + * hidden references. + */ + if (allow_hidden_refs(data->allow_uor)) + excludes = hidden_refs_to_excludes(&data->hidden_refs); + + for_each_namespaced_ref(excludes, fn, data); +} + + static int is_our_ref(struct object *o, enum allow_uor allow_uor) { - int allow_hidden_ref = (allow_uor & - (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)); - return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF); + return o->flags & ((allow_hidden_refs(allow_uor) ? 0 : HIDDEN_REF) | OUR_REF); } /* @@ -855,7 +880,7 @@ static void deepen(struct upload_pack_data *data, int depth) * marked with OUR_REF. */ head_ref_namespaced(check_ref, data); - for_each_namespaced_ref(NULL, check_ref, data); + for_each_namespaced_ref_1(check_ref, data); get_reachable_list(data, &reachable_shallows); result = get_shallow_commits(&reachable_shallows, @@ -1386,7 +1411,7 @@ void upload_pack(const int advertise_refs, const int stateless_rpc, if (advertise_refs) data.no_done = 1; head_ref_namespaced(send_ref, &data); - for_each_namespaced_ref(NULL, send_ref, &data); + for_each_namespaced_ref_1(send_ref, &data); if (!data.sent_capabilities) { const char *refname = "capabilities^{}"; write_v0_ref(&data, refname, refname, null_oid()); @@ -1400,7 +1425,7 @@ void upload_pack(const int advertise_refs, const int stateless_rpc, packet_flush(1); } else { head_ref_namespaced(check_ref, &data); - for_each_namespaced_ref(NULL, check_ref, &data); + for_each_namespaced_ref_1(check_ref, &data); } if (!advertise_refs) { -- cgit v1.2.3