{"thread":{"id":"29224","subject":"[PATCH 5/6] Add MD support for fetch, pull, and push.","startedAt":"2011-12-21T07:12:48Z","lastAt":"2011-12-21T07:12:48Z","messageCount":1,"participants":["Bill Zaumen"],"isPatch":true,"patchVersion":1,"patchTotal":6},"messages":[{"id":"181566","messageId":"1324451568.1684.23.camel@yos","threadId":"29224","inReplyTo":null,"subject":"[PATCH 5/6] Add MD support for fetch, pull, and push.","fromName":"Bill Zaumen","fromEmail":"bill.zaumen+git@gmail.com","sentAt":"2011-12-21T07:12:48Z","receivedAt":"2011-12-21T07:12:48Z","isPatch":true,"sender":{"key":"bill.zaumen+git@gmail.com","avatar":null},"body":"A new capability, \"mds-check\" is defined. When present, a client will\n(when possible and useful) send the server a SHA-1 value and a message\ndigest, separated by a '-'.  This is used to detect hash collisions,\nwith a goal of finding problems early if a malicious attempt is made\nto forge commits with different commits with the same SHA-1 value in\ndifferent repositories.  It is a simple, fast test - a look-up and a\ncomparison.\n\nSigned-off-by: Bill Zaumen <bill.zaumen+git@gmail.com>\n---\n builtin/fetch-pack.c   |   29 +++++++++++-\n builtin/receive-pack.c |  117 +++++++++++++++++++++++++++++++++++++++++++-----\n builtin/send-pack.c    |   26 ++++++++++-\n http.c                 |   19 ++++++--\n t/t5500-fetch-pack.sh  |   10 +++--\n t/t5510-fetch.sh       |   12 ++++-\n upload-pack.c          |   11 ++++-\n 7 files changed, 196 insertions(+), 28 deletions(-)\n\ndiff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c\nindex 6207ecd..2f5b7ef 100644\n--- a/builtin/fetch-pack.c\n+++ b/builtin/fetch-pack.c\n@@ -18,6 +18,8 @@ static int prefer_ofs_delta = 1;\n static int no_done;\n static int fetch_fsck_objects = -1;\n static int transfer_fsck_objects = -1;\n+static int mds_check = 0;\n+\n static struct fetch_pack_args args = {\n \t/* .uploadpack = */ \"git-upload-pack\",\n };\n@@ -390,9 +392,25 @@ static int find_common(int fd[2], unsigned char *result_sha1,\n \tflushes = 0;\n \tretval = -1;\n \twhile ((sha1 = get_rev())) {\n-\t\tpacket_buf_write(&req_buf, \"have %s\\n\", sha1_to_hex(sha1));\n-\t\tif (args.verbose)\n-\t\t\tfprintf(stderr, \"have %s\\n\", sha1_to_hex(sha1));\n+\t\tif (mds_check) {\n+\t\t\tmdigest_t digest;\n+\t\t\tif (has_sha1_file_digest(sha1, &digest)) {\n+\t\t\t\tpacket_buf_write(&req_buf, \"have %s\\n\",\n+\t\t\t\t\t\t sha1_to_hex_digest(sha1,\n+\t\t\t\t\t\t\t\t    &digest));\n+\n+\t\t\t} else {\n+\t\t\t\tpacket_buf_write(&req_buf, \"have %s\\n\",\n+\t\t\t\t\t\t sha1_to_hex(sha1));\n+\t\t\t}\n+\t\t\tif (args.verbose)\n+\t\t\t\tfprintf(stderr, \"have %s\\n\", sha1_to_hex(sha1));\n+\t\t} else {\n+\t\t\tpacket_buf_write(&req_buf, \"have %s\\n\",\n+\t\t\t\t\t sha1_to_hex(sha1));\n+\t\t\tif (args.verbose)\n+\t\t\t\tfprintf(stderr, \"have %s\\n\", sha1_to_hex(sha1));\n+\t\t}\n \t\tin_vain++;\n \t\tif (flush_at <= ++count) {\n \t\t\tint ack;\n@@ -807,6 +825,11 @@ static struct ref *do_fetch_pack(int fd[2],\n \t\t\tfprintf(stderr, \"Server supports ofs-delta\\n\");\n \t} else\n \t\tprefer_ofs_delta = 0;\n+\tif (server_supports(\"mds-check\")) {\n+\t\tif (args.verbose)\n+\t\t\tfprintf(stderr, \"Server supports mds-check\\n\");\n+\t\tmds_check = 1;\n+\t}\n \tif (everything_local(&ref, nr_match, match)) {\n \t\tpacket_flush(fd[1]);\n \t\tgoto all_done;\ndiff --git a/builtin/receive-pack.c b/builtin/receive-pack.c\nindex d2dcb7e..b9d1c1f 100644\n--- a/builtin/receive-pack.c\n+++ b/builtin/receive-pack.c\n@@ -122,7 +122,8 @@ static int show_ref(const char *path, const unsigned char *sha1, int flag, void\n \telse\n \t\tpacket_write(1, \"%s %s%c%s%s\\n\",\n \t\t\t     sha1_to_hex(sha1), path, 0,\n-\t\t\t     \" report-status delete-refs side-band-64k\",\n+\t\t\t     \" report-status delete-refs side-band-64k\"\n+\t\t\t     \" mds-check\",\n \t\t\t     prefer_ofs_delta ? \" ofs-delta\" : \"\");\n \tsent_capabilities = 1;\n \treturn 0;\n@@ -709,37 +710,131 @@ static struct command *read_head_info(void)\n \tstruct command *commands = NULL;\n \tstruct command **p = &commands;\n \tfor (;;) {\n-\t\tstatic char line[1000];\n+\t\tstatic char line[1500];\n \t\tunsigned char old_sha1[20], new_sha1[20];\n \t\tstruct command *cmd;\n \t\tchar *refname;\n \t\tint len, reflen;\n+\t\tint has_old_sha1_digest = 0;\n+\t\tint has_new_sha1_digest = 0;\n+\t\tmdigest_t old_sha1_digest;\n+\t\tmdigest_t new_sha1_digest;\n+\t\tmdigest_t digest;\n+\t\tint old_hashlen = 40;\n+\t\tint new_hashlen = 40;\n+\t\tint hashlen = 81; /* includes blank between two hashes */\n+\t\tint digest_field_len = 0;\n+\n+\t\tmdigest_clear(&old_sha1_digest);\n+\t\tmdigest_clear(&new_sha1_digest);\n \n \t\tlen = packet_read_line(0, line, sizeof(line));\n \t\tif (!len)\n \t\t\tbreak;\n \t\tif (line[len-1] == '\\n')\n \t\t\tline[--len] = 0;\n-\t\tif (len < 83 ||\n-\t\t    line[40] != ' ' ||\n-\t\t    line[81] != ' ' ||\n-\t\t    get_sha1_hex(line, old_sha1) ||\n-\t\t    get_sha1_hex(line + 41, new_sha1))\n+\t\tif (len > (old_hashlen + 1) && line[old_hashlen] == '-') {\n+\t\t\tdigest_field_len = 1\n+\t\t\t\t+ get_hex_field_size(line+old_hashlen+1);\n+\t\t}\n+\t\tif (len > (old_hashlen + digest_field_len) &&\n+\t\t    line[old_hashlen] == '-') {\n+\t\t\told_hashlen += digest_field_len;\n+\t\t\thashlen += digest_field_len;\n+\t\t\tdigest_field_len = 0;\n+\t\t\tif (len > (old_hashlen + 1)\n+\t\t\t    && line[old_hashlen] == '-') {\n+\t\t\t\tdigest_field_len = 1 +\n+\t\t\t\t\tget_hex_field_size(line+old_hashlen+1);\n+\t\t\t}\n+\t\t\tif (len > (old_hashlen + digest_field_len + 1) &&\n+\t\t\t    line[old_hashlen] == '-') {\n+\t\t\t\told_hashlen += digest_field_len;\n+\t\t\t\thashlen += digest_field_len;\n+\t\t\t}\n+\t\t}\n+\t\tif (line[old_hashlen] != ' ') {\n+\t\t\tdie(\"protocol error: expected old/new/ref, got '%s'\",\n+\t\t\t    line);\n+\t\t}\n+\t\tdigest_field_len = 0;\n+\t\tif (len > hashlen + 1 && line[hashlen] == '-') {\n+\t\t\tdigest_field_len = 1 +\n+\t\t\t  get_hex_field_size(line+hashlen+1);\n+\t\t}\n+\t\tif (len > (hashlen + digest_field_len + 1) &&\n+\t\t    line[hashlen] == '-') {\n+\t\t\tnew_hashlen += digest_field_len;\n+\t\t\thashlen += digest_field_len;\n+\t\t\tdigest_field_len = 0;\n+\t\t\tif (len > (hashlen + 1)\n+\t\t\t    && line[hashlen] == '-') {\n+\t\t\t\tdigest_field_len = 1 +\n+\t\t\t\t\tget_hex_field_size(line+hashlen+1);\n+\t\t\t}\n+\t\t\tif (len > (hashlen + digest_field_len + 1) &&\n+\t\t\t    line[hashlen] == '-') {\n+\t\t\t\tnew_hashlen += digest_field_len;\n+\t\t\t\thashlen += digest_field_len;\n+\t\t\t}\n+\t\t}\n+\t\tif (line[hashlen] != ' ') {\n \t\t\tdie(\"protocol error: expected old/new/ref, got '%s'\",\n \t\t\t    line);\n+\t\t}\n+\n+\t\tif (len < hashlen + 1 ||\n+\t\t    line[old_hashlen] != ' ' ||\n+\t\t    line[hashlen] != ' ' ||\n+\t\t    get_sha1_hex_digest(line, old_sha1,\n+\t\t\t\t     &has_old_sha1_digest, &old_sha1_digest) ||\n+\t\t    get_sha1_hex_digest(line + old_hashlen + 1, new_sha1,\n+\t\t\t\t\t&has_new_sha1_digest,\n+\t\t\t\t\t&new_sha1_digest)) {\n+\t\t\tdie(\"protocol error: expected old/new/ref, got '%s'\",\n+\t\t\t    line);\n+\t\t}\n+\n+\t\tif (has_old_sha1_digest &&\n+\t\t    has_sha1_file_digest(old_sha1, &digest)) {\n+\t\t\tif (mdigest_tst(&old_sha1_digest, &digest)) {\n+\t\t\t\tdie(\"hash collision for %s\",\n+\t\t\t\t    sha1_to_hex(old_sha1));\n+\t\t\t}\n+\t\t}\n+\n+\n+\t\tif (has_new_sha1_digest &&\n+\t\t    has_sha1_file_digest(new_sha1, &digest)) {\n+\t\t\tif (mdigest_tst(&new_sha1_digest, &digest)) {\n+\t\t\t\tdie(\"hash collision for %s\",\n+\t\t\t\t    sha1_to_hex(new_sha1));\n+\t\t\t}\n+\t\t}\n \n-\t\trefname = line + 82;\n+\t\trefname = line + hashlen + 1;\n \t\treflen = strlen(refname);\n-\t\tif (reflen + 82 < len) {\n+\t\tif (reflen + hashlen + 1 < len) {\n \t\t\tif (strstr(refname + reflen + 1, \"report-status\"))\n \t\t\t\treport_status = 1;\n \t\t\tif (strstr(refname + reflen + 1, \"side-band-64k\"))\n \t\t\t\tuse_sideband = LARGE_PACKET_MAX;\n \t\t}\n-\t\tcmd = xcalloc(1, sizeof(struct command) + len - 80);\n+\t\t/*\n+\t\t * Without the additional digests,\n+\t\t *   old_hashlen + new_hashlen = 80\n+\t\t *   hashlen = 81,\n+\t\t *   hashlen + 1 = 82\n+\t\t * which puts the same numeric values into the last argument\n+\t\t * of xcalloc, and the second & third argument of memcpy\n+\t\t * that were used in commit\n+\t\t * fc14b89a7e6899b8ac3b5751ec2d8c98203ea4c2.\n+\t\t */\n+\t\tcmd = xcalloc(1, sizeof(struct command) + len\n+\t\t\t      - (old_hashlen + new_hashlen));\n \t\thashcpy(cmd->old_sha1, old_sha1);\n \t\thashcpy(cmd->new_sha1, new_sha1);\n-\t\tmemcpy(cmd->ref_name, line + 82, len - 81);\n+\t\tmemcpy(cmd->ref_name, line + hashlen + 1, len - (hashlen));\n \t\t*p = cmd;\n \t\tp = &cmd->next;\n \t}\ndiff --git a/builtin/send-pack.c b/builtin/send-pack.c\nindex cd1115f..1eb9704 100644\n--- a/builtin/send-pack.c\n+++ b/builtin/send-pack.c\n@@ -250,6 +250,7 @@ int send_pack(struct send_pack_args *args,\n \tint allow_deleting_refs = 0;\n \tint status_report = 0;\n \tint use_sideband = 0;\n+\tint mds_check = 0;\n \tunsigned cmds_sent = 0;\n \tint ret;\n \tstruct async demux;\n@@ -263,6 +264,8 @@ int send_pack(struct send_pack_args *args,\n \t\targs->use_ofs_delta = 1;\n \tif (server_supports(\"side-band-64k\"))\n \t\tuse_sideband = 1;\n+\tif (server_supports(\"mds-check\"))\n+\t\tmds_check = 1;\n \n \tif (!remote_refs) {\n \t\tfprintf(stderr, \"No refs in common and none specified; doing nothing.\\n\"\n@@ -298,8 +301,27 @@ int send_pack(struct send_pack_args *args,\n \t\tif (args->dry_run) {\n \t\t\tref->status = REF_STATUS_OK;\n \t\t} else {\n-\t\t\tchar *old_hex = sha1_to_hex(ref->old_sha1);\n-\t\t\tchar *new_hex = sha1_to_hex(ref->new_sha1);\n+\t\t\tchar *old_hex, *new_hex;\n+\t\t\tif (mds_check) {\n+\t\t\t\tmdigest_t digest;\n+\t\t\t\tif (has_sha1_file_digest(ref->old_sha1,\n+\t\t\t\t\t\t      &digest)) {\n+\t\t\t\t\told_hex = sha1_to_hex_digest\n+\t\t\t\t\t\t(ref->old_sha1, &digest);\n+\t\t\t\t} else {\n+\t\t\t\t\told_hex = sha1_to_hex(ref->old_sha1);\n+\t\t\t\t}\n+\t\t\t\tif (has_sha1_file_digest(ref->new_sha1,\n+\t\t\t\t\t\t      &digest)) {\n+\t\t\t\t\tnew_hex = sha1_to_hex_digest\n+\t\t\t\t\t\t(ref->new_sha1, &digest);\n+\t\t\t\t} else {\n+\t\t\t\t\tnew_hex = sha1_to_hex(ref->new_sha1);\n+\t\t\t\t}\n+\t\t\t} else {\n+\t\t\t\told_hex = sha1_to_hex(ref->old_sha1);\n+\t\t\t\tnew_hex = sha1_to_hex(ref->new_sha1);\n+\t\t\t}\n \n \t\t\tif (!cmds_sent && (status_report || use_sideband)) {\n \t\t\t\tpacket_buf_write(&req_buf, \"%s %s %s%c%s%s\",\ndiff --git a/http.c b/http.c\nindex 0ffd79c..e4e3ec7 100644\n--- a/http.c\n+++ b/http.c\n@@ -1014,8 +1014,9 @@ int finish_http_pack_request(struct http_pack_request *preq)\n \tstruct packed_git **lst;\n \tstruct packed_git *p = preq->target;\n \tchar *tmp_idx;\n+\tchar *tmp_mds;\n \tstruct child_process ip;\n-\tconst char *ip_argv[8];\n+\tconst char *ip_argv[10];\n \n \tclose_pack_index(p);\n \n@@ -1028,14 +1029,20 @@ int finish_http_pack_request(struct http_pack_request *preq)\n \t*lst = (*lst)->next;\n \n \ttmp_idx = xstrdup(preq->tmpfile);\n+\ttmp_mds = xstrdup(preq->tmpfile);\n \tstrcpy(tmp_idx + strlen(tmp_idx) - strlen(\".pack.temp\"),\n \t       \".idx.temp\");\n+\tstrcpy(tmp_mds + strlen(tmp_mds) - strlen(\".pack.temp\"),\n+\t       \".mds.temp\");\n+\n \n \tip_argv[0] = \"index-pack\";\n \tip_argv[1] = \"-o\";\n \tip_argv[2] = tmp_idx;\n-\tip_argv[3] = preq->tmpfile;\n-\tip_argv[4] = NULL;\n+\tip_argv[3] = \"-m\";\n+\tip_argv[4] = tmp_mds;\n+\tip_argv[5] = preq->tmpfile;\n+\tip_argv[6] = NULL;\n \n \tmemset(&ip, 0, sizeof(ip));\n \tip.argv = ip_argv;\n@@ -1046,20 +1053,24 @@ int finish_http_pack_request(struct http_pack_request *preq)\n \tif (run_command(&ip)) {\n \t\tunlink(preq->tmpfile);\n \t\tunlink(tmp_idx);\n+\t\tunlink(tmp_mds);\n \t\tfree(tmp_idx);\n+\t\tfree(tmp_mds);\n \t\treturn -1;\n \t}\n \n \tunlink(sha1_pack_index_name(p->sha1));\n \n \tif (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))\n-\t || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {\n+\t || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))\n+\t || move_temp_to_file(tmp_mds, sha1_pack_mds_name(p->sha1))) {\n \t\tfree(tmp_idx);\n \t\treturn -1;\n \t}\n \n \tinstall_packed_git(p);\n \tfree(tmp_idx);\n+\tfree(tmp_mds);\n \treturn 0;\n }\n \ndiff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh\nindex 9bf69e9..b6632d2 100755\n--- a/t/t5500-fetch-pack.sh\n+++ b/t/t5500-fetch-pack.sh\n@@ -53,8 +53,8 @@ pull_to_client () {\n \t\t\tgit symbolic-ref HEAD refs/heads/`echo $heads \\\n \t\t\t\t| sed -e \"s/^\\(.\\).*$/\\1/\"` &&\n \n-\t\t\tgit fsck --full &&\n-\n+\t\t\tgit fsck --full  &&\n+\t\t\ttest -z \"`git count-objects -v -M | grep MD`\" &&\n \t\t\tmv .git/objects/pack/pack-* . &&\n \t\t\tp=`ls -1 pack-*.pack` &&\n \t\t\tgit unpack-objects <$p &&\n@@ -142,7 +142,8 @@ test_expect_success 'fsck in shallow repo' '\n test_expect_success 'simple fetch in shallow repo' '\n \t(\n \t\tcd shallow &&\n-\t\tgit fetch\n+\t\tgit fetch &&\n+\t\ttest -z \"`git count-objects -v -M | grep MD`\"\n \t)\n '\n \n@@ -245,7 +246,8 @@ test_expect_success 'clone shallow object count' '\n \t\tcd shallow &&\n \t\tgit count-objects -v\n \t) > count.shallow &&\n-\tgrep \"^count: 52\" count.shallow\n+\tgrep \"^count: 52\" count.shallow  &&\n+\ttest -z \"`git count-objects -v -M | grep MD`\"\n '\n \n test_done\ndiff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh\nindex e88dbd5..5e3b8c6 100755\n--- a/t/t5510-fetch.sh\n+++ b/t/t5510-fetch.sh\n@@ -14,6 +14,12 @@ test_bundle_object_count () {\n \ttest \"$2\" = $(grep '^[0-9a-f]\\{40\\} ' verify.out | wc -l)\n }\n \n+test_bundle_mds_count () {\n+\tgit verify-pack -v -M \"$1\" >verify.out &&\n+\ttest \"$2\" = $(grep '^[0-9a-f]\\{40\\} ' verify.out | grep -v \"<no md>\" | wc -l)\n+}\n+\n+\n test_expect_success setup '\n \techo >file original &&\n \tgit add file &&\n@@ -214,7 +220,8 @@ test_expect_success 'bundle 1 has only 3 files ' '\n \t\tcat\n \t) <bundle1 >bundle.pack &&\n \tgit index-pack bundle.pack &&\n-\ttest_bundle_object_count bundle.pack 3\n+\ttest_bundle_object_count bundle.pack 3 &&\n+\ttest_bundle_mds_count bundle.pack 3\n '\n \n test_expect_success 'unbundle 2' '\n@@ -237,7 +244,8 @@ test_expect_success 'bundle does not prerequisite objects' '\n \t\tcat\n \t) <bundle3 >bundle.pack &&\n \tgit index-pack bundle.pack &&\n-\ttest_bundle_object_count bundle.pack 3\n+\ttest_bundle_object_count bundle.pack 3 &&\n+\ttest_bundle_mds_count bundle.pack 3\n '\n \n test_expect_success 'bundle should be able to create a full history' '\ndiff --git a/upload-pack.c b/upload-pack.c\nindex 6f36f62..1e77826 100644\n--- a/upload-pack.c\n+++ b/upload-pack.c\n@@ -320,11 +320,18 @@ static int got_sha1(char *hex, unsigned char *sha1)\n {\n \tstruct object *o;\n \tint we_knew_they_have = 0;\n+\tint has_sha1_digest, has_digest;\n+\tmdigest_t sha1_digest, digest;\n \n-\tif (get_sha1_hex(hex, sha1))\n+\tif (get_sha1_hex_digest(hex, sha1, &has_sha1_digest, &sha1_digest))\n \t\tdie(\"git upload-pack: expected SHA1 object, got '%s'\", hex);\n \tif (!has_sha1_file(sha1))\n \t\treturn -1;\n+\thas_digest = has_sha1_file_digest(sha1, &digest);\n+\tif (has_sha1_digest && has_digest\n+\t    && mdigest_tst(&digest, &sha1_digest)) {\n+\t\tdie(\"git upload-pack: SHA1 collision on MD for %s\", hex);\n+\t}\n \n \to = lookup_object(sha1);\n \tif (!(o && o->parsed))\n@@ -719,7 +726,7 @@ static int send_ref(const char *refname, const unsigned char *sha1, int flag, vo\n {\n \tstatic const char *capabilities = \"multi_ack thin-pack side-band\"\n \t\t\" side-band-64k ofs-delta shallow no-progress\"\n-\t\t\" include-tag multi_ack_detailed\";\n+\t\t\" include-tag multi_ack_detailed\" \" mds-check\";\n \tstruct object *o = parse_object(sha1);\n \tconst char *refname_nons = strip_namespace(refname);\n \n-- \n1.7.1\n"}]}