diff options
88 files changed, 2551 insertions, 661 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bd6f75b8e0..592f9193a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,14 +37,14 @@ jobs: echo "::set-output name=enabled::$enabled" - name: skip if the commit or tree was already tested id: skip-if-redundant - uses: actions/github-script@v3 + uses: actions/github-script@v6 if: steps.check-ref.outputs.enabled == 'yes' with: github-token: ${{secrets.GITHUB_TOKEN}} script: | try { // Figure out workflow ID, commit and tree - const { data: run } = await github.actions.getWorkflowRun({ + const { data: run } = await github.rest.actions.getWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId, @@ -54,7 +54,7 @@ jobs: const tree_id = run.head_commit.tree_id; // See whether there is a successful run for that commit or tree - const { data: runs } = await github.actions.listWorkflowRuns({ + const { data: runs } = await github.rest.actions.listWorkflowRuns({ owner: context.repo.owner, repo: context.repo.repo, per_page: 500, diff --git a/Documentation/MyFirstContribution.txt b/Documentation/MyFirstContribution.txt index 1a4be8ee0a..ccfd0cb5f3 100644 --- a/Documentation/MyFirstContribution.txt +++ b/Documentation/MyFirstContribution.txt @@ -736,7 +736,7 @@ the {lore}[Git mailing list archive]: 2022-02-21 1:43 ` John Cai 2022-02-21 1:50 ` Taylor Blau 2022-02-23 19:50 ` John Cai -2022-02-18 20:00 ` // other replies ellided +2022-02-18 20:00 ` // other replies elided 2022-02-18 18:40 ` [PATCH 2/3] reflog: call reflog_delete from reflog.c John Cai via GitGitGadget 2022-02-18 19:15 ` Ævar Arnfjörð Bjarmason 2022-02-18 20:26 ` Junio C Hamano diff --git a/Documentation/RelNotes/2.39.0.txt b/Documentation/RelNotes/2.39.0.txt index 2aae4107c2..153bf6d89b 100644 --- a/Documentation/RelNotes/2.39.0.txt +++ b/Documentation/RelNotes/2.39.0.txt @@ -29,6 +29,12 @@ UI, Workflows & Features existing bugs in the internal patch-id logic that did not match what "git patch-id" produces have been corrected. + * Enable gc.cruftpacks by default for those who opt into + feature.experimental setting. + + * "git repack" learns to send cruft objects out of the way into + packfiles outside the repository. + Performance, Internal Implementation, Development Support etc. -------------------------------------------------------------- @@ -84,6 +90,40 @@ Performance, Internal Implementation, Development Support etc. * Make sure generated dependency file is stably sorted to help developers debugging their build issues. + * The glossary entries for "commit-graph file" and "reachability + bitmap" have been added. + + * Various tests exercising the transfer.credentialsInUrl + configuration are taught to avoid making requests which require + resolving localhost to reduce CI-flakiness. + + * A redundant diagnostic message is dropped from test_path_is_missing(). + + * Simplify the run-command API. + + * Update the actions/github-script dependency in CI to avoid a + deprecation warning. + + * Progress on being able to initialize a rev_info struct with a + macro. + + * Add trace2 counters to the region to clear skip worktree bits in a + sparse checkout. + + * Modernize test script to avoid "test -f" and friends. + + * Avoid calling 'cache_tree_update()' when doing so would be + redundant. + + * Update the credential-cache documentation to provide a more + realistic example. + + * Makefile comments updates and reordering to clarify knobs used to + choose SHA implementations. + + * A design document for sparse-checkout's future directions has been + added. + Fixes since v2.38 ----------------- @@ -215,6 +255,22 @@ Fixes since v2.38 * The way "git repack" creared temporary files when it received a signal was prone to deadlocking, which has been corrected. + * Various tests exercising the transfer.credentialsInUrl + configuration are taught to avoid making requests which require + resolving localhost to reduce CI-flakiness. + + * The adjust_shared_perm() helper function learned to refrain from + setting the "g+s" bit on directories when it is not necessary. + + * "git archive" mistakenly complained twice about a missing + executable, which has been corrected. + + * Fix a bug where `git branch -d` did not work on an orphaned HEAD. + + * `git rebase --update-refs` would delete references when all + `update-ref` commands in the sequencer were removed, which has been + corrected. + * Other code cleanup, docfix, build fix, etc. (merge 413bc6d20a ds/cmd-main-reorder later to maint). (merge 8d2863e4ed nw/t1002-cleanup later to maint). diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 37afbaf5a4..dfbdaf00b8 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -618,7 +618,7 @@ but risks losing recent work in the event of an unclean system shutdown. * `loose-object` hardens objects added to the repo in loose-object form. * `pack` hardens objects added to the repo in packfile form. * `pack-metadata` hardens packfile bitmaps and indexes. -* `commit-graph` hardens the commit graph file. +* `commit-graph` hardens the commit-graph file. * `index` hardens the index when it is modified. * `objects` is an aggregate option that is equivalent to `loose-object,pack`. diff --git a/Documentation/config/feature.txt b/Documentation/config/feature.txt index cdecd04e5b..95975e5091 100644 --- a/Documentation/config/feature.txt +++ b/Documentation/config/feature.txt @@ -14,6 +14,9 @@ feature.experimental:: + * `fetch.negotiationAlgorithm=skipping` may improve fetch negotiation times by skipping more commits at a time, reducing the number of round trips. ++ +* `gc.cruftPacks=true` reduces disk space used by unreachable objects during +garbage collection, preventing loose object explosions. feature.manyFiles:: Enable config options that optimize for repos with many files in the diff --git a/Documentation/git-credential-cache.txt b/Documentation/git-credential-cache.txt index 0216c18ef8..432e159d95 100644 --- a/Documentation/git-credential-cache.txt +++ b/Documentation/git-credential-cache.txt @@ -69,10 +69,10 @@ $ git push http://example.com/repo.git ------------------------------------ You can provide options via the credential.helper configuration -variable (this example drops the cache time to 5 minutes): +variable (this example increases the cache time to 1 hour): ------------------------------------------------------- -$ git config credential.helper 'cache --timeout=300' +$ git config credential.helper 'cache --timeout=3600' ------------------------------------------------------- GIT diff --git a/Documentation/git-credential.txt b/Documentation/git-credential.txt index f18673017f..ac2818b9f6 100644 --- a/Documentation/git-credential.txt +++ b/Documentation/git-credential.txt @@ -160,6 +160,8 @@ empty string. Components which are missing from the URL (e.g., there is no username in the example above) will be left unset. +Unrecognised attributes are silently discarded. + GIT --- Part of the linkgit:git[1] suite diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt index d7986419c2..440043cdb8 100644 --- a/Documentation/git-ls-files.txt +++ b/Documentation/git-ls-files.txt @@ -10,8 +10,8 @@ SYNOPSIS -------- [verse] 'git ls-files' [-z] [-t] [-v] [-f] - [-c|--cached] [-d|--deleted] [-o|--others] [-i|--|ignored] - [-s|--stage] [-u|--unmerged] [-k|--|killed] [-m|--modified] + [-c|--cached] [-d|--deleted] [-o|--others] [-i|--ignored] + [-s|--stage] [-u|--unmerged] [-k|--killed] [-m|--modified] [--directory [--no-empty-directory]] [--eol] [--deduplicate] [-x <pattern>|--exclude=<pattern>] diff --git a/Documentation/git-pack-redundant.txt b/Documentation/git-pack-redundant.txt index dda80a740c..99ef13839d 100644 --- a/Documentation/git-pack-redundant.txt +++ b/Documentation/git-pack-redundant.txt @@ -34,7 +34,7 @@ OPTIONS --alt-odb:: Don't require objects present in packs from alternate object - directories to be present in local packs. + database (odb) directories to be present in local packs. --verbose:: Outputs some statistics to stderr. Has a small performance penalty. diff --git a/Documentation/git-repack.txt b/Documentation/git-repack.txt index 0bf13893d8..4017157949 100644 --- a/Documentation/git-repack.txt +++ b/Documentation/git-repack.txt @@ -74,6 +74,12 @@ to the new separate pack will be written. immediately instead of waiting for the next `git gc` invocation. Only useful with `--cruft -d`. +--expire-to=<dir>:: + Write a cruft pack containing pruned objects (if any) to the + directory `<dir>`. This option is useful for keeping a copy of + any pruned objects in a separate directory as a backup. Only + useful with `--cruft -d`. + -l:: Pass the `--local` option to 'git pack-objects'. See linkgit:git-pack-objects[1]. diff --git a/Documentation/gitcredentials.txt b/Documentation/gitcredentials.txt index 80517b4eb2..d6665d2f31 100644 --- a/Documentation/gitcredentials.txt +++ b/Documentation/gitcredentials.txt @@ -17,9 +17,10 @@ DESCRIPTION Git will sometimes need credentials from the user in order to perform operations; for example, it may need to ask for a username and password -in order to access a remote repository over HTTP. This manual describes -the mechanisms Git uses to request these credentials, as well as some -features to avoid inputting these credentials repeatedly. +in order to access a remote repository over HTTP. Some remotes accept +a personal access token or OAuth access token as a password. This +manual describes the mechanisms Git uses to request these credentials, +as well as some features to avoid inputting these credentials repeatedly. REQUESTING CREDENTIALS ---------------------- @@ -269,6 +270,7 @@ stdout in the same format (see linkgit:git-credential[1] for common attributes). A helper is free to produce a subset, or even no values at all if it has nothing useful to provide. Any provided attributes will overwrite those already known about by Git's credential subsystem. +Unrecognised attributes are silently discarded. While it is possible to override all attributes, well behaving helpers should refrain from doing so for any attribute other than username and diff --git a/Documentation/gitformat-commit-graph.txt b/Documentation/gitformat-commit-graph.txt index 7324665716..31cad585e2 100644 --- a/Documentation/gitformat-commit-graph.txt +++ b/Documentation/gitformat-commit-graph.txt @@ -3,7 +3,7 @@ gitformat-commit-graph(5) NAME ---- -gitformat-commit-graph - Git commit graph format +gitformat-commit-graph - Git commit-graph format SYNOPSIS -------- @@ -14,7 +14,7 @@ $GIT_DIR/objects/info/commit-graphs/* DESCRIPTION ----------- -The Git commit graph stores a list of commit OIDs and some associated +The Git commit-graph stores a list of commit OIDs and some associated metadata, including: - The generation number of the commit. @@ -34,7 +34,7 @@ corresponding to the array position within the list of commit OIDs. Due to some special constants we use to track parents, we can store at most (1 << 30) + (1 << 29) + (1 << 28) - 1 (around 1.8 billion) commits. -== Commit graph files have the following format: +== Commit-graph files have the following format: In order to allow extensions that add extra data to the graph, we organize the body into "chunks" and provide a binary lookup table at the beginning diff --git a/Documentation/glossary-content.txt b/Documentation/glossary-content.txt index aa2f41f5e7..5a537268e2 100644 --- a/Documentation/glossary-content.txt +++ b/Documentation/glossary-content.txt @@ -20,7 +20,7 @@ [[def_branch]]branch:: A "branch" is a line of development. The most recent <<def_commit,commit>> on a branch is referred to as the tip of - that branch. The tip of the branch is referenced by a branch + that branch. The tip of the branch is <<def_ref,referenced>> by a branch <<def_head,head>>, which moves forward as additional development is done on the branch. A single Git <<def_repository,repository>> can track an arbitrary number of @@ -75,6 +75,21 @@ state in the Git history, by creating a new commit representing the current state of the <<def_index,index>> and advancing <<def_HEAD,HEAD>> to point at the new commit. +[[def_commit_graph_general]]commit graph concept, representations and usage:: + A synonym for the <<def_DAG,DAG>> structure formed by the commits + in the object database, <<def_ref,referenced>> by branch tips, + using their <<def_chain,chain>> of linked commits. + This structure is the definitive commit graph. The + graph can be represented in other ways, e.g. the + <<def_commit_graph_file,"commit-graph" file>>. + +[[def_commit_graph_file]]commit-graph file:: + The "commit-graph" (normally hyphenated) file is a supplemental + representation of the <<def_commit_graph_general,commit graph>> + which accelerates commit graph walks. The "commit-graph" file is + stored either in the .git/objects/info directory or in the info + directory of an alternate object database. + [[def_commit_object]]commit object:: An <<def_object,object>> which contains the information about a particular <<def_revision,revision>>, such as <<def_parent,parents>>, committer, @@ -262,7 +277,7 @@ This commit is referred to as a "merge commit", or sometimes just a identified by its <<def_object_name,object name>>. The objects usually live in `$GIT_DIR/objects/`. -[[def_object_identifier]]object identifier:: +[[def_object_identifier]]object identifier (oid):: Synonym for <<def_object_name,object name>>. [[def_object_name]]object name:: @@ -493,6 +508,14 @@ exclude;; <<def_tree_object,trees>> to the trees or <<def_blob_object,blobs>> that they contain. +[[def_reachability_bitmap]]reachability bitmaps:: + Reachability bitmaps store information about the + <<def_reachable,reachability>> of a selected set of commits in + a packfile, or a multi-pack index (MIDX), to speed up object search. + The bitmaps are stored in a ".bitmap" file. A repository may have at + most one bitmap file in use. The bitmap file may belong to either one + pack, or the repository's multi-pack index (if it exists). + [[def_rebase]]rebase:: To reapply a series of changes from a <<def_branch,branch>> to a different base, and reset the <<def_head,head>> of that branch diff --git a/Documentation/howto/maintain-git.txt b/Documentation/howto/maintain-git.txt index a67130debb..215e2edb0f 100644 --- a/Documentation/howto/maintain-git.txt +++ b/Documentation/howto/maintain-git.txt @@ -256,7 +256,7 @@ by doing the following: merged to 'next', add it at the end of the list. Then: $ git checkout -B jch master - $ Meta/redo-jch.sh -c1 + $ sh Meta/redo-jch.sh -c1 to rebuild the 'jch' branch from scratch. "-c1" tells the script to stop merging at the first line that begins with '###' diff --git a/Documentation/technical/commit-graph.txt b/Documentation/technical/commit-graph.txt index 90c9760c23..86fed0de0f 100644 --- a/Documentation/technical/commit-graph.txt +++ b/Documentation/technical/commit-graph.txt @@ -1,4 +1,4 @@ -Git Commit Graph Design Notes +Git Commit-Graph Design Notes ============================= Git walks the commit graph for many reasons, including: @@ -17,7 +17,7 @@ There are two main costs here: The commit-graph file is a supplemental data structure that accelerates commit graph walks. If a user downgrades or disables the 'core.commitGraph' -config setting, then the existing ODB is sufficient. The file is stored +config setting, then the existing object database is sufficient. The file is stored as "commit-graph" either in the .git/objects/info directory or in the info directory of an alternate. @@ -95,7 +95,7 @@ with default order), but is not used when the topological order is required (such as merge base calculations, "git log --graph"). In practice, we expect some commits to be created recently and not stored -in the commit graph. We can treat these commits as having "infinite" +in the commit-graph. We can treat these commits as having "infinite" generation number and walk until reaching commits with known generation number. @@ -149,7 +149,7 @@ Design Details helpful for these clones, anyway. The commit-graph will not be read or written when shallow commits are present. -Commit Graphs Chains +Commit-Graphs Chains -------------------- Typically, repos grow with near-constant velocity (commits per day). Over time, diff --git a/Documentation/technical/parallel-checkout.txt b/Documentation/technical/parallel-checkout.txt index e790258a1a..47c9b6183c 100644 --- a/Documentation/technical/parallel-checkout.txt +++ b/Documentation/technical/parallel-checkout.txt @@ -56,7 +56,7 @@ Rejected Multi-Threaded Solution The most "straightforward" implementation would be to spread the set of to-be-updated cache entries across multiple threads. But due to the -thread-unsafe functions in the ODB code, we would have to use locks to +thread-unsafe functions in the object database code, we would have to use locks to coordinate the parallel operation. An early prototype of this solution showed that the multi-threaded checkout would bring performance improvements over the sequential code, but there was still too much lock diff --git a/Documentation/technical/repository-version.txt b/Documentation/technical/repository-version.txt index 7844ef30ff..8ef664b0b9 100644 --- a/Documentation/technical/repository-version.txt +++ b/Documentation/technical/repository-version.txt @@ -82,9 +82,9 @@ When the config key `extensions.preciousObjects` is set to `true`, objects in the repository MUST NOT be deleted (e.g., by `git-prune` or `git repack -d`). -==== `partialclone` +==== `partialClone` -When the config key `extensions.partialclone` is set, it indicates +When the config key `extensions.partialClone` is set, it indicates that the repo was created with a partial clone (or later performed a partial fetch) and that the remote may have omitted sending certain unwanted objects. Such a remote is called a "promisor remote" diff --git a/Documentation/technical/sparse-checkout.txt b/Documentation/technical/sparse-checkout.txt new file mode 100644 index 0000000000..fa0d01cbda --- /dev/null +++ b/Documentation/technical/sparse-checkout.txt @@ -0,0 +1,1103 @@ +Table of contents: + + * Terminology + * Purpose of sparse-checkouts + * Usecases of primary concern + * Oversimplified mental models ("Cliff Notes" for this document!) + * Desired behavior + * Behavior classes + * Subcommand-dependent defaults + * Sparse specification vs. sparsity patterns + * Implementation Questions + * Implementation Goals/Plans + * Known bugs + * Reference Emails + + +=== Terminology === + +cone mode: one of two modes for specifying the desired subset of files + in a sparse-checkout. In cone-mode, the user specifies + directories (getting both everything under that directory as + well as everything in leading directories), while in non-cone + mode, the user specifies gitignore-style patterns. Controlled + by the --[no-]cone option to sparse-checkout init|set. + +SKIP_WORKTREE: When tracked files do not match the sparse specification and + are removed from the working tree, the file in the index is marked + with a SKIP_WORKTREE bit. Note that if a tracked file has the + SKIP_WORKTREE bit set but the file is later written by the user to + the working tree anyway, the SKIP_WORKTREE bit will be cleared at + the beginning of any subsequent Git operation. + + Most sparse checkout users are unaware of this implementation + detail, and the term should generally be avoided in user-facing + descriptions and command flags. Unfortunately, prior to the + `sparse-checkout` subcommand this low-level detail was exposed, + and as of time of writing, is still exposed in various places. + +sparse-checkout: a subcommand in git used to reduce the files present in + the working tree to a subset of all tracked files. Also, the + name of the file in the $GIT_DIR/info directory used to track + the sparsity patterns corresponding to the user's desired + subset. + +sparse cone: see cone mode + +sparse directory: An entry in the index corresponding to a directory, which + appears in the index instead of all the files under that directory + that would normally appear. See also sparse-index. Something that + can cause confusion is that the "sparse directory" does NOT match + the sparse specification, i.e. the directory is NOT present in the + working tree. May be renamed in the future (e.g. to "skipped + directory"). + +sparse index: A special mode for sparse-checkout that also makes the + index sparse by recording a directory entry in lieu of all the + files underneath that directory (thus making that a "skipped + directory" which unfortunately has also been called a "sparse + directory"), and does this for potentially multiple + directories. Controlled by the --[no-]sparse-index option to + init|set|reapply. + +sparsity patterns: patterns from $GIT_DIR/info/sparse-checkout used to + define the set of files of interest. A warning: It is easy to + over-use this term (or the shortened "patterns" term), for two + reasons: (1) users in cone mode specify directories rather than + patterns (their directories are transformed into patterns, but + users may think you are talking about non-cone mode if you use the + word "patterns"), and (b) the sparse specification might + transiently differ in the working tree or index from the sparsity + patterns (see "Sparse specification vs. sparsity patterns"). + +sparse specification: The set of paths in the user's area of focus. This + is typically just the tracked files that match the sparsity + patterns, but the sparse specification can temporarily differ and + include additional files. (See also "Sparse specification + vs. sparsity patterns") + + * When working with history, the sparse specification is exactly + the set of files matching the sparsity patterns. + * When interacting with the working tree, the sparse specification + is the set of tracked files with a clear SKIP_WORKTREE bit or + tracked files present in the working copy. + * When modifying or showing results from the index, the sparse + specification is the set of files with a clear SKIP_WORKTREE bit + or that differ in the index from HEAD. + * If working with the index and the working copy, the sparse + specification is the union of the paths from above. + +vivifying: When a command restores a tracked file to the working tree (and + hopefully also clears the SKIP_WORKTREE bit in the index for that + file), this is referred to as "vivifying" the file. + + +=== Purpose of sparse-checkouts === + +sparse-checkouts exist to allow users to work with a subset of their +files. + +You can think of sparse-checkouts as subdividing "tracked" files into two +categories -- a sparse subset, and all the rest. Implementationally, we +mark "all the rest" in the index with a SKIP_WORKTREE bit and leave them +out of the working tree. The SKIP_WORKTREE files are still tracked, just +not present in the working tree. + +In the past, sparse-checkouts were defined by "SKIP_WORKTREE means the file +is missing from the working tree but pretend the file contents match HEAD". +That was not only bogus (it actually meant the file missing from the +working tree matched the index rather than HEAD), but it was also a +low-level detail which only provided decent behavior for a few commands. +There were a surprising number of ways in which that guiding principle gave +command results that violated user expectations, and as such was a bad +mental model. However, it persisted for many years and may still be found +in some corners of the code base. + +Anyway, the idea of "working with a subset of files" is simple enough, but +there are multiple different high-level usecases which affect how some Git +subcommands should behave. Further, even if we only considered one of +those usecases, sparse-checkouts can modify different subcommands in over a +half dozen different ways. Let's start by considering the high level +usecases: + + A) Users are _only_ interested in the sparse portion of the repo + + A*) Users are _only_ interested in the sparse portion of the repo + that they have downloaded so far + + B) Users want a sparse working tree, but are working in a larger whole + + C) sparse-checkout is a behind-the-scenes implementation detail allowing + Git to work with a specially crafted in-house virtual file system; + users are actually working with a "full" working tree that is + lazily populated, and sparse-checkout helps with the lazy population + piece. + +It may be worth explaining each of these in a bit more detail: + + + (Behavior A) Users are _only_ interested in the sparse portion of the repo + +These folks might know there are other things in the repository, but +don't care. They are uninterested in other parts of the repository, and +only want to know about changes within their area of interest. Showing +them other files from history (e.g. from diff/log/grep/etc.) is a +usability annoyance, potentially a huge one since other changes in +history may dwarf the changes they are interested in. + +Some of these users also arrive at this usecase from wanting to use partial +clones together with sparse checkouts (in a way where they have downloaded +blobs within the sparse specification) and do disconnected development. +Not only do these users generally not care about other parts of the +repository, but consider it a blocker for Git commands to try to operate on +those. If commands attempt to access paths in history outside the sparsity +specification, then the partial clone will attempt to download additional +blobs on demand, fail, and then fail the user's command. (This may be +unavoidable in some cases, e.g. when `git merge` has non-trivial changes to +reconcile outside the sparse specification, but we should limit how often +users are forced to connect to the network.) + +Also, even for users using partial clones that do not mind being +always connected to the network, the need to download blobs as +side-effects of various other commands (such as the printed diffstat +after a merge or pull) can lead to worries about local repository size +growing unnecessarily[10]. + + (Behavior A*) Users are _only_ interested in the sparse portion of the repo + that they have downloaded so far (a variant on the first usecase) + +This variant is driven by folks who using partial clones together with +sparse checkouts and do disconnected development (so far sounding like a +subset of behavior A users) and doing so on very large repositories. The +reason for yet another variant is that downloading even just the blobs +through history within their sparse specification may be too much, so they +only download some. They would still like operations to succeed without +network connectivity, though, so things like `git log -S${SEARCH_TERM} -p` +or `git grep ${SEARCH_TERM} OLDREV ` would need to be prepared to provide +partial results that depend on what happens to have been downloaded. + +This variant could be viewed as Behavior A with the sparse specification +for history querying operations modified from "sparsity patterns" to +"sparsity patterns limited to the blobs we have already downloaded". + + (Behavior B) Users want a sparse working tree, but are working in a + larger whole + +Stolee described this usecase this way[11]: + +"I'm also focused on users that know that they are a part of a larger +whole. They know they are operating on a large repository but focus on +what they need to contribute their part. I expect multiple "roles" to +use very different, almost disjoint parts of the codebase. Some other +"architect" users operate across the entire tree or hop between different +sections of the codebase as necessary. In this situation, I'm wary of +scoping too many features to the sparse-checkout definition, especially +"git log," as it can be too confusing to have their view of the codebase +depend on your "point of view." + +People might also end up wanting behavior B due to complex inter-project +dependencies. The initial attempts to use sparse-checkouts usually involve +the directories you are directly interested in plus what those directories +depend upon within your repository. But there's a monkey wrench here: if +you have integration tests, they invert the hierarchy: to run integration +tests, you need not only what you are interested in and its in-tree +dependencies, you also need everything that depends upon what you are +interested in or that depends upon one of your dependencies...AND you need +all the in-tree dependencies of that expanded group. That can easily +change your sparse-checkout into a nearly dense one. + +Naturally, that tends to kill the benefits of sparse-checkouts. There are +a couple solutions to this conundrum: either avoid grabbing in-repo +dependencies (maybe have built versions of your in-repo dependencies pulled +from a CI cache somewhere), or say that users shouldn't run integration +tests directly and instead do it on the CI server when they submit a code +review. Or do both. Regardless of whether you stub out your in-repo +dependencies or stub out the things that depend upon you, there is +certainly a reason to want to query and be aware of those other stubbed-out +parts of the repository, particularly when the dependencies are complex or +change relatively frequently. Thus, for such uses, sparse-checkouts can be +used to limit what you directly build and modify, but these users do not +necessarily want their sparse checkout paths to limit their queries of +versions in history. + +Some people may also be interested in behavior B over behavior A simply as +a performance workaround: if they are using non-cone mode, then they have +to deal with its inherent quadratic performance problems. In that mode, +every operation that checks whether paths match the sparsity specification +can be expensive. As such, these users may only be willing to pay for +those expensive checks when interacting with the working copy, and may +prefer getting "unrelated" results from their history queries over having +slow commands. + + (Behavior C) sparse-checkout is an implementational detail supporting a + special VFS. + +This usecase goes slightly against the traditional definition of +sparse-checkout in that it actually tries to present a full or dense +checkout to the user. However, this usecase utilizes the same underlying +technical underpinnings in a new way which does provide some performance +advantages to users. The basic idea is that a company can have an in-house +Git-aware Virtual File System which pretends all files are present in the +working tree, by intercepting all file system accesses and using those to +fetch and write accessed files on demand via partial clones. The VFS uses +sparse-checkout to prevent Git from writing or paying attention to many +files, and manually updates the sparse checkout patterns itself based on +user access and modification of files in the working tree. See commit +ecc7c8841d ("repo_read_index: add config to expect files outside sparse +patterns", 2022-02-25) and the link at [17] for a more detailed description +of such a VFS. + +The biggest difference here is that users are completely unaware that the +sparse-checkout machinery is even in use. The sparse patterns are not +specified by the user but rather are under the complete control of the VFS +(and the patterns are updated frequently and dynamically by it). The user +will perceive the checkout as dense, and commands should thus behave as if +all files are present. + + +=== Usecases of primary concern === + +Most of the rest of this document will focus on Behavior A and Behavior +B. Some notes about the other two cases and why we are not focusing on +them: + + (Behavior A*) + +Supporting this usecase is estimated to be difficult and a lot of work. +There are no plans to implement it currently, but it may be a potential +future alternative. Knowing about the existence of additional alternatives +may affect our choice of command line flags (e.g. if we need tri-state or +quad-state flags rather than just binary flags), so it was still important +to at least note. + +Further, I believe the descriptions below for Behavior A are probably still +valid for this usecase, with the only exception being that it redefines the +sparse specification to restrict it to already-downloaded blobs. The hard +part is in making commands capable of respecting that modified definition. + + (Behavior C) + +This usecase violates some of the early sparse-checkout documented +assumptions (since files marked as SKIP_WORKTREE will be displayed to users +as present in the working tree). That violation may mean various +sparse-checkout related behaviors are not well suited to this usecase and +we may need tweaks -- to both documentation and code -- to handle it. +However, this usecase is also perhaps the simplest model to support in that +everything behaves like a dense checkout with a few exceptions (e.g. branch +checkouts and switches write fewer things, knowing the VFS will lazily +write the rest on an as-needed basis). + +Since there is no publically available VFS-related code for folks to try, +the number of folks who can test such a usecase is limited. + +The primary reason to note the Behavior C usecase is that as we fix things +to better support Behaviors A and B, there may be additional places where +we need to make tweaks allowing folks in this usecase to get the original +non-sparse treatment. For an example, see ecc7c8841d ("repo_read_index: +add config to expect files outside sparse patterns", 2022-02-25). The +secondary reason to note Behavior C, is so that folks taking advantage of +Behavior C do not assume they are part of the Behavior B camp and propose +patches that break things for the real Behavior B folks. + + +=== Oversimplified mental models === + +An oversimplification of the differences in the above behaviors is: + + Behavior A: Restrict worktree and history operations to sparse specification + Behavior B: Restrict worktree operations to sparse specification; have any + history operations work across all files + Behavior C: Do not restrict either worktree or history operations to the + sparse specification...with the exception of branch checkouts or + switches which avoid writing files that will match the index so + they can later lazily be populated instead. + + +=== Desired behavior === + +As noted previously, despite the simple idea of just working with a subset +of files, there are a range of different behavioral changes that need to be +made to different subcommands to work well with such a feature. See +[1,2,3,4,5,6,7,8,9,10] for various examples. In particular, at [2], we saw +that mere composition of other commands that individually worked correctly +in a sparse-checkout context did not imply that the higher level command +would work correctly; it sometimes requires further tweaks. So, +understanding these differences can be beneficial. + +* Commands behaving the same regardless of high-level use-case + + * commands that only look at files within the sparsity specification + + * diff (without --cached or REVISION arguments) + * grep (without --cached or REVISION arguments) + * diff-files + + * commands that restore files to the working tree that match sparsity + patterns, and remove unmodified files that don't match those + patterns: + + * switch + * checkout (the switch-like half) + * read-tree + * reset --hard + + * commands that write conflicted files to the working tree, but otherwise + will omit writing files to the working tree that do not match the + sparsity patterns: + + * merge + * rebase + * cherry-pick + * revert + + * `am` and `apply --cached` should probably be in this section but + are buggy (see the "Known bugs" section below) + + The behavior for these commands somewhat depends upon the merge + strategy being used: + * `ort` behaves as described above + * `recursive` tries to not vivify files unnecessarily, but does sometimes + vivify files without conflicts. + * `octopus` and `resolve` will always vivify any file changed in the merge + relative to the first parent, which is rather suboptimal. + + It is also important to note that these commands WILL update the index + outside the sparse specification relative to when the operation began, + BUT these commands often make a commit just before or after such that + by the end of the operation there is no change to the index outside the + sparse specification. Of course, if the operation hits conflicts or + does not make a commit, then these operations clearly can modify the + index outside the sparse specification. + + Finally, it is important to note that at least the first four of these + commands also try to remove differences between the sparse + specification and the sparsity patterns (much like the commands in the + previous section). + + * commands that always ignore sparsity since commits must be full-tree + + * archive + * bundle + * commit + * format-patch + * fast-export + * fast-import + * commit-tree + + * commands that write any modified file to the working tree (conflicted + or not, and whether those paths match sparsity patterns or not): + + * stash + * apply (without `--index` or `--cached`) + +* Commands that may slightly differ for behavior A vs. behavior B: + + Commands in this category behave mostly the same between the two + behaviors, but may differ in verbosity and types of warning and error + messages. + + * commands that make modifications to which files are tracked: + * add + * rm + * mv + * update-index + + The fact that files can move between the 'tracked' and 'untracked' + categories means some commands will have to treat untracked files + differently. But if we have to treat untracked files differently, + then additional commands may also need changes: + + * status + * clean + + In particular, `status` may need to report any untracked files outside + the sparsity specification as an erroneous condition (especially to + avoid the user trying to `git add` them, forcing `git add` to display + an error). + + It's not clear to me exactly how (or even if) `clean` would change, + but it's the other command that also affects untracked files. + + `update-index` may be slightly special. Its --[no-]skip-worktree flag + may need to ignore the sparse specification by its nature. Also, its + current --[no-]ignore-skip-worktree-entries default is totally bogus. + + * commands for manually tweaking paths in both the index and the working tree + * `restore` + * the restore-like half of `checkout` + + These commands should be similar to add/rm/mv in that they should + only operate on the sparse specification by default, and require a + special flag to operate on all files. + + Also, note that these commands currently have a number of issues (see + the "Known bugs" section below) + +* Commands that significantly differ for behavior A vs. behavior B: + + * commands that query history + * diff (with --cached or REVISION arguments) + * grep (with --cached or REVISION arguments) + * show (when given commit arguments) + * blame (only matters when one or more -C flags are passed) + * and annotate + * log + * whatchanged + * ls-files + * diff-index + * diff-tree + * ls-tree + + Note: for log and whatchanged, revision walking logic is unaffected + but displaying of patches is affected by scoping the command to the + sparse-checkout. (The fact that revision walking is unaffected is + why rev-list, shortlog, show-branch, and bisect are not in this + list.) + + ls-files may be slightly special in that e.g. `git ls-files -t` is + often used to see what is sparse and what is not. Perhaps -t should + always work on the full tree? + +* Commands I don't know how to classify + + * range-diff + + Is this like `log` or `format-patch`? + + * cherry + + See range-diff + +* Commands unaffected by sparse-checkouts + + * shortlog + * show-branch + * rev-list + * bisect + + * branch + * describe + * fetch + * gc + * init + * maintenance + * notes + * pull (merge & rebase have the necessary changes) + * push + * submodule + * tag + + * config + * filter-branch (works in separate checkout without sparse-checkout setup) + * pack-refs + * prune + * remote + * repack + * replace + + * bugreport + * count-objects + * fsck + * gitweb + * help + * instaweb + * merge-tree (doesn't touch worktree or index, and merges always compute full-tree) + * rerere + * verify-commit + * verify-tag + + * commit-graph + * hash-object + * index-pack + * mktag + * mktree + * multi-pack-index + * pack-objects + * prune-packed + * symbolic-ref + * unpack-objects + * update-ref + * write-tree (operates on index, possibly optimized to use sparse dir entries) + + * for-each-ref + * get-tar-commit-id + * ls-remote + * merge-base (merges are computed full tree, so merge base should be too) + * name-rev + * pack-redundant + * rev-parse + * show-index + * show-ref + * unpack-file + * var + * verify-pack + + * <Everything under 'Interacting with Others' in 'git help --all'> + * <Everything under 'Low-level...Syncing' in 'git help --all'> + * <Everything under 'Low-level...Internal Helpers' in 'git help --all'> + * <Everything under 'External commands' in 'git help --all'> + +* Commands that might be affected, but who cares? + + * merge-file + * merge-index + * gitk? + + +=== Behavior classes === + +From the above there are a few classes of behavior: + + * "restrict" + + Commands in this class only read or write files in the working tree + within the sparse specification. + + When moving to a new commit (e.g. switch, reset --hard), these commands + may update index files outside the sparse specification as of the start + of the operation, but by the end of the operation those index files + will match HEAD again and thus those files will again be outside the + sparse specification. + + When paths are explicitly specified, these paths are intersected with + the sparse specification and will only operate on such paths. + (e.g. `git restore [--staged] -- '*.png'`, `git reset -p -- '*.md'`) + + Some of these commands may also attempt, at the end of their operation, + to cull transient differences between the sparse specification and the + sparsity patterns (see "Sparse specification vs. sparsity patterns" for + details, but this basically means either removing unmodified files not + matching the sparsity patterns and marking those files as + SKIP_WORKTREE, or vivifying files that match the sparsity patterns and + marking those files as !SKIP_WORKTREE). + + * "restrict modulo conflicts" + + Commands in this class generally behave like the "restrict" class, + except that: + (1) they will ignore the sparse specification and write files with + conflicts to the working tree (thus temporarily expanding the + sparse specification to include such files.) + (2) they are grouped with commands which move to a new commit, since + they often create a commit and then move to it, even though we + know there are many exceptions to moving to the new commit. (For + example, the user may rebase a commit that becomes empty, or have + a cherry-pick which conflicts, or a user could run `merge + --no-commit`, and we also view `apply --index` kind of like `am + --no-commit`.) As such, these commands can make changes to index + files outside the sparse specification, though they'll mark such + files with SKIP_WORKTREE. + + * "restrict also specially applied to untracked files" + + Commands in this class generally behave like the "restrict" class, + except that they have to handle untracked files differently too, often + because these commands are dealing with files changing state between + 'tracked' and 'untracked'. Often, this may mean printing an error + message if the command had nothing to do, but the arguments may have + referred to files whose tracked-ness state could have changed were it + not for the sparsity patterns excluding them. + + * "no restrict" + + Commands in this class ignore the sparse specification entirely. + + * "restrict or no restrict dependent upon behavior A vs. behavior B" + + Commands in this class behave like "no restrict" for folks in the + behavior B camp, and like "restrict" for folks in the behavior A camp. + However, when behaving like "restrict" a warning of some sort might be + provided that history queries have been limited by the sparse-checkout + specification. + + +=== Subcommand-dependent defaults === + +Note that we have different defaults depending on the command for the +desired behavior : + + * Commands defaulting to "restrict": + * diff-files + * diff (without --cached or REVISION arguments) + * grep (without --cached or REVISION arguments) + * switch + * checkout (the switch-like half) + * reset (<commit>) + + * restore + * checkout (the restore-like half) + * checkout-index + * reset (with pathspec) + + This behavior makes sense; these interact with the working tree. + + * Commands defaulting to "restrict modulo conflicts": + * merge + * rebase + * cherry-pick + * revert + + * am + * apply --index (which is kind of like an `am --no-commit`) + + * read-tree (especially with -m or -u; is kind of like a --no-commit merge) + * reset (<tree-ish>, due to similarity to read-tree) + + These also interact with the working tree, but require slightly + different behavior either so that (a) conflicts can be resolved or (b) + because they are kind of like a merge-without-commit operation. + + (See also the "Known bugs" section below regarding `am` and `apply`) + + * Commands defaulting to "no restrict": + * archive + * bundle + * commit + * format-patch + * fast-export + * fast-import + * commit-tree + + * stash + * apply (without `--index`) + + These have completely different defaults and perhaps deserve the most + detailed explanation: + + In the case of commands in the first group (format-patch, + fast-export, bundle, archive, etc.), these are commands for + communicating history, which will be broken if they restrict to a + subset of the repository. As such, they operate on full paths and + have no `--restrict` option for overriding. Some of these commands may + take paths for manually restricting what is exported, but it needs to + be very explicit. + + In the case of stash, it needs to vivify files to avoid losing the + user's changes. + + In the case of apply without `--index`, that command needs to update + the working tree without the index (or the index without the working + tree if `--cached` is passed), and if we restrict those updates to the + sparse specification then we'll lose changes from the user. + + * Commands defaulting to "restrict also specially applied to untracked files": + * add + * rm + * mv + * update-index + * status + * clean (?) + + Our original implementation for the first three of these commands was + "no restrict", but it had some severe usability issues: + * `git add <somefile>` if honored and outside the sparse + specification, can result in the file randomly disappearing later + when some subsequent command is run (since various commands + automatically clean up unmodified files outside the sparse + specification). + * `git rm '*.jpg'` could very negatively surprise users if it deletes + files outside the range of the user's interest. + * `git mv` has similar surprises when moving into or out of the cone, + so best to restrict by default + + So, we switched `add` and `rm` to default to "restrict", which made + usability problems much less severe and less frequent, but we still got + complaints because commands like: + git add <file-outside-sparse-specification> + git rm <file-outside-sparse-specification> + would silently do nothing. We should instead print an error in those + cases to get usability right. + + update-index needs to be updated to match, and status and maybe clean + also need to be updated to specially handle untracked paths. + + There may be a difference in here between behavior A and behavior B in + terms of verboseness of errors or additional warnings. + + * Commands falling under "restrict or no restrict dependent upon behavior + A vs. behavior B" + + * diff (with --cached or REVISION arguments) + * grep (with --cached or REVISION arguments) + * show (when given commit arguments) + * blame (only matters when one or more -C flags passed) + * and annotate + * log + * and variants: shortlog, gitk, show-branch, whatchanged, rev-list + * ls-files + * diff-index + * diff-tree + * ls-tree + + For now, we default to behavior B for these, which want a default of + "no restrict". + + Note that two of these commands -- diff and grep -- also appeared in a + different list with a default of "restrict", but only when limited to + searching the working tree. The working tree vs. history distinction + is fundamental in how behavior B operates, so this is expected. Note, + though, that for diff and grep with --cached, when doing "restrict" + behavior, the difference between sparse specification and sparsity + patterns is important to handle. + + "restrict" may make more sense as the long term default for these[12]. + Also, supporting "restrict" for these commands might be a fair amount + of work to implement, meaning it might be implemented over multiple + releases. If that behavior were the default in the commands that + supported it, that would force behavior B users to need to learn to + slowly add additional flags to their commands, depending on git + version, to get the behavior they want. That gradual switchover would + be painful, so we should avoid it at least until it's fully + implemented. + + +=== Sparse specification vs. sparsity patterns === + +In a well-behaved situation, the sparse specification is given directly +by the $GIT_DIR/info/sparse-checkout file. However, it can transiently +diverge for a few reasons: + + * needing to resolve conflicts (merging will vivify conflicted files) + * running Git commands that implicitly vivify files (e.g. "git stash apply") + * running Git commands that explicitly vivify files (e.g. "git checkout + --ignore-skip-worktree-bits FILENAME") + * other commands that write to these files (perhaps a user copies it + from elsewhere) + +For the last item, note that we do automatically clear the SKIP_WORKTREE +bit for files that are present in the working tree. This has been true +since 82386b4496 ("Merge branch 'en/present-despite-skipped'", +2022-03-09) + +However, such a situation is transient because: + + * Such transient differences can and will be automatically removed as + a side-effect of commands which call unpack_trees() (checkout, + merge, reset, etc.). + * Users can also request such transient differences be corrected via + running `git sparse-checkout reapply`. Various places recommend + running that command. + * Additional commands are also welcome to implicitly fix these + differences; we may add more in the future. + +While we avoid dropping unstaged changes or files which have conflicts, +we otherwise aggressively try to fix these transient differences. If +users want these differences to persist, they should run the `set` or +`add` subcommands of `git sparse-checkout` to reflect their intended +sparse specification. + +However, when we need to do a query on history restricted to the +"relevant subset of files" such a transiently expanded sparse +specification is ignored. There are a couple reasons for this: + + * The behavior wanted when doing something like + git grep expression REVISION + is roughly what the users would expect from + git checkout REVISION && git grep expression + (modulo a "REVISION:" prefix), which has a couple ramifications: + + * REVISION may have paths not in the current index, so there is no + path we can consult for a SKIP_WORKTREE setting for those paths. + + * Since `checkout` is one of those commands that tries to remove + transient differences in the sparse specification, it makes sense + to use the corrected sparse specification + (i.e. $GIT_DIR/info/sparse-checkout) rather than attempting to + consult SKIP_WORKTREE anyway. + +So, a transiently expanded (or restricted) sparse specification applies to +the working tree, but not to history queries where we always use the +sparsity patterns. (See [16] for an early discussion of this.) + +Similar to a transiently expanded sparse specification of the working tree +based on additional files being present in the working tree, we also need +to consider additional files being modified in the index. In particular, +if the user has staged changes to files (relative to HEAD) that do not +match the sparsity patterns, and the file is not present in the working +tree, we still want to consider the file part of the sparse specification +if we are specifically performing a query related to the index (e.g. git +diff --cached [REVISION], git diff-index [REVISION], git restore --staged +--source=REVISION -- PATHS, etc.) Note that a transiently expanded sparse +specification for the index usually only matters under behavior A, since +under behavior B index operations are lumped with history and tend to +operate full-tree. + + +=== Implementation Questions === + + * Do the options --scope={sparse,all} sound good to others? Are there better + options? + * Names in use, or appearing in patches, or previously suggested: + * --sparse/--dense + * --ignore-skip-worktree-bits + * --ignore-skip-worktree-entries + * --ignore-sparsity + * --[no-]restrict-to-sparse-paths + * --full-tree/--sparse-tree + * --[no-]restrict + * --scope={sparse,all} + * --focus/--unfocus + * --limit/--unlimited + * Rationale making me lean slightly towards --scope={sparse,all}: + * We want a name that works for many commands, so we need a name that + does not conflict + * We know that we have more than two possible usecases, so it is best + to avoid a flag that appears to be binary. + * --scope={sparse,all} isn't overly long and seems relatively + explanatory + * `--sparse`, as used in add/rm/mv, is totally backwards for + grep/log/etc. Changing the meaning of `--sparse` for these + commands would fix the backwardness, but possibly break existing + scripts. Using a new name pairing would allow us to treat + `--sparse` in these commands as a deprecated alias. + * There is a different `--sparse`/`--dense` pair for commands using + revision machinery, so using that naming might cause confusion + * There is also a `--sparse` in both pack-objects and show-branch, which + don't conflict but do suggest that `--sparse` is overloaded + * The name --ignore-skip-worktree-bits is a double negative, is + quite a mouthful, refers to an implementation detail that many + users may not be familiar with, and we'd need a negation for it + which would probably be even more ridiculously long. (But we + can make --ignore-skip-worktree-bits a deprecated alias for + --no-restrict.) + + * If a config option is added (sparse.scope?) what should the values and + description be? "sparse" (behavior A), "worktree-sparse-history-dense" + (behavior B), "dense" (behavior C)? There's a risk of confusion, + because even for Behaviors A and B we want some commands to be + full-tree and others to operate sparsely, so the wording may need to be + more tied to the usecases and somehow explain that. Also, right now, + the primary difference we are focusing is just the history-querying + commands (log/diff/grep). Previous config suggestion here: [13] + + * Is `--no-expand` a good alias for ls-files's `--sparse` option? + (`--sparse` does not map to either `--scope=sparse` or `--scope=all`, + because in non-cone mode it does nothing and in cone-mode it shows the + sparse directory entries which are technically outside the sparse + specification) + + * Under Behavior A: + * Does ls-files' `--no-expand` override the default `--scope=all`, or + does it need an extra flag? + * Does ls-files' `-t` option imply `--scope=all`? + * Does update-index's `--[no-]skip-worktree` option imply `--scope=all`? + + * sparse-checkout: once behavior A is fully implemented, should we take + an interim measure to ease people into switching the default? Namely, + if folks are not already in a sparse checkout, then require + `sparse-checkout init/set` to take a + `--set-scope=(sparse|worktree-sparse-history-dense|dense)` flag (which + would set sparse.scope according to the setting given), and throw an + error if the flag is not provided? That error would be a great place + to warn folks that the default may change in the future, and get them + used to specifying what they want so that the eventual default switch + is seamless for them. + + +=== Implementation Goals/Plans === + + * Get buy-in on this document in general. + + * Figure out answers to the 'Implementation Questions' sections (above) + + * Fix bugs in the 'Known bugs' section (below) + + * Provide some kind of method for backfilling the blobs within the sparse + specification in a partial clone + + [Below here is kind of spitballing since the first two haven't been resolved] + + * update-index: flip the default to --no-ignore-skip-worktree-entries, + nuke this stupid "Oh, there's a bug? Let me add a flag to let users + request that they not trigger this bug." flag + + * Flags & Config + * Make `--sparse` in add/rm/mv a deprecated alias for `--scope=all` + * Make `--ignore-skip-worktree-bits` in checkout-index/checkout/restore + a deprecated aliases for `--scope=all` + * Create config option (sparse.scope?), tie it to the "Cliff notes" + overview + + * Add --scope=sparse (and --scope=all) flag to each of the history querying + commands. IMPORTANT: make sure diff machinery changes don't mess with + format-patch, fast-export, etc. + +=== Known bugs === + +This list used to be a lot longer (see e.g. [1,2,3,4,5,6,7,8,9]), but we've +been working on it. + +0. Behavior A is not well supported in Git. (Behavior B didn't used to + be either, but was the easier of the two to implement.) + +1. am and apply: + + apply, without `--index` or `--cached`, relies on files being present + in the working copy, and also writes to them unconditionally. As + such, it should first check for the files' presence, and if found to + be SKIP_WORKTREE, then clear the bit and vivify the paths, then do + its work. Currently, it just throws an error. + + apply, with either `--cached` or `--index`, will not preserve the + SKIP_WORKTREE bit. This is fine if the file has conflicts, but + otherwise SKIP_WORKTREE bits should be preserved for --cached and + probably also for --index. + + am, if there are no conflicts, will vivify files and fail to preserve + the SKIP_WORKTREE bit. If there are conflicts and `-3` is not + specified, it will vivify files and then complain the patch doesn't + apply. If there are conflicts and `-3` is specified, it will vivify + files and then complain that those vivified files would be + overwritten by merge. + +2. reset --hard: + + reset --hard provides confusing error message (works correctly, but + misleads the user into believing it didn't): + + $ touch addme + $ git add addme + $ git ls-files -t + H addme + H tracked + S tracked-but-maybe-skipped + $ git reset --hard # usually works great + error: Path 'addme' not uptodate; will not remove from working tree. + HEAD is now at bdbbb6f third + $ git ls-files -t + H tracked + S tracked-but-maybe-skipped + $ ls -1 + tracked + + `git reset --hard` DID remove addme from the index and the working tree, contrary + to the error message, but in line with how reset --hard should behave. + +3. read-tree + + `read-tree` doesn't apply the 'SKIP_WORKTREE' bit to *any* of the + entries it reads into the index, resulting in all your files suddenly + appearing to be "deleted". + +4. Checkout, restore: + + These command do not handle path & revision arguments appropriately: + + $ ls + tracked + $ git ls-files -t + H tracked + S tracked-but-maybe-skipped + $ git status --porcelain + $ git checkout -- '*skipped' + error: pathspec '*skipped' did not match any file(s) known to git + $ git ls-files -- '*skipped' + tracked-but-maybe-skipped + $ git checkout HEAD -- '*skipped' + error: pathspec '*skipped' did not match any file(s) known to git + $ git ls-tree HEAD | grep skipped + 100644 blob 276f5a64354b791b13840f02047738c77ad0584f tracked-but-maybe-skipped + $ git status --porcelain + $ git checkout HEAD~1 -- '*skipped' + $ git ls-files -t + H tracked + H tracked-but-maybe-skipped + $ git status --porcelain + M tracked-but-maybe-skipped + $ git checkout HEAD -- '*skipped' + $ git status --porcelain + $ + + Note that checkout without a revision (or restore --staged) fails to + find a file to restore from the index, even though ls-files shows + such a file certainly exists. + + Similar issues occur with HEAD (--source=HEAD in restore's case), + but suddenly works when HEAD~1 is specified. And then after that it + will work with HEAD specified, even though it didn't before. + + Directories are also an issue: + + $ git sparse-checkout set nomatches + $ git status + On branch main + You are in a sparse checkout with 0% of tracked files present. + + nothing to commit, working tree clean + $ git checkout . + error: pathspec '.' did not match any file(s) known to git + $ git checkout HEAD~1 . + Updated 1 path from 58916d9 + $ git ls-files -t + S tracked + H tracked-but-maybe-skipped + +5. checkout and restore --staged, continued: + + These commands do not correctly scope operations to the sparse + specification, and make it worse by not setting important SKIP_WORKTREE + bits: + + $ git restore --source OLDREV --staged outside-sparse-cone/ + $ git status --porcelain + MD outside-sparse-cone/file1 + MD outside-sparse-cone/file2 + MD outside-sparse-cone/file3 + + We can add a --scope=all mode to `git restore` to let it operate outside + the sparse specification, but then it will be important to set the + SKIP_WORKTREE bits appropriately. + +6. Performance issues; see: + https://lore.kernel.org/git/CABPp-BEkJQoKZsQGCYioyga_uoDQ6iBeW+FKr8JhyuuTMK1RDw@mail.gmail.com/ + + +=== Reference Emails === + +Emails that detail various bugs we've had in sparse-checkout: + +[1] (Original descriptions of behavior A & behavior B) + https://lore.kernel.org/git/CABPp-BGJ_Nvi5TmgriD9Bh6eNXE2EDq2f8e8QKXAeYG3BxZafA@mail.gmail.com/ +[2] (Fix stash applications in sparse checkouts; bugs from behavioral differences) + https://lore.kernel.org/git/ccfedc7140dbf63ba26a15f93bd3885180b26517.1606861519.git.gitgitgadget@gmail.com/ +[3] (Present-despite-skipped entries) + https://lore.kernel.org/git/11d46a399d26c913787b704d2b7169cafc28d639.1642175983.git.gitgitgadget@gmail.com/ +[4] (Clone --no-checkout interaction) + https://lore.kernel.org/git/pull.801.v2.git.git.1591324899170.gitgitgadget@gmail.com/ (clone --no-checkout) +[5] (The need for update_sparsity() and avoiding `read-tree -mu HEAD`) + https://lore.kernel.org/git/3a1f084641eb47515b5a41ed4409a36128913309.1585270142.git.gitgitgadget@gmail.com/ +[6] (SKIP_WORKTREE is advisory, not mandatory) + https://lore.kernel.org/git/844306c3e86ef67591cc086decb2b760e7d710a3.1585270142.git.gitgitgadget@gmail.com/ +[7] (`worktree add` should copy sparsity settings from current worktree) + https://lore.kernel.org/git/c51cb3714e7b1d2f8c9370fe87eca9984ff4859f.1644269584.git.gitgitgadget@gmail.com/ +[8] (Avoid negative surprises in add, rm, and mv) + https://lore.kernel.org/git/cover.1617914011.git.matheus.bernardino@usp.br/ + https://lore.kernel.org/git/pull.1018.v4.git.1632497954.gitgitgadget@gmail.com/ +[9] (Move from out-of-cone to in-cone) + https://lore.kernel.org/git/20220630023737.473690-6-shaoxuan.yuan02@gmail.com/ + https://lore.kernel.org/git/20220630023737.473690-4-shaoxuan.yuan02@gmail.com/ +[10] (Unnecessarily downloading objects outside sparse specification) + https://lore.kernel.org/git/CAOLTT8QfwOi9yx_qZZgyGa8iL8kHWutEED7ok_jxwTcYT_hf9Q@mail.gmail.com/ + +[11] (Stolee's comments on high-level usecases) + https://lore.kernel.org/git/1a1e33f6-3514-9afc-0a28-5a6b85bd8014@gmail.com/ + +[12] Others commenting on eventually switching default to behavior A: + * https://lore.kernel.org/git/xmqqh719pcoo.fsf@gitster.g/ + * https://lore.kernel.org/git/xmqqzgeqw0sy.fsf@gitster.g/ + * https://lore.kernel.org/git/a86af661-cf58-a4e5-0214-a67d3a794d7e@github.com/ + +[13] Previous config name suggestion and description + * https://lore.kernel.org/git/CABPp-BE6zW0nJSStcVU=_DoDBnPgLqOR8pkTXK3dW11=T01OhA@mail.gmail.com/ + +[14] Tangential issue: switch to cone mode as default sparse specification mechanism: + https://lore.kernel.org/git/a1b68fd6126eb341ef3637bb93fedad4309b36d0.1650594746.git.gitgitgadget@gmail.com/ + +[15] Lengthy email on grep behavior, covering what should be searched: + * https://lore.kernel.org/git/CABPp-BGVO3QdbfE84uF_3QDF0-y2iHHh6G5FAFzNRfeRitkuHw@mail.gmail.com/ + +[16] Email explaining sparsity patterns vs. SKIP_WORKTREE and history operations, + search for the parenthetical comment starting "We do not check". + https://lore.kernel.org/git/CABPp-BFsCPPNOZ92JQRJeGyNd0e-TCW-LcLyr0i_+VSQJP+GCg@mail.gmail.com/ + +[17] https://lore.kernel.org/git/20220207190320.2960362-1-jonathantanmy@google.com/ @@ -133,10 +133,6 @@ Issues of note: you are using libcurl older than 7.34.0. Otherwise you can use NO_OPENSSL without losing git-imap-send. - By default, git uses OpenSSL for SHA1 but it will use its own - library (inspired by Mozilla's) with either NO_OPENSSL or - BLK_SHA1. - - "libcurl" library is used for fetching and pushing repositories over http:// or https://, as well as by git-imap-send if the curl version is >= 7.34.0. If you do @@ -4,8 +4,20 @@ all:: # Import tree-wide shared Makefile behavior and libraries include shared.mak +# == Makefile defines == +# +# These defines change the behavior of the Makefile itself, but have +# no impact on what it builds: +# # Define V=1 to have a more verbose compile. # +# == Portability and optional library defines == +# +# These defines indicate what Git can expect from the OS, what +# libraries are available etc. Much of this is auto-detected in +# config.mak.uname, or in configure.ac when using the optional "make +# configure && ./configure" (see INSTALL). +# # Define SHELL_PATH to a POSIX shell if your /bin/sh is broken. # # Define SANE_TOOL_PATH to a colon-separated list of paths to prepend @@ -30,68 +42,8 @@ include shared.mak # # Define NO_OPENSSL environment variable if you do not have OpenSSL. # -# Define USE_LIBPCRE if you have and want to use libpcre. Various -# commands such as log and grep offer runtime options to use -# Perl-compatible regular expressions instead of standard or extended -# POSIX regular expressions. -# -# Only libpcre version 2 is supported. USE_LIBPCRE2 is a synonym for -# USE_LIBPCRE, support for the old USE_LIBPCRE1 has been removed. -# -# Define LIBPCREDIR=/foo/bar if your PCRE header and library files are -# in /foo/bar/include and /foo/bar/lib directories. -# # Define HAVE_ALLOCA_H if you have working alloca(3) defined in that header. # -# Define NO_CURL if you do not have libcurl installed. git-http-fetch and -# git-http-push are not built, and you cannot use http:// and https:// -# transports (neither smart nor dumb). -# -# Define CURLDIR=/foo/bar if your curl header and library files are in -# /foo/bar/include and /foo/bar/lib directories. -# -# Define CURL_CONFIG to curl's configuration program that prints information -# about the library (e.g., its version number). The default is 'curl-config'. -# -# Define CURL_LDFLAGS to specify flags that you need to link when using libcurl, -# if you do not want to rely on the libraries provided by CURL_CONFIG. The -# default value is a result of `curl-config --libs`. An example value for -# CURL_LDFLAGS is as follows: -# -# CURL_LDFLAGS=-lcurl -# -# Define NO_EXPAT if you do not have expat installed. git-http-push is -# not built, and you cannot push using http:// and https:// transports (dumb). -# -# Define EXPATDIR=/foo/bar if your expat header and library files are in -# /foo/bar/include and /foo/bar/lib directories. -# -# Define EXPAT_NEEDS_XMLPARSE_H if you have an old version of expat (e.g., -# 1.1 or 1.2) that provides xmlparse.h instead of expat.h. -# -# Define NO_GETTEXT if you don't want Git output to be translated. -# A translated Git requires GNU libintl or another gettext implementation, -# plus libintl-perl at runtime. -# -# Define USE_GETTEXT_SCHEME and set it to 'fallthrough', if you don't trust -# the installed gettext translation of the shell scripts output. -# -# Define HAVE_LIBCHARSET_H if you haven't set NO_GETTEXT and you can't -# trust the langinfo.h's nl_langinfo(CODESET) function to return the -# current character set. GNU and Solaris have a nl_langinfo(CODESET), -# FreeBSD can use either, but MinGW and some others need to use -# libcharset.h's locale_charset() instead. -# -# Define CHARSET_LIB to the library you need to link with in order to -# use locale_charset() function. On some platforms this needs to set to -# -lcharset, on others to -liconv . -# -# Define LIBC_CONTAINS_LIBINTL if your gettext implementation doesn't -# need -lintl when linking. -# -# Define NO_MSGFMT_EXTENDED_OPTIONS if your implementation of msgfmt -# doesn't support GNU extensions like --check and --statistics -# # Define HAVE_PATHS_H if you have paths.h and want to use the default PATH # it specifies. # @@ -152,39 +104,6 @@ include shared.mak # and do not want to use Apple's CommonCrypto library. This allows you # to provide your own OpenSSL library, for example from MacPorts. # -# Define BLK_SHA1 environment variable to make use of the bundled -# optimized C SHA1 routine. -# -# Define DC_SHA1 to unconditionally enable the collision-detecting sha1 -# algorithm. This is slower, but may detect attempted collision attacks. -# Takes priority over other *_SHA1 knobs. -# -# Define DC_SHA1_EXTERNAL in addition to DC_SHA1 if you want to build / link -# git with the external SHA1 collision-detect library. -# Without this option, i.e. the default behavior is to build git with its -# own built-in code (or submodule). -# -# Define DC_SHA1_SUBMODULE in addition to DC_SHA1 to use the -# sha1collisiondetection shipped as a submodule instead of the -# non-submodule copy in sha1dc/. This is an experimental option used -# by the git project to migrate to using sha1collisiondetection as a -# submodule. -# -# Define OPENSSL_SHA1 environment variable when running make to link -# with the SHA1 routine from openssl library. -# -# Define SHA1_MAX_BLOCK_SIZE to limit the amount of data that will be hashed -# in one call to the platform's SHA1_Update(). e.g. APPLE_COMMON_CRYPTO -# wants 'SHA1_MAX_BLOCK_SIZE=1024L*1024L*1024L' defined. -# -# Define BLK_SHA256 to use the built-in SHA-256 routines. -# -# Define NETTLE_SHA256 to use the SHA-256 routines in libnettle. -# -# Define GCRYPT_SHA256 to use the SHA-256 routines in libgcrypt. -# -# Define OPENSSL_SHA256 to use the SHA-256 routines in OpenSSL. -# # Define NEEDS_CRYPTO_WITH_SSL if you need -lcrypto when using -lssl (Darwin). # # Define NEEDS_SSL_WITH_CRYPTO if you need -lssl when using -lcrypto (Darwin). @@ -490,6 +409,151 @@ include shared.mak # to the "<name>" of the corresponding `compat/fsmonitor/fsm-settings-<name>.c` # that implements the `fsm_os_settings__*()` routines. # +# === Optional library: libintl === +# +# Define NO_GETTEXT if you don't want Git output to be translated. +# A translated Git requires GNU libintl or another gettext implementation, +# plus libintl-perl at runtime. +# +# Define USE_GETTEXT_SCHEME and set it to 'fallthrough', if you don't trust +# the installed gettext translation of the shell scripts output. +# +# Define HAVE_LIBCHARSET_H if you haven't set NO_GETTEXT and you can't +# trust the langinfo.h's nl_langinfo(CODESET) function to return the +# current character set. GNU and Solaris have a nl_langinfo(CODESET), +# FreeBSD can use either, but MinGW and some others need to use +# libcharset.h's locale_charset() instead. +# +# Define CHARSET_LIB to the library you need to link with in order to +# use locale_charset() function. On some platforms this needs to set to +# -lcharset, on others to -liconv . +# +# Define LIBC_CONTAINS_LIBINTL if your gettext implementation doesn't +# need -lintl when linking. +# +# Define NO_MSGFMT_EXTENDED_OPTIONS if your implementation of msgfmt +# doesn't support GNU extensions like --check and --statistics +# +# === Optional library: libexpat === +# +# Define NO_EXPAT if you do not have expat installed. git-http-push is +# not built, and you cannot push using http:// and https:// transports (dumb). +# +# Define EXPATDIR=/foo/bar if your expat header and library files are in +# /foo/bar/include and /foo/bar/lib directories. +# +# Define EXPAT_NEEDS_XMLPARSE_H if you have an old version of expat (e.g., +# 1.1 or 1.2) that provides xmlparse.h instead of expat.h. + +# === Optional library: libcurl === +# +# Define NO_CURL if you do not have libcurl installed. git-http-fetch and +# git-http-push are not built, and you cannot use http:// and https:// +# transports (neither smart nor dumb). +# +# Define CURLDIR=/foo/bar if your curl header and library files are in +# /foo/bar/include and /foo/bar/lib directories. +# +# Define CURL_CONFIG to curl's configuration program that prints information +# about the library (e.g., its version number). The default is 'curl-config'. +# +# Define CURL_LDFLAGS to specify flags that you need to link when using libcurl, +# if you do not want to rely on the libraries provided by CURL_CONFIG. The +# default value is a result of `curl-config --libs`. An example value for +# CURL_LDFLAGS is as follows: +# +# CURL_LDFLAGS=-lcurl +# +# === Optional library: libpcre2 === +# +# Define USE_LIBPCRE if you have and want to use libpcre. Various +# commands such as log and grep offer runtime options to use +# Perl-compatible regular expressions instead of standard or extended +# POSIX regular expressions. +# +# Only libpcre version 2 is supported. USE_LIBPCRE2 is a synonym for +# USE_LIBPCRE, support for the old USE_LIBPCRE1 has been removed. +# +# Define LIBPCREDIR=/foo/bar if your PCRE header and library files are +# in /foo/bar/include and /foo/bar/lib directories. +# +# == SHA-1 and SHA-256 defines == +# +# === SHA-1 backend === +# +# ==== Security ==== +# +# Due to the SHAttered (https://shattered.io) attack vector on SHA-1 +# it's strongly recommended to use the sha1collisiondetection +# counter-cryptanalysis library for SHA-1 hashing. +# +# If you know that you can trust the repository contents, or where +# potential SHA-1 attacks are otherwise mitigated the other backends +# listed in "SHA-1 implementations" are faster than +# sha1collisiondetection. +# +# ==== Default SHA-1 backend ==== +# +# If no *_SHA1 backend is picked, the first supported one listed in +# "SHA-1 implementations" will be picked. +# +# ==== Options common to all SHA-1 implementations ==== +# +# Define SHA1_MAX_BLOCK_SIZE to limit the amount of data that will be hashed +# in one call to the platform's SHA1_Update(). e.g. APPLE_COMMON_CRYPTO +# wants 'SHA1_MAX_BLOCK_SIZE=1024L*1024L*1024L' defined. +# +# ==== SHA-1 implementations ==== +# +# Define OPENSSL_SHA1 to link to the SHA-1 routines from the OpenSSL +# library. +# +# Define BLK_SHA1 to make use of optimized C SHA-1 routines bundled +# with git (in the block-sha1/ directory). +# +# Define NO_APPLE_COMMON_CRYPTO on OSX to opt-out of using the +# "APPLE_COMMON_CRYPTO" backend for SHA-1, which is currently the +# default on that OS. On macOS 01.4 (Tiger) or older, +# NO_APPLE_COMMON_CRYPTO is defined by default. +# +# If don't enable any of the *_SHA1 settings in this section, Git will +# default to its built-in sha1collisiondetection library, which is a +# collision-detecting sha1 This is slower, but may detect attempted +# collision attacks. +# +# ==== Options for the sha1collisiondetection library ==== +# +# Define DC_SHA1_EXTERNAL if you want to build / link +# git with the external SHA1 collision-detect library. +# Without this option, i.e. the default behavior is to build git with its +# own built-in code (or submodule). +# +# Define DC_SHA1_SUBMODULE to use the +# sha1collisiondetection shipped as a submodule instead of the +# non-submodule copy in sha1dc/. This is an experimental option used +# by the git project to migrate to using sha1collisiondetection as a +# submodule. +# +# === SHA-256 backend === +# +# ==== Security ==== +# +# Unlike SHA-1 the SHA-256 algorithm does not suffer from any known +# vulnerabilities, so any implementation will do. +# +# ==== SHA-256 implementations ==== +# +# Define OPENSSL_SHA256 to use the SHA-256 routines in OpenSSL. +# +# Define NETTLE_SHA256 to use the SHA-256 routines in libnettle. +# +# Define GCRYPT_SHA256 to use the SHA-256 routines in libgcrypt. +# +# If don't enable any of the *_SHA256 settings in this section, Git +# will default to its built-in sha256 implementation. +# +# == DEVELOPER defines == +# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -723,6 +787,7 @@ TEST_BUILTINS_OBJS += test-advise.o TEST_BUILTINS_OBJS += test-bitmap.o TEST_BUILTINS_OBJS += test-bloom.o TEST_BUILTINS_OBJS += test-bundle-uri.o +TEST_BUILTINS_OBJS += test-cache-tree.o TEST_BUILTINS_OBJS += test-chmtime.o TEST_BUILTINS_OBJS += test-config.o TEST_BUILTINS_OBJS += test-crontab.o @@ -1826,7 +1891,6 @@ ifdef APPLE_COMMON_CRYPTO COMPAT_CFLAGS += -DCOMMON_DIGEST_FOR_OPENSSL BASIC_CFLAGS += -DSHA1_APPLE else - DC_SHA1 := YesPlease BASIC_CFLAGS += -DSHA1_DC LIB_OBJS += sha1dc_git.o ifdef DC_SHA1_EXTERNAL @@ -2989,7 +3053,6 @@ GIT-BUILD-OPTIONS: FORCE @echo NO_REGEX=\''$(subst ','\'',$(subst ','\'',$(NO_REGEX)))'\' >>$@+ @echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@+ @echo PAGER_ENV=\''$(subst ','\'',$(subst ','\'',$(PAGER_ENV)))'\' >>$@+ - @echo DC_SHA1=\''$(subst ','\'',$(subst ','\'',$(DC_SHA1)))'\' >>$@+ @echo SANITIZE_LEAK=\''$(subst ','\'',$(subst ','\'',$(SANITIZE_LEAK)))'\' >>$@+ @echo SANITIZE_ADDRESS=\''$(subst ','\'',$(subst ','\'',$(SANITIZE_ADDRESS)))'\' >>$@+ @echo X=\'$(X)\' >>$@+ diff --git a/add-interactive.c b/add-interactive.c index f071b2a1b4..ecc5ae1b24 100644 --- a/add-interactive.c +++ b/add-interactive.c @@ -997,18 +997,17 @@ static int run_diff(struct add_i_state *s, const struct pathspec *ps, count = list_and_choose(s, files, opts); opts->flags = 0; if (count > 0) { - struct strvec args = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; - strvec_pushl(&args, "git", "diff", "-p", "--cached", + strvec_pushl(&cmd.args, "git", "diff", "-p", "--cached", oid_to_hex(!is_initial ? &oid : s->r->hash_algo->empty_tree), "--", NULL); for (i = 0; i < files->items.nr; i++) if (files->selected[i]) - strvec_push(&args, + strvec_push(&cmd.args, files->items.items[i].string); - res = run_command_v_opt(args.v, 0); - strvec_clear(&args); + res = run_command(&cmd); } putchar('\n'); diff --git a/archive-tar.c b/archive-tar.c index 3e4822b684..f8fad2946e 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -498,6 +498,7 @@ static int write_tar_filter_archive(const struct archiver *ar, strvec_push(&filter.args, cmd.buf); filter.use_shell = 1; filter.in = -1; + filter.silent_exec_failure = 1; if (start_command(&filter) < 0) die_errno(_("unable to start '%s' filter"), cmd.buf); @@ -22,8 +22,6 @@ static struct oid_array skipped_revs; static struct object_id *current_bad_oid; -static const char *argv_checkout[] = {"checkout", "-q", NULL, "--", NULL}; - static const char *term_bad; static const char *term_good; @@ -729,20 +727,22 @@ static int is_expected_rev(const struct object_id *oid) enum bisect_error bisect_checkout(const struct object_id *bisect_rev, int no_checkout) { - char bisect_rev_hex[GIT_MAX_HEXSZ + 1]; struct commit *commit; struct pretty_print_context pp = {0}; struct strbuf commit_msg = STRBUF_INIT; - oid_to_hex_r(bisect_rev_hex, bisect_rev); update_ref(NULL, "BISECT_EXPECTED_REV", bisect_rev, NULL, 0, UPDATE_REFS_DIE_ON_ERR); - argv_checkout[2] = bisect_rev_hex; if (no_checkout) { update_ref(NULL, "BISECT_HEAD", bisect_rev, NULL, 0, UPDATE_REFS_DIE_ON_ERR); } else { - if (run_command_v_opt(argv_checkout, RUN_GIT_CMD)) + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "checkout", "-q", + oid_to_hex(bisect_rev), "--", NULL); + if (run_command(&cmd)) /* * Errors in `run_command()` itself, signaled by res < 0, * and errors in the child process, signaled by res > 0 diff --git a/builtin/add.c b/builtin/add.c index f84372964c..626c71ec6a 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -240,8 +240,8 @@ static int refresh(int verbose, const struct pathspec *pathspec) int run_add_interactive(const char *revision, const char *patch_mode, const struct pathspec *pathspec) { - int status, i; - struct strvec argv = STRVEC_INIT; + int i; + struct child_process cmd = CHILD_PROCESS_INIT; int use_builtin_add_i = git_env_bool("GIT_TEST_ADD_I_USE_BUILTIN", -1); @@ -272,19 +272,18 @@ int run_add_interactive(const char *revision, const char *patch_mode, return !!run_add_p(the_repository, mode, revision, pathspec); } - strvec_push(&argv, "add--interactive"); + strvec_push(&cmd.args, "add--interactive"); if (patch_mode) - strvec_push(&argv, patch_mode); + strvec_push(&cmd.args, patch_mode); if (revision) - strvec_push(&argv, revision); - strvec_push(&argv, "--"); + strvec_push(&cmd.args, revision); + strvec_push(&cmd.args, "--"); for (i = 0; i < pathspec->nr; i++) /* pass original pathspec, to be re-parsed */ - strvec_push(&argv, pathspec->items[i].original); + strvec_push(&cmd.args, pathspec->items[i].original); - status = run_command_v_opt(argv.v, RUN_GIT_CMD); - strvec_clear(&argv); - return status; + cmd.git_cmd = 1; + return run_command(&cmd); } int interactive_add(const char **argv, const char *prefix, int patch) diff --git a/builtin/am.c b/builtin/am.c index 39fea24833..20aea0d248 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -2187,14 +2187,12 @@ static int show_patch(struct am_state *state, enum show_patch_type sub_mode) int len; if (!is_null_oid(&state->orig_commit)) { - const char *av[4] = { "show", NULL, "--", NULL }; - char *new_oid_str; - int ret; + struct child_process cmd = CHILD_PROCESS_INIT; - av[1] = new_oid_str = xstrdup(oid_to_hex(&state->orig_commit)); - ret = run_command_v_opt(av, RUN_GIT_CMD); - free(new_oid_str); - return ret; + strvec_pushl(&cmd.args, "show", oid_to_hex(&state->orig_commit), + "--", NULL); + cmd.git_cmd = 1; + return run_command(&cmd); } switch (sub_mode) { diff --git a/builtin/bisect--helper.c b/builtin/bisect--helper.c index 28ef7ec2a4..1d2ce8a0e1 100644 --- a/builtin/bisect--helper.c +++ b/builtin/bisect--helper.c @@ -220,18 +220,17 @@ static int bisect_reset(const char *commit) } if (!ref_exists("BISECT_HEAD")) { - struct strvec argv = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; - strvec_pushl(&argv, "checkout", branch.buf, "--", NULL); - if (run_command_v_opt(argv.v, RUN_GIT_CMD)) { + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "checkout", branch.buf, "--", NULL); + if (run_command(&cmd)) { error(_("could not check out original" " HEAD '%s'. Try 'git bisect" " reset <commit>'."), branch.buf); strbuf_release(&branch); - strvec_clear(&argv); return -1; } - strvec_clear(&argv); } strbuf_release(&branch); @@ -765,10 +764,12 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, const char **a strbuf_read_file(&start_head, git_path_bisect_start(), 0); strbuf_trim(&start_head); if (!no_checkout) { - const char *argv[] = { "checkout", start_head.buf, - "--", NULL }; + struct child_process cmd = CHILD_PROCESS_INIT; - if (run_command_v_opt(argv, RUN_GIT_CMD)) { + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "checkout", start_head.buf, + "--", NULL); + if (run_command(&cmd)) { res = error(_("checking out '%s' failed." " Try 'git bisect start " "<valid-branch>'."), @@ -1098,40 +1099,38 @@ static enum bisect_error bisect_skip(struct bisect_terms *terms, const char **ar static int bisect_visualize(struct bisect_terms *terms, const char **argv, int argc) { - struct strvec args = STRVEC_INIT; - int flags = RUN_COMMAND_NO_STDIN, res = 0; + struct child_process cmd = CHILD_PROCESS_INIT; struct strbuf sb = STRBUF_INIT; if (bisect_next_check(terms, NULL) != 0) return BISECT_FAILED; + cmd.no_stdin = 1; if (!argc) { if ((getenv("DISPLAY") || getenv("SESSIONNAME") || getenv("MSYSTEM") || getenv("SECURITYSESSIONID")) && exists_in_PATH("gitk")) { - strvec_push(&args, "gitk"); + strvec_push(&cmd.args, "gitk"); } else { - strvec_push(&args, "log"); - flags |= RUN_GIT_CMD; + strvec_push(&cmd.args, "log"); + cmd.git_cmd = 1; } } else { if (argv[0][0] == '-') { - strvec_push(&args, "log"); - flags |= RUN_GIT_CMD; + strvec_push(&cmd.args, "log"); + cmd.git_cmd = 1; } else if (strcmp(argv[0], "tig") && !starts_with(argv[0], "git")) - flags |= RUN_GIT_CMD; + cmd.git_cmd = 1; - strvec_pushv(&args, argv); + strvec_pushv(&cmd.args, argv); } - strvec_pushl(&args, "--bisect", "--", NULL); + strvec_pushl(&cmd.args, "--bisect", "--", NULL); strbuf_read_file(&sb, git_path_bisect_names(), 0); - sq_dequote_to_strvec(sb.buf, &args); + sq_dequote_to_strvec(sb.buf, &cmd.args); strbuf_release(&sb); - res = run_command_v_opt(args.v, flags); - strvec_clear(&args); - return res; + return run_command(&cmd); } static int get_first_good(const char *refname UNUSED, @@ -1142,8 +1141,17 @@ static int get_first_good(const char *refname UNUSED, return 1; } -static int verify_good(const struct bisect_terms *terms, - const char **quoted_argv) +static int do_bisect_run(const char *command) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + + printf(_("running %s\n"), command); + cmd.use_shell = 1; + strvec_push(&cmd.args, command); + return run_command(&cmd); +} + +static int verify_good(const struct bisect_terms *terms, const char *command) { int rc; enum bisect_error res; @@ -1163,8 +1171,7 @@ static int verify_good(const struct bisect_terms *terms, if (res != BISECT_OK) return -1; - printf(_("running %s\n"), quoted_argv[0]); - rc = run_command_v_opt(quoted_argv, RUN_USING_SHELL); + rc = do_bisect_run(command); res = bisect_checkout(¤t_rev, no_checkout); if (res != BISECT_OK) @@ -1177,7 +1184,6 @@ static int bisect_run(struct bisect_terms *terms, const char **argv, int argc) { int res = BISECT_OK; struct strbuf command = STRBUF_INIT; - struct strvec run_args = STRVEC_INIT; const char *new_state; int temporary_stdout_fd, saved_stdout; int is_first_run = 1; @@ -1192,11 +1198,8 @@ static int bisect_run(struct bisect_terms *terms, const char **argv, int argc) return BISECT_FAILED; } - strvec_push(&run_args, command.buf); - while (1) { - printf(_("running %s\n"), command.buf); - res = run_command_v_opt(run_args.v, RUN_USING_SHELL); + res = do_bisect_run(command.buf); /* * Exit code 126 and 127 can either come from the shell @@ -1206,7 +1209,7 @@ static int bisect_run(struct bisect_terms *terms, const char **argv, int argc) * missing or non-executable script. */ if (is_first_run && (res == 126 || res == 127)) { - int rc = verify_good(terms, run_args.v); + int rc = verify_good(terms, command.buf); is_first_run = 0; if (rc < 0) { error(_("unable to verify '%s' on good" @@ -1273,7 +1276,6 @@ static int bisect_run(struct bisect_terms *terms, const char **argv, int argc) } strbuf_release(&command); - strvec_clear(&run_args); return res; } diff --git a/builtin/branch.c b/builtin/branch.c index 15be0c03ef..9470c980c1 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -150,7 +150,7 @@ static int branch_merged(int kind, const char *name, if (!reference_rev) reference_rev = head_rev; - merged = in_merge_bases(rev, reference_rev); + merged = reference_rev ? in_merge_bases(rev, reference_rev) : 0; /* * After the safety valve is fully redefined to "check with @@ -160,7 +160,7 @@ static int branch_merged(int kind, const char *name, * a gentle reminder is in order. */ if ((head_rev != reference_rev) && - in_merge_bases(rev, head_rev) != merged) { + (head_rev ? in_merge_bases(rev, head_rev) : 0) != merged) { if (merged) warning(_("deleting branch '%s' that has been merged to\n" " '%s', but not yet merged to HEAD."), @@ -235,11 +235,8 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, } branch_name_pos = strcspn(fmt, "%"); - if (!force) { + if (!force) head_rev = lookup_commit_reference(the_repository, &head_oid); - if (!head_rev) - die(_("Couldn't look up commit object for HEAD")); - } for (i = 0; i < argc; i++, strbuf_reset(&bname)) { char *target = NULL; diff --git a/builtin/clone.c b/builtin/clone.c index 547d6464b3..0e4348686b 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -653,9 +653,9 @@ static void update_head(const struct ref *our, const struct ref *remote, static int git_sparse_checkout_init(const char *repo) { - struct strvec argv = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; int result = 0; - strvec_pushl(&argv, "-C", repo, "sparse-checkout", "set", NULL); + strvec_pushl(&cmd.args, "-C", repo, "sparse-checkout", "set", NULL); /* * We must apply the setting in the current process @@ -663,12 +663,12 @@ static int git_sparse_checkout_init(const char *repo) */ core_apply_sparse_checkout = 1; - if (run_command_v_opt(argv.v, RUN_GIT_CMD)) { + cmd.git_cmd = 1; + if (run_command(&cmd)) { error(_("failed to initialize sparse-checkout")); result = 1; } - strvec_clear(&argv); return result; } @@ -733,37 +733,38 @@ static int checkout(int submodule_progress, int filter_submodules) oid_to_hex(&oid), "1", NULL); if (!err && (option_recurse_submodules.nr > 0)) { - struct strvec args = STRVEC_INIT; - strvec_pushl(&args, "submodule", "update", "--require-init", "--recursive", NULL); + struct child_process cmd = CHILD_PROCESS_INIT; + strvec_pushl(&cmd.args, "submodule", "update", "--require-init", + "--recursive", NULL); if (option_shallow_submodules == 1) - strvec_push(&args, "--depth=1"); + strvec_push(&cmd.args, "--depth=1"); if (max_jobs != -1) - strvec_pushf(&args, "--jobs=%d", max_jobs); + strvec_pushf(&cmd.args, "--jobs=%d", max_jobs); if (submodule_progress) - strvec_push(&args, "--progress"); + strvec_push(&cmd.args, "--progress"); if (option_verbosity < 0) - strvec_push(&args, "--quiet"); + strvec_push(&cmd.args, "--quiet"); if (option_remote_submodules) { - strvec_push(&args, "--remote"); - strvec_push(&args, "--no-fetch"); + strvec_push(&cmd.args, "--remote"); + strvec_push(&cmd.args, "--no-fetch"); } if (filter_submodules && filter_options.choice) - strvec_pushf(&args, "--filter=%s", + strvec_pushf(&cmd.args, "--filter=%s", expand_list_objects_filter_spec(&filter_options)); if (option_single_branch >= 0) - strvec_push(&args, option_single_branch ? + strvec_push(&cmd.args, option_single_branch ? "--single-branch" : "--no-single-branch"); - err = run_command_v_opt(args.v, RUN_GIT_CMD); - strvec_clear(&args); + cmd.git_cmd = 1; + err = run_command(&cmd); } return err; @@ -864,11 +865,15 @@ static void write_refspec_config(const char *src_ref_prefix, static void dissociate_from_references(void) { - static const char* argv[] = { "repack", "-a", "-d", NULL }; char *alternates = git_pathdup("objects/info/alternates"); if (!access(alternates, F_OK)) { - if (run_command_v_opt(argv, RUN_GIT_CMD|RUN_COMMAND_NO_STDIN)) + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + cmd.no_stdin = 1; + strvec_pushl(&cmd.args, "repack", "-a", "-d", NULL); + if (run_command(&cmd)) die(_("cannot repack to clean up")); if (unlink(alternates) && errno != ENOENT) die_errno(_("cannot unlink temporary alternates file")); diff --git a/builtin/difftool.c b/builtin/difftool.c index 4b10ad1a36..d7f08c8a7f 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -44,8 +44,11 @@ static int difftool_config(const char *var, const char *value, void *cb) static int print_tool_help(void) { - const char *argv[] = { "mergetool", "--tool-help=diff", NULL }; - return run_command_v_opt(argv, RUN_GIT_CMD); + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "mergetool", "--tool-help=diff", NULL); + return run_command(&cmd); } static int parse_index_info(char *p, int *mode1, int *mode2, @@ -360,8 +363,8 @@ static int run_dir_diff(const char *extcmd, int symlinks, const char *prefix, struct pair_entry *entry; struct index_state wtindex; struct checkout lstate, rstate; - int flags = RUN_GIT_CMD, err = 0; - const char *helper_argv[] = { "difftool--helper", NULL, NULL, NULL }; + int err = 0; + struct child_process cmd = CHILD_PROCESS_INIT; struct hashmap wt_modified, tmp_modified; int indices_loaded = 0; @@ -563,16 +566,17 @@ static int run_dir_diff(const char *extcmd, int symlinks, const char *prefix, } strbuf_setlen(&ldir, ldir_len); - helper_argv[1] = ldir.buf; strbuf_setlen(&rdir, rdir_len); - helper_argv[2] = rdir.buf; if (extcmd) { - helper_argv[0] = extcmd; - flags = 0; - } else + strvec_push(&cmd.args, extcmd); + } else { + strvec_push(&cmd.args, "difftool--helper"); + cmd.git_cmd = 1; setenv("GIT_DIFFTOOL_DIRDIFF", "true", 1); - ret = run_command_v_opt(helper_argv, flags); + } + strvec_pushl(&cmd.args, ldir.buf, rdir.buf, NULL); + ret = run_command(&cmd); /* TODO: audit for interaction with sparse-index. */ ensure_full_index(&wtindex); diff --git a/builtin/fetch.c b/builtin/fetch.c index b06e454cbd..7378cafeec 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1972,14 +1972,17 @@ static int fetch_multiple(struct string_list *list, int max_children) } else for (i = 0; i < list->nr; i++) { const char *name = list->items[i].string; - strvec_push(&argv, name); + struct child_process cmd = CHILD_PROCESS_INIT; + + strvec_pushv(&cmd.args, argv.v); + strvec_push(&cmd.args, name); if (verbosity >= 0) printf(_("Fetching %s\n"), name); - if (run_command_v_opt(argv.v, RUN_GIT_CMD)) { + cmd.git_cmd = 1; + if (run_command(&cmd)) { error(_("could not fetch %s"), name); result = 1; } - strvec_pop(&argv); } strvec_clear(&argv); diff --git a/builtin/gc.c b/builtin/gc.c index 24ea85c7af..6b08dcf3c5 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -42,7 +42,7 @@ static const char * const builtin_gc_usage[] = { static int pack_refs = 1; static int prune_reflogs = 1; -static int cruft_packs = 0; +static int cruft_packs = -1; static int aggressive_depth = 50; static int aggressive_window = 250; static int gc_auto_threshold = 6700; @@ -167,9 +167,11 @@ static void gc_config(void) struct maintenance_run_opts; static int maintenance_task_pack_refs(MAYBE_UNUSED struct maintenance_run_opts *opts) { - const char *argv[] = { "pack-refs", "--all", "--prune", NULL }; + struct child_process cmd = CHILD_PROCESS_INIT; - return run_command_v_opt(argv, RUN_GIT_CMD); + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "pack-refs", "--all", "--prune", NULL); + return run_command(&cmd); } static int too_many_loose_objects(void) @@ -535,8 +537,14 @@ static void gc_before_repack(void) if (pack_refs && maintenance_task_pack_refs(NULL)) die(FAILED_RUN, "pack-refs"); - if (prune_reflogs && run_command_v_opt(reflog.v, RUN_GIT_CMD)) - die(FAILED_RUN, reflog.v[0]); + if (prune_reflogs) { + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushv(&cmd.args, reflog.v); + if (run_command(&cmd)) + die(FAILED_RUN, reflog.v[0]); + } } int cmd_gc(int argc, const char **argv, const char *prefix) @@ -550,6 +558,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix) int daemonized = 0; int keep_largest_pack = -1; timestamp_t dummy; + struct child_process rerere_cmd = CHILD_PROCESS_INIT; struct option builtin_gc_options[] = { OPT__QUIET(&quiet, N_("suppress progress reporting")), @@ -593,6 +602,10 @@ int cmd_gc(int argc, const char **argv, const char *prefix) if (prune_expire && parse_expiry_date(prune_expire, &dummy)) die(_("failed to parse prune expiry value %s"), prune_expire); + prepare_repo_settings(the_repository); + if (cruft_packs < 0) + cruft_packs = the_repository->settings.gc_cruft_packs; + if (aggressive) { strvec_push(&repack, "-f"); if (aggressive_depth > 0) @@ -671,11 +684,17 @@ int cmd_gc(int argc, const char **argv, const char *prefix) gc_before_repack(); if (!repository_format_precious_objects) { - if (run_command_v_opt(repack.v, - RUN_GIT_CMD | RUN_CLOSE_OBJECT_STORE)) + struct child_process repack_cmd = CHILD_PROCESS_INIT; + + repack_cmd.git_cmd = 1; + repack_cmd.close_object_store = 1; + strvec_pushv(&repack_cmd.args, repack.v); + if (run_command(&repack_cmd)) die(FAILED_RUN, repack.v[0]); if (prune_expire) { + struct child_process prune_cmd = CHILD_PROCESS_INIT; + /* run `git prune` even if using cruft packs */ strvec_push(&prune, prune_expire); if (quiet) @@ -683,18 +702,26 @@ int cmd_gc(int argc, const char **argv, const char *prefix) if (has_promisor_remote()) strvec_push(&prune, "--exclude-promisor-objects"); - if (run_command_v_opt(prune.v, RUN_GIT_CMD)) + prune_cmd.git_cmd = 1; + strvec_pushv(&prune_cmd.args, prune.v); + if (run_command(&prune_cmd)) die(FAILED_RUN, prune.v[0]); } } if (prune_worktrees_expire) { + struct child_process prune_worktrees_cmd = CHILD_PROCESS_INIT; + strvec_push(&prune_worktrees, prune_worktrees_expire); - if (run_command_v_opt(prune_worktrees.v, RUN_GIT_CMD)) + prune_worktrees_cmd.git_cmd = 1; + strvec_pushv(&prune_worktrees_cmd.args, prune_worktrees.v); + if (run_command(&prune_worktrees_cmd)) die(FAILED_RUN, prune_worktrees.v[0]); } - if (run_command_v_opt(rerere.v, RUN_GIT_CMD)) + rerere_cmd.git_cmd = 1; + strvec_pushv(&rerere_cmd.args, rerere.v); + if (run_command(&rerere_cmd)) die(FAILED_RUN, rerere.v[0]); report_garbage = report_pack_garbage; @@ -704,7 +731,6 @@ int cmd_gc(int argc, const char **argv, const char *prefix) clean_pack_garbage(); } - prepare_repo_settings(the_repository); if (the_repository->settings.gc_write_commit_graph == 1) write_commit_graph_reachable(the_repository->objects->odb, !quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, @@ -1910,20 +1936,16 @@ static char *schtasks_task_name(const char *frequency) static int schtasks_remove_task(enum schedule_priority schedule) { const char *cmd = "schtasks"; - int result; - struct strvec args = STRVEC_INIT; + struct child_process child = CHILD_PROCESS_INIT; const char *frequency = get_frequency(schedule); char *name = schtasks_task_name(frequency); get_schedule_cmd(&cmd, NULL); - strvec_split(&args, cmd); - strvec_pushl(&args, "/delete", "/tn", name, "/f", NULL); - - result = run_command_v_opt(args.v, 0); - - strvec_clear(&args); + strvec_split(&child.args, cmd); + strvec_pushl(&child.args, "/delete", "/tn", name, "/f", NULL); free(name); - return result; + + return run_command(&child); } static int schtasks_remove_tasks(void) diff --git a/builtin/merge-index.c b/builtin/merge-index.c index c0383fe9df..012f52bd00 100644 --- a/builtin/merge-index.c +++ b/builtin/merge-index.c @@ -12,6 +12,7 @@ static int merge_entry(int pos, const char *path) const char *arguments[] = { pgm, "", "", "", path, "", "", "", NULL }; char hexbuf[4][GIT_MAX_HEXSZ + 1]; char ownbuf[4][60]; + struct child_process cmd = CHILD_PROCESS_INIT; if (pos >= active_nr) die("git merge-index: %s not in the cache", path); @@ -31,7 +32,8 @@ static int merge_entry(int pos, const char *path) if (!found) die("git merge-index: %s not in the cache", path); - if (run_command_v_opt(arguments, 0)) { + strvec_pushv(&cmd.args, arguments); + if (run_command(&cmd)) { if (one_shot) err++; else { diff --git a/builtin/merge.c b/builtin/merge.c index 5900b81729..b3f75f55c8 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -345,60 +345,49 @@ out: return rc; } -static void read_empty(const struct object_id *oid, int verbose) +static void read_empty(const struct object_id *oid) { - int i = 0; - const char *args[7]; - - args[i++] = "read-tree"; - if (verbose) - args[i++] = "-v"; - args[i++] = "-m"; - args[i++] = "-u"; - args[i++] = empty_tree_oid_hex(); - args[i++] = oid_to_hex(oid); - args[i] = NULL; + struct child_process cmd = CHILD_PROCESS_INIT; + + strvec_pushl(&cmd.args, "read-tree", "-m", "-u", empty_tree_oid_hex(), + oid_to_hex(oid), NULL); + cmd.git_cmd = 1; - if (run_command_v_opt(args, RUN_GIT_CMD)) + if (run_command(&cmd)) die(_("read-tree failed")); } -static void reset_hard(const struct object_id *oid, int verbose) +static void reset_hard(const struct object_id *oid) { - int i = 0; - const char *args[6]; - - args[i++] = "read-tree"; - if (verbose) - args[i++] = "-v"; - args[i++] = "--reset"; - args[i++] = "-u"; - args[i++] = oid_to_hex(oid); - args[i] = NULL; + struct child_process cmd = CHILD_PROCESS_INIT; + + strvec_pushl(&cmd.args, "read-tree", "-v", "--reset", "-u", + oid_to_hex(oid), NULL); + cmd.git_cmd = 1; - if (run_command_v_opt(args, RUN_GIT_CMD)) + if (run_command(&cmd)) die(_("read-tree failed")); } static void restore_state(const struct object_id *head, const struct object_id *stash) { - struct strvec args = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; - reset_hard(head, 1); + reset_hard(head); if (is_null_oid(stash)) goto refresh_cache; - strvec_pushl(&args, "stash", "apply", "--index", "--quiet", NULL); - strvec_push(&args, oid_to_hex(stash)); + strvec_pushl(&cmd.args, "stash", "apply", "--index", "--quiet", NULL); + strvec_push(&cmd.args, oid_to_hex(stash)); /* * It is OK to ignore error here, for example when there was * nothing to restore. */ - run_command_v_opt(args.v, RUN_GIT_CMD); - strvec_clear(&args); + cmd.git_cmd = 1; + run_command(&cmd); refresh_cache: if (discard_cache() < 0 || read_cache() < 0) @@ -1470,7 +1459,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) check_trust_level); remote_head_oid = &remoteheads->item->object.oid; - read_empty(remote_head_oid, 0); + read_empty(remote_head_oid); update_ref("initial pull", "HEAD", remote_head_oid, NULL, 0, UPDATE_REFS_DIE_ON_ERR); goto done; diff --git a/builtin/pull.c b/builtin/pull.c index 403a24d7ca..b21edd767a 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -515,76 +515,75 @@ static void parse_repo_refspecs(int argc, const char **argv, const char **repo, */ static int run_fetch(const char *repo, const char **refspecs) { - struct strvec args = STRVEC_INIT; - int ret; + struct child_process cmd = CHILD_PROCESS_INIT; - strvec_pushl(&args, "fetch", "--update-head-ok", NULL); + strvec_pushl(&cmd.args, "fetch", "--update-head-ok", NULL); /* Shared options */ - argv_push_verbosity(&args); + argv_push_verbosity(&cmd.args); if (opt_progress) - strvec_push(&args, opt_progress); + strvec_push(&cmd.args, opt_progress); /* Options passed to git-fetch */ if (opt_all) - strvec_push(&args, opt_all); + strvec_push(&cmd.args, opt_all); if (opt_append) - strvec_push(&args, opt_append); + strvec_push(&cmd.args, opt_append); if (opt_upload_pack) - strvec_push(&args, opt_upload_pack); - argv_push_force(&args); + strvec_push(&cmd.args, opt_upload_pack); + argv_push_force(&cmd.args); if (opt_tags) - strvec_push(&args, opt_tags); + strvec_push(&cmd.args, opt_tags); if (opt_prune) - strvec_push(&args, opt_prune); + strvec_push(&cmd.args, opt_prune); if (recurse_submodules_cli != RECURSE_SUBMODULES_DEFAULT) switch (recurse_submodules_cli) { case RECURSE_SUBMODULES_ON: - strvec_push(&args, "--recurse-submodules=on"); + strvec_push(&cmd.args, "--recurse-submodules=on"); break; case RECURSE_SUBMODULES_OFF: - strvec_push(&args, "--recurse-submodules=no"); + strvec_push(&cmd.args, "--recurse-submodules=no"); break; case RECURSE_SUBMODULES_ON_DEMAND: - strvec_push(&args, "--recurse-submodules=on-demand"); + strvec_push(&cmd.args, "--recurse-submodules=on-demand"); break; default: BUG("submodule recursion option not understood"); } if (max_children) - strvec_push(&args, max_children); + strvec_push(&cmd.args, max_children); if (opt_dry_run) - strvec_push(&args, "--dry-run"); + strvec_push(&cmd.args, "--dry-run"); if (opt_keep) - strvec_push(&args, opt_keep); + strvec_push(&cmd.args, opt_keep); if (opt_depth) - strvec_push(&args, opt_depth); + strvec_push(&cmd.args, opt_depth); if (opt_unshallow) - strvec_push(&args, opt_unshallow); + strvec_push(&cmd.args, opt_unshallow); if (opt_update_shallow) - strvec_push(&args, opt_update_shallow); + strvec_push(&cmd.args, opt_update_shallow); if (opt_refmap) - strvec_push(&args, opt_refmap); + strvec_push(&cmd.args, opt_refmap); if (opt_ipv4) - strvec_push(&args, opt_ipv4); + strvec_push(&cmd.args, opt_ipv4); if (opt_ipv6) - strvec_push(&args, opt_ipv6); + strvec_push(&cmd.args, opt_ipv6); if (opt_show_forced_updates > 0) - strvec_push(&args, "--show-forced-updates"); + strvec_push(&cmd.args, "--show-forced-updates"); else if (opt_show_forced_updates == 0) - strvec_push(&args, "--no-show-forced-updates"); + strvec_push(&cmd.args, "--no-show-forced-updates"); if (set_upstream) - strvec_push(&args, set_upstream); - strvec_pushv(&args, opt_fetch.v); + strvec_push(&cmd.args, set_upstream); + strvec_pushv(&cmd.args, opt_fetch.v); if (repo) { - strvec_push(&args, repo); - strvec_pushv(&args, refspecs); + strvec_push(&cmd.args, repo); + strvec_pushv(&cmd.args, refspecs); } else if (*refspecs) BUG("refspecs without repo?"); - ret = run_command_v_opt(args.v, RUN_GIT_CMD | RUN_CLOSE_OBJECT_STORE); - strvec_clear(&args); - return ret; + cmd.git_cmd = 1; + cmd.close_object_store = 1; + return run_command(&cmd); } /** @@ -653,52 +652,50 @@ static int update_submodules(void) */ static int run_merge(void) { - int ret; - struct strvec args = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; - strvec_pushl(&args, "merge", NULL); + strvec_pushl(&cmd.args, "merge", NULL); /* Shared options */ - argv_push_verbosity(&args); + argv_push_verbosity(&cmd.args); if (opt_progress) - strvec_push(&args, opt_progress); + strvec_push(&cmd.args, opt_progress); /* Options passed to git-merge */ if (opt_diffstat) - strvec_push(&args, opt_diffstat); + strvec_push(&cmd.args, opt_diffstat); if (opt_log) - strvec_push(&args, opt_log); + strvec_push(&cmd.args, opt_log); if (opt_signoff) - strvec_push(&args, opt_signoff); + strvec_push(&cmd.args, opt_signoff); if (opt_squash) - strvec_push(&args, opt_squash); + strvec_push(&cmd.args, opt_squash); if (opt_commit) - strvec_push(&args, opt_commit); + strvec_push(&cmd.args, opt_commit); if (opt_edit) - strvec_push(&args, opt_edit); + strvec_push(&cmd.args, opt_edit); if (cleanup_arg) - strvec_pushf(&args, "--cleanup=%s", cleanup_arg); + strvec_pushf(&cmd.args, "--cleanup=%s", cleanup_arg); if (opt_ff) - strvec_push(&args, opt_ff); + strvec_push(&cmd.args, opt_ff); if (opt_verify) - strvec_push(&args, opt_verify); + strvec_push(&cmd.args, opt_verify); if (opt_verify_signatures) - strvec_push(&args, opt_verify_signatures); - strvec_pushv(&args, opt_strategies.v); - strvec_pushv(&args, opt_strategy_opts.v); + strvec_push(&cmd.args, opt_verify_signatures); + strvec_pushv(&cmd.args, opt_strategies.v); + strvec_pushv(&cmd.args, opt_strategy_opts.v); if (opt_gpg_sign) - strvec_push(&args, opt_gpg_sign); + strvec_push(&cmd.args, opt_gpg_sign); if (opt_autostash == 0) - strvec_push(&args, "--no-autostash"); + strvec_push(&cmd.args, "--no-autostash"); else if (opt_autostash == 1) - strvec_push(&args, "--autostash"); + strvec_push(&cmd.args, "--autostash"); if (opt_allow_unrelated_histories > 0) - strvec_push(&args, "--allow-unrelated-histories"); + strvec_push(&cmd.args, "--allow-unrelated-histories"); - strvec_push(&args, "FETCH_HEAD"); - ret = run_command_v_opt(args.v, RUN_GIT_CMD); - strvec_clear(&args); - return ret; + strvec_push(&cmd.args, "FETCH_HEAD"); + cmd.git_cmd = 1; + return run_command(&cmd); } /** @@ -879,43 +876,41 @@ static int get_rebase_newbase_and_upstream(struct object_id *newbase, static int run_rebase(const struct object_id *newbase, const struct object_id *upstream) { - int ret; - struct strvec args = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; - strvec_push(&args, "rebase"); + strvec_push(&cmd.args, "rebase"); /* Shared options */ - argv_push_verbosity(&args); + argv_push_verbosity(&cmd.args); /* Options passed to git-rebase */ if (opt_rebase == REBASE_MERGES) - strvec_push(&args, "--rebase-merges"); + strvec_push(&cmd.args, "--rebase-merges"); else if (opt_rebase == REBASE_INTERACTIVE) - strvec_push(&args, "--interactive"); + strvec_push(&cmd.args, "--interactive"); if (opt_diffstat) - strvec_push(&args, opt_diffstat); - strvec_pushv(&args, opt_strategies.v); - strvec_pushv(&args, opt_strategy_opts.v); + strvec_push(&cmd.args, opt_diffstat); + strvec_pushv(&cmd.args, opt_strategies.v); + strvec_pushv(&cmd.args, opt_strategy_opts.v); if (opt_gpg_sign) - strvec_push(&args, opt_gpg_sign); + strvec_push(&cmd.args, opt_gpg_sign); if (opt_signoff) - strvec_push(&args, opt_signoff); + strvec_push(&cmd.args, opt_signoff); if (opt_autostash == 0) - strvec_push(&args, "--no-autostash"); + strvec_push(&cmd.args, "--no-autostash"); else if (opt_autostash == 1) - strvec_push(&args, "--autostash"); + strvec_push(&cmd.args, "--autostash"); if (opt_verify_signatures && !strcmp(opt_verify_signatures, "--verify-signatures")) warning(_("ignoring --verify-signatures for rebase")); - strvec_push(&args, "--onto"); - strvec_push(&args, oid_to_hex(newbase)); + strvec_push(&cmd.args, "--onto"); + strvec_push(&cmd.args, oid_to_hex(newbase)); - strvec_push(&args, oid_to_hex(upstream)); + strvec_push(&cmd.args, oid_to_hex(upstream)); - ret = run_command_v_opt(args.v, RUN_GIT_CMD); - strvec_clear(&args); - return ret; + cmd.git_cmd = 1; + return run_command(&cmd); } static int get_can_ff(struct object_id *orig_head, diff --git a/builtin/read-tree.c b/builtin/read-tree.c index f4cbe460b9..45c6652444 100644 --- a/builtin/read-tree.c +++ b/builtin/read-tree.c @@ -249,6 +249,10 @@ int cmd_read_tree(int argc, const char **argv, const char *cmd_prefix) if (opts.debug_unpack) opts.fn = debug_merge; + /* If we're going to prime_cache_tree later, skip cache tree update */ + if (nr_trees == 1 && !opts.prefix) + opts.skip_cache_tree_update = 1; + cache_tree_free(&active_cache_tree); for (i = 0; i < nr_trees; i++) { struct tree *tree = trees[i]; diff --git a/builtin/remote.c b/builtin/remote.c index 93285fc06e..729f6f3643 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -92,13 +92,15 @@ static int verbose; static int fetch_remote(const char *name) { - const char *argv[] = { "fetch", name, NULL, NULL }; - if (verbose) { - argv[1] = "-v"; - argv[2] = name; - } + struct child_process cmd = CHILD_PROCESS_INIT; + + strvec_push(&cmd.args, "fetch"); + if (verbose) + strvec_push(&cmd.args, "-v"); + strvec_push(&cmd.args, name); + cmd.git_cmd = 1; printf_ln(_("Updating %s"), name); - if (run_command_v_opt(argv, RUN_GIT_CMD)) + if (run_command(&cmd)) return error(_("Could not fetch %s"), name); return 0; } @@ -1508,37 +1510,35 @@ static int update(int argc, const char **argv, const char *prefix) N_("prune remotes after fetching")), OPT_END() }; - struct strvec fetch_argv = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; int default_defined = 0; - int retval; argc = parse_options(argc, argv, prefix, options, builtin_remote_update_usage, PARSE_OPT_KEEP_ARGV0); - strvec_push(&fetch_argv, "fetch"); + strvec_push(&cmd.args, "fetch"); if (prune != -1) - strvec_push(&fetch_argv, prune ? "--prune" : "--no-prune"); + strvec_push(&cmd.args, prune ? "--prune" : "--no-prune"); if (verbose) - strvec_push(&fetch_argv, "-v"); - strvec_push(&fetch_argv, "--multiple"); + strvec_push(&cmd.args, "-v"); + strvec_push(&cmd.args, "--multiple"); if (argc < 2) - strvec_push(&fetch_argv, "default"); + strvec_push(&cmd.args, "default"); for (i = 1; i < argc; i++) - strvec_push(&fetch_argv, argv[i]); + strvec_push(&cmd.args, argv[i]); - if (strcmp(fetch_argv.v[fetch_argv.nr-1], "default") == 0) { + if (strcmp(cmd.args.v[cmd.args.nr-1], "default") == 0) { git_config(get_remote_default, &default_defined); if (!default_defined) { - strvec_pop(&fetch_argv); - strvec_push(&fetch_argv, "--all"); + strvec_pop(&cmd.args); + strvec_push(&cmd.args, "--all"); } } - retval = run_command_v_opt(fetch_argv.v, RUN_GIT_CMD); - strvec_clear(&fetch_argv); - return retval; + cmd.git_cmd = 1; + return run_command(&cmd); } static int remove_all_fetch_refspecs(const char *key) diff --git a/builtin/repack.c b/builtin/repack.c index 10e23f9ee1..65eb1b8bd2 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -32,7 +32,6 @@ static int write_bitmaps = -1; static int use_delta_islands; static int run_update_server_info = 1; static char *packdir, *packtmp_name, *packtmp; -static char *cruft_expiration; static const char *const git_repack_usage[] = { N_("git repack [<options>]"), @@ -150,7 +149,8 @@ static void remove_redundant_pack(const char *dir_name, const char *base_name) } static void prepare_pack_objects(struct child_process *cmd, - const struct pack_objects_args *args) + const struct pack_objects_args *args, + const char *out) { strvec_push(&cmd->args, "pack-objects"); if (args->window) @@ -173,7 +173,7 @@ static void prepare_pack_objects(struct child_process *cmd, strvec_push(&cmd->args, "--quiet"); if (delta_base_offset) strvec_push(&cmd->args, "--delta-base-offset"); - strvec_push(&cmd->args, packtmp); + strvec_push(&cmd->args, out); cmd->git_cmd = 1; cmd->out = -1; } @@ -241,7 +241,7 @@ static void repack_promisor_objects(const struct pack_objects_args *args, FILE *out; struct strbuf line = STRBUF_INIT; - prepare_pack_objects(&cmd, args); + prepare_pack_objects(&cmd, args, packtmp); cmd.in = -1; /* @@ -657,7 +657,9 @@ static void remove_redundant_bitmaps(struct string_list *include, } static int write_cruft_pack(const struct pack_objects_args *args, + const char *destination, const char *pack_prefix, + const char *cruft_expiration, struct string_list *names, struct string_list *existing_packs, struct string_list *existing_kept_packs) @@ -667,8 +669,10 @@ static int write_cruft_pack(const struct pack_objects_args *args, struct string_list_item *item; FILE *in, *out; int ret; + const char *scratch; + int local = skip_prefix(destination, packdir, &scratch); - prepare_pack_objects(&cmd, args); + prepare_pack_objects(&cmd, args, destination); strvec_push(&cmd.args, "--cruft"); if (cruft_expiration) @@ -693,6 +697,10 @@ static int write_cruft_pack(const struct pack_objects_args *args, * By the time it is read here, it contains only the pack(s) * that were just written, which is exactly the set of packs we * want to consider kept. + * + * If `--expire-to` is given, the double-use served by `names` + * ensures that the pack written to `--expire-to` excludes any + * objects contained in the cruft pack. */ in = xfdopen(cmd.in, "w"); for_each_string_list_item(item, names) @@ -710,9 +718,14 @@ static int write_cruft_pack(const struct pack_objects_args *args, if (line.len != the_hash_algo->hexsz) die(_("repack: Expecting full hex object ID lines only " "from pack-objects.")); - - item = string_list_append(names, line.buf); - item->util = populate_pack_exts(line.buf); + /* + * avoid putting packs written outside of the repository in the + * list of names + */ + if (local) { + item = string_list_append(names, line.buf); + item->util = populate_pack_exts(line.buf); + } } fclose(out); @@ -744,6 +757,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) struct pack_objects_args cruft_po_args = {NULL}; int geometric_factor = 0; int write_midx = 0; + const char *cruft_expiration = NULL; + const char *expire_to = NULL; struct option builtin_repack_options[] = { OPT_BIT('a', NULL, &pack_everything, @@ -793,6 +808,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) N_("find a geometric progression with factor <N>")), OPT_BOOL('m', "write-midx", &write_midx, N_("write a multi-pack index of the resulting packs")), + OPT_STRING(0, "expire-to", &expire_to, N_("dir"), + N_("pack prefix to store a pack containing pruned objects")), OPT_END() }; @@ -858,7 +875,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) split_pack_geometry(geometry, geometric_factor); } - prepare_pack_objects(&cmd, &po_args); + prepare_pack_objects(&cmd, &po_args, packtmp); show_progress = !po_args.quiet && isatty(2); @@ -984,11 +1001,45 @@ int cmd_repack(int argc, const char **argv, const char *prefix) cruft_po_args.local = po_args.local; cruft_po_args.quiet = po_args.quiet; - ret = write_cruft_pack(&cruft_po_args, pack_prefix, &names, + ret = write_cruft_pack(&cruft_po_args, packtmp, pack_prefix, + cruft_expiration, &names, &existing_nonkept_packs, &existing_kept_packs); if (ret) return ret; + + if (delete_redundant && expire_to) { + /* + * If `--expire-to` is given with `-d`, it's possible + * that we're about to prune some objects. With cruft + * packs, pruning is implicit: any objects from existing + * packs that weren't picked up by new packs are removed + * when their packs are deleted. + * + * Generate an additional cruft pack, with one twist: + * `names` now includes the name of the cruft pack + * written in the previous step. So the contents of + * _this_ cruft pack exclude everything contained in the + * existing cruft pack (that is, all of the unreachable + * objects which are no older than + * `--cruft-expiration`). + * + * To make this work, cruft_expiration must become NULL + * so that this cruft pack doesn't actually prune any + * objects. If it were non-NULL, this call would always + * generate an empty pack (since every object not in the + * cruft pack generated above will have an mtime older + * than the expiration). + */ + ret = write_cruft_pack(&cruft_po_args, expire_to, + pack_prefix, + NULL, + &names, + &existing_nonkept_packs, + &existing_kept_packs); + if (ret) + return ret; + } } string_list_sort(&names); diff --git a/builtin/reset.c b/builtin/reset.c index fdce6f8c85..ab02777482 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -73,9 +73,11 @@ static int reset_index(const char *ref, const struct object_id *oid, int reset_t case HARD: opts.update = 1; opts.reset = UNPACK_RESET_OVERWRITE_UNTRACKED; + opts.skip_cache_tree_update = 1; break; case MIXED: opts.reset = UNPACK_RESET_PROTECT_UNTRACKED; + opts.skip_cache_tree_update = 1; /* but opts.update=0, so working tree not updated */ break; default: @@ -260,7 +260,7 @@ macos-latest) else MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)" MAKEFLAGS="$MAKEFLAGS NO_APPLE_COMMON_CRYPTO=NoThanks" - MAKEFLAGS="$MAKEFLAGS DC_SHA1=YesPlease NO_OPENSSL=NoThanks" + MAKEFLAGS="$MAKEFLAGS NO_OPENSSL=NoThanks" fi ;; esac diff --git a/compat/mingw.c b/compat/mingw.c index 901375d584..d614f156df 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -196,16 +196,19 @@ static int read_yes_no_answer(void) static int ask_yes_no_if_possible(const char *format, ...) { char question[4096]; - const char *retry_hook[] = { NULL, NULL, NULL }; + const char *retry_hook; va_list args; va_start(args, format); vsnprintf(question, sizeof(question), format, args); va_end(args); - if ((retry_hook[0] = mingw_getenv("GIT_ASK_YESNO"))) { - retry_hook[1] = question; - return !run_command_v_opt(retry_hook, 0); + retry_hook = mingw_getenv("GIT_ASK_YESNO"); + if (retry_hook) { + struct child_process cmd = CHILD_PROCESS_INIT; + + strvec_pushl(&cmd.args, retry_hook, question, NULL); + return !run_command(&cmd); } if (!isatty(_fileno(stdin)) || !isatty(_fileno(stderr))) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 3957e4cf8c..2f6e0197ff 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -1025,7 +1025,6 @@ set(NO_PERL ) set(NO_PTHREADS ) set(NO_PYTHON ) set(PAGER_ENV "LESS=FRX LV=-c") -set(DC_SHA1 YesPlease) set(RUNTIME_PREFIX true) set(NO_GETTEXT ) @@ -1061,7 +1060,6 @@ file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_PERL='${NO_PERL}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_PTHREADS='${NO_PTHREADS}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_UNIX_SOCKETS='${NO_UNIX_SOCKETS}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "PAGER_ENV='${PAGER_ENV}'\n") -file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "DC_SHA1='${DC_SHA1}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "X='${EXE_EXTENSION}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_GETTEXT='${NO_GETTEXT}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "RUNTIME_PREFIX='${RUNTIME_PREFIX}'\n") @@ -4301,35 +4301,34 @@ static void run_external_diff(const char *pgm, const char *xfrm_msg, struct diff_options *o) { - struct strvec argv = STRVEC_INIT; - struct strvec env = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; struct diff_queue_struct *q = &diff_queued_diff; - strvec_push(&argv, pgm); - strvec_push(&argv, name); + strvec_push(&cmd.args, pgm); + strvec_push(&cmd.args, name); if (one && two) { - add_external_diff_name(o->repo, &argv, name, one); + add_external_diff_name(o->repo, &cmd.args, name, one); if (!other) - add_external_diff_name(o->repo, &argv, name, two); + add_external_diff_name(o->repo, &cmd.args, name, two); else { - add_external_diff_name(o->repo, &argv, other, two); - strvec_push(&argv, other); - strvec_push(&argv, xfrm_msg); + add_external_diff_name(o->repo, &cmd.args, other, two); + strvec_push(&cmd.args, other); + strvec_push(&cmd.args, xfrm_msg); } } - strvec_pushf(&env, "GIT_DIFF_PATH_COUNTER=%d", ++o->diff_path_counter); - strvec_pushf(&env, "GIT_DIFF_PATH_TOTAL=%d", q->nr); + strvec_pushf(&cmd.env, "GIT_DIFF_PATH_COUNTER=%d", + ++o->diff_path_counter); + strvec_pushf(&cmd.env, "GIT_DIFF_PATH_TOTAL=%d", q->nr); diff_free_filespec_data(one); diff_free_filespec_data(two); - if (run_command_v_opt_cd_env(argv.v, RUN_USING_SHELL, NULL, env.v)) + cmd.use_shell = 1; + if (run_command(&cmd)) die(_("external diff died, stopping at %s"), name); remove_tempfile(); - strvec_clear(&argv); - strvec_clear(&env); } static int similarity_index(struct diff_filepair *p) diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index c0f42301c8..19d772f0f3 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -54,10 +54,14 @@ enum ipc_active_state fsmonitor_ipc__get_state(void) static int spawn_daemon(void) { - const char *args[] = { "fsmonitor--daemon", "start", NULL }; + struct child_process cmd = CHILD_PROCESS_INIT; - return run_command_v_opt_tr2(args, RUN_COMMAND_NO_STDIN | RUN_GIT_CMD, - "fsmonitor"); + cmd.git_cmd = 1; + cmd.no_stdin = 1; + cmd.trace2_child_class = "fsmonitor"; + strvec_pushl(&cmd.args, "fsmonitor--daemon", "start", NULL); + + return run_command(&cmd); } int fsmonitor_ipc__send_query(const char *since_token, @@ -787,7 +787,7 @@ static int run_argv(int *argcp, const char ***argv) if (!done_alias) handle_builtin(*argcp, *argv); else if (get_builtin(**argv)) { - struct strvec args = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; int i; /* @@ -804,18 +804,21 @@ static int run_argv(int *argcp, const char ***argv) commit_pager_choice(); - strvec_push(&args, "git"); + strvec_push(&cmd.args, "git"); for (i = 0; i < *argcp; i++) - strvec_push(&args, (*argv)[i]); + strvec_push(&cmd.args, (*argv)[i]); - trace_argv_printf(args.v, "trace: exec:"); + trace_argv_printf(cmd.args.v, "trace: exec:"); /* * if we fail because the command is not found, it is * OK to return. Otherwise, we just pass along the status code. */ - i = run_command_v_opt_tr2(args.v, RUN_SILENT_EXEC_FAILURE | - RUN_CLEAN_ON_EXIT | RUN_WAIT_AFTER_CLEAN, "git_alias"); + cmd.silent_exec_failure = 1; + cmd.clean_on_exit = 1; + cmd.wait_after_clean = 1; + cmd.trace2_child_class = "git_alias"; + i = run_command(&cmd); if (i >= 0 || errno != ENOENT) exit(i); die("could not execute builtin %s", **argv); diff --git a/ll-merge.c b/ll-merge.c index a8e2db9336..22a603e8af 100644 --- a/ll-merge.c +++ b/ll-merge.c @@ -193,7 +193,7 @@ static enum ll_merge_result ll_ext_merge(const struct ll_merge_driver *fn, struct strbuf cmd = STRBUF_INIT; struct strbuf_expand_dict_entry dict[6]; struct strbuf path_sq = STRBUF_INIT; - const char *args[] = { NULL, NULL }; + struct child_process child = CHILD_PROCESS_INIT; int status, fd, i; struct stat st; enum ll_merge_result ret; @@ -219,8 +219,9 @@ static enum ll_merge_result ll_ext_merge(const struct ll_merge_driver *fn, strbuf_expand(&cmd, fn->cmdline, strbuf_expand_dict_cb, &dict); - args[0] = cmd.buf; - status = run_command_v_opt(args, RUN_USING_SHELL); + child.use_shell = 1; + strvec_push(&child.args, cmd.buf); + status = run_command(&child); fd = open(temp[1], O_RDONLY); if (fd < 0) goto bad; @@ -19,22 +19,22 @@ int try_merge_command(struct repository *r, const char **xopts, struct commit_list *common, const char *head_arg, struct commit_list *remotes) { - struct strvec args = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; int i, ret; struct commit_list *j; - strvec_pushf(&args, "merge-%s", strategy); + strvec_pushf(&cmd.args, "merge-%s", strategy); for (i = 0; i < xopts_nr; i++) - strvec_pushf(&args, "--%s", xopts[i]); + strvec_pushf(&cmd.args, "--%s", xopts[i]); for (j = common; j; j = j->next) - strvec_push(&args, merge_argument(j->item)); - strvec_push(&args, "--"); - strvec_push(&args, head_arg); + strvec_push(&cmd.args, merge_argument(j->item)); + strvec_push(&cmd.args, "--"); + strvec_push(&cmd.args, head_arg); for (j = remotes; j; j = j->next) - strvec_push(&args, merge_argument(j->item)); + strvec_push(&cmd.args, merge_argument(j->item)); - ret = run_command_v_opt(args.v, RUN_GIT_CMD); - strvec_clear(&args); + cmd.git_cmd = 1; + ret = run_command(&cmd); discard_index(r->index); if (repo_read_index(r) < 0) @@ -901,7 +901,13 @@ int adjust_shared_perm(const char *path) if (S_ISDIR(old_mode)) { /* Copy read bits to execute bits */ new_mode |= (new_mode & 0444) >> 2; - new_mode |= FORCE_DIR_SET_GID; + + /* + * g+s matters only if any extra access is granted + * based on group membership. + */ + if (FORCE_DIR_SET_GID && (new_mode & 060)) + new_mode |= FORCE_DIR_SET_GID; } if (((old_mode ^ new_mode) & ~S_IFMT) && diff --git a/ref-filter.c b/ref-filter.c index 914908fac5..9dc2cd1451 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -1375,12 +1375,12 @@ static void find_subpos(const char *buf, /* subject is first non-empty line */ *sub = buf; /* subject goes to first empty line before signature begins */ - if ((eol = strstr(*sub, "\n\n"))) { + if ((eol = strstr(*sub, "\n\n")) || + (eol = strstr(*sub, "\r\n\r\n"))) { eol = eol < sigstart ? eol : sigstart; - /* check if message uses CRLF */ - } else if (! (eol = strstr(*sub, "\r\n\r\n"))) { + } else { /* treat whole message as subject */ - eol = strrchr(*sub, '\0'); + eol = sigstart; } buf = eol; *sublen = buf - *sub; diff --git a/repo-settings.c b/repo-settings.c index e8b58151bc..3021921c53 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -43,6 +43,7 @@ void prepare_repo_settings(struct repository *r) /* Defaults modified by feature.* */ if (experimental) { r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_SKIPPING; + r->settings.gc_cruft_packs = 1; } if (manyfiles) { r->settings.index_version = 4; diff --git a/repository.h b/repository.h index 24316ac944..6c461c5b9d 100644 --- a/repository.h +++ b/repository.h @@ -34,6 +34,7 @@ struct repo_settings { int commit_graph_generation_version; int commit_graph_read_changed_paths; int gc_write_commit_graph; + int gc_cruft_packs; int fetch_write_commit_graph; int command_requires_full_index; int sparse_index; @@ -128,6 +128,7 @@ int reset_head(struct repository *r, const struct reset_head_opts *opts) unpack_tree_opts.update = 1; unpack_tree_opts.merge = 1; unpack_tree_opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ + unpack_tree_opts.skip_cache_tree_update = 1; init_checkout_metadata(&unpack_tree_opts.meta, switch_to_branch, oid, NULL); if (reset_hard) unpack_tree_opts.reset = UNPACK_RESET_PROTECT_UNTRACKED; diff --git a/revision.c b/revision.c index 0760e78936..c6b3996583 100644 --- a/revision.c +++ b/revision.c @@ -1865,30 +1865,15 @@ void repo_init_revisions(struct repository *r, struct rev_info *revs, const char *prefix) { - memset(revs, 0, sizeof(*revs)); + struct rev_info blank = REV_INFO_INIT; + memcpy(revs, &blank, sizeof(*revs)); revs->repo = r; - revs->abbrev = DEFAULT_ABBREV; - revs->simplify_history = 1; revs->pruning.repo = r; - revs->pruning.flags.recursive = 1; - revs->pruning.flags.quick = 1; revs->pruning.add_remove = file_add_remove; revs->pruning.change = file_change; revs->pruning.change_fn_data = revs; - revs->sort_order = REV_SORT_IN_GRAPH_ORDER; - revs->dense = 1; revs->prefix = prefix; - revs->max_age = -1; - revs->max_age_as_filter = -1; - revs->min_age = -1; - revs->skip_count = -1; - revs->max_count = -1; - revs->max_parents = -1; - revs->expand_tabs_in_log = -1; - - revs->commit_format = CMIT_FMT_DEFAULT; - revs->expand_tabs_in_log_default = 8; grep_init(&revs->grep_filter, revs->repo); revs->grep_filter.status_only = 1; diff --git a/revision.h b/revision.h index afe1b77985..8493a3f3b9 100644 --- a/revision.h +++ b/revision.h @@ -357,7 +357,23 @@ struct rev_info { * called before release_revisions() the "struct rev_info" can be left * uninitialized. */ -#define REV_INFO_INIT { 0 } +#define REV_INFO_INIT { \ + .abbrev = DEFAULT_ABBREV, \ + .simplify_history = 1, \ + .pruning.flags.recursive = 1, \ + .pruning.flags.quick = 1, \ + .sort_order = REV_SORT_IN_GRAPH_ORDER, \ + .dense = 1, \ + .max_age = -1, \ + .max_age_as_filter = -1, \ + .min_age = -1, \ + .skip_count = -1, \ + .max_count = -1, \ + .max_parents = -1, \ + .expand_tabs_in_log = -1, \ + .commit_format = CMIT_FMT_DEFAULT, \ + .expand_tabs_in_log_default = 8, \ +} /** * Initialize a rev_info structure with default values. The third parameter may diff --git a/run-command.c b/run-command.c index c772acd743..48b9ba6d6f 100644 --- a/run-command.c +++ b/run-command.c @@ -1004,41 +1004,6 @@ int run_command(struct child_process *cmd) return finish_command(cmd); } -int run_command_v_opt(const char **argv, int opt) -{ - return run_command_v_opt_cd_env(argv, opt, NULL, NULL); -} - -int run_command_v_opt_tr2(const char **argv, int opt, const char *tr2_class) -{ - return run_command_v_opt_cd_env_tr2(argv, opt, NULL, NULL, tr2_class); -} - -int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env) -{ - return run_command_v_opt_cd_env_tr2(argv, opt, dir, env, NULL); -} - -int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir, - const char *const *env, const char *tr2_class) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - strvec_pushv(&cmd.args, argv); - cmd.no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0; - cmd.git_cmd = opt & RUN_GIT_CMD ? 1 : 0; - cmd.stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0; - cmd.silent_exec_failure = opt & RUN_SILENT_EXEC_FAILURE ? 1 : 0; - cmd.use_shell = opt & RUN_USING_SHELL ? 1 : 0; - cmd.clean_on_exit = opt & RUN_CLEAN_ON_EXIT ? 1 : 0; - cmd.wait_after_clean = opt & RUN_WAIT_AFTER_CLEAN ? 1 : 0; - cmd.close_object_store = opt & RUN_CLOSE_OBJECT_STORE ? 1 : 0; - cmd.dir = dir; - if (env) - strvec_pushv(&cmd.env, (const char **)env); - cmd.trace2_child_class = tr2_class; - return run_command(&cmd); -} - #ifndef NO_PTHREADS static pthread_t main_thread; static int main_thread_set; diff --git a/run-command.h b/run-command.h index e3e1ea01ad..072db56a4d 100644 --- a/run-command.h +++ b/run-command.h @@ -150,9 +150,7 @@ struct child_process { } /** - * The functions: child_process_init, start_command, finish_command, - * run_command, run_command_v_opt, run_command_v_opt_cd_env, child_process_clear - * do the following: + * The functions: start_command, finish_command, run_command do the following: * * - If a system call failed, errno is set and -1 is returned. A diagnostic * is printed. @@ -224,36 +222,6 @@ int run_command(struct child_process *); */ int run_auto_maintenance(int quiet); -#define RUN_COMMAND_NO_STDIN (1<<0) -#define RUN_GIT_CMD (1<<1) -#define RUN_COMMAND_STDOUT_TO_STDERR (1<<2) -#define RUN_SILENT_EXEC_FAILURE (1<<3) -#define RUN_USING_SHELL (1<<4) -#define RUN_CLEAN_ON_EXIT (1<<5) -#define RUN_WAIT_AFTER_CLEAN (1<<6) -#define RUN_CLOSE_OBJECT_STORE (1<<7) - -/** - * Convenience functions that encapsulate a sequence of - * start_command() followed by finish_command(). The argument argv - * specifies the program and its arguments. The argument opt is zero - * or more of the flags `RUN_COMMAND_NO_STDIN`, `RUN_GIT_CMD`, - * `RUN_COMMAND_STDOUT_TO_STDERR`, or `RUN_SILENT_EXEC_FAILURE` - * that correspond to the members .no_stdin, .git_cmd, - * .stdout_to_stderr, .silent_exec_failure of `struct child_process`. - * The argument dir corresponds the member .dir. The argument env - * corresponds to the member .env. - */ -int run_command_v_opt(const char **argv, int opt); -int run_command_v_opt_tr2(const char **argv, int opt, const char *tr2_class); -/* - * env (the environment) is to be formatted like environ: "VAR=VALUE". - * To unset an environment variable use just "VAR". - */ -int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env); -int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir, - const char *const *env, const char *tr2_class); - /** * Execute the given command, sending "in" to its stdin, and capturing its * stdout and stderr in the "out" and "err" strbufs. Any of the three may @@ -69,21 +69,18 @@ static void setup_enlistment_directory(int argc, const char **argv, static int run_git(const char *arg, ...) { - struct strvec argv = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; va_list args; const char *p; - int res; va_start(args, arg); - strvec_push(&argv, arg); + strvec_push(&cmd.args, arg); while ((p = va_arg(args, const char *))) - strvec_push(&argv, p); + strvec_push(&cmd.args, p); va_end(args); - res = run_command_v_opt(argv.v, RUN_GIT_CMD); - - strvec_clear(&argv); - return res; + cmd.git_cmd = 1; + return run_command(&cmd); } struct scalar_config { diff --git a/sequencer.c b/sequencer.c index e658df7e8f..a46f3ad2e3 100644 --- a/sequencer.c +++ b/sequencer.c @@ -3183,18 +3183,15 @@ static int rollback_is_safe(void) static int reset_merge(const struct object_id *oid) { - int ret; - struct strvec argv = STRVEC_INIT; + struct child_process cmd = CHILD_PROCESS_INIT; - strvec_pushl(&argv, "reset", "--merge", NULL); + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "reset", "--merge", NULL); if (!is_null_oid(oid)) - strvec_push(&argv, oid_to_hex(oid)); + strvec_push(&cmd.args, oid_to_hex(oid)); - ret = run_command_v_opt(argv.v, RUN_GIT_CMD); - strvec_clear(&argv); - - return ret; + return run_command(&cmd); } static int rollback_single_pick(struct repository *r) @@ -3558,12 +3555,13 @@ static int error_failed_squash(struct repository *r, static int do_exec(struct repository *r, const char *command_line) { - const char *child_argv[] = { NULL, NULL }; + struct child_process cmd = CHILD_PROCESS_INIT; int dirty, status; fprintf(stderr, _("Executing: %s\n"), command_line); - child_argv[0] = command_line; - status = run_command_v_opt(child_argv, RUN_USING_SHELL); + cmd.use_shell = 1; + strvec_push(&cmd.args, command_line); + status = run_command(&cmd); /* force re-reading of the cache */ if (discard_index(r->index) < 0 || repo_read_index(r) < 0) @@ -3750,6 +3748,7 @@ static int do_reset(struct repository *r, unpack_tree_opts.merge = 1; unpack_tree_opts.update = 1; unpack_tree_opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ + unpack_tree_opts.skip_cache_tree_update = 1; init_checkout_metadata(&unpack_tree_opts.meta, name, &oid, NULL); if (repo_read_index_unmerged(r)) { @@ -4130,11 +4129,14 @@ static int write_update_refs_state(struct string_list *refs_to_oids) struct string_list_item *item; char *path; - if (!refs_to_oids->nr) - return 0; - path = rebase_path_update_refs(the_repository->gitdir); + if (!refs_to_oids->nr) { + if (unlink(path) && errno != ENOENT) + result = error_errno(_("could not unlink: %s"), path); + goto cleanup; + } + if (safe_create_leading_directories(path)) { result = error(_("unable to create leading directories of %s"), path); @@ -4867,14 +4869,14 @@ cleanup_head_ref: static int continue_single_pick(struct repository *r, struct replay_opts *opts) { - struct strvec argv = STRVEC_INIT; - int ret; + struct child_process cmd = CHILD_PROCESS_INIT; if (!refs_ref_exists(get_main_ref_store(r), "CHERRY_PICK_HEAD") && !refs_ref_exists(get_main_ref_store(r), "REVERT_HEAD")) return error(_("no cherry-pick or revert in progress")); - strvec_push(&argv, "commit"); + cmd.git_cmd = 1; + strvec_push(&cmd.args, "commit"); /* * continue_single_pick() handles the case of recovering from a @@ -4887,11 +4889,9 @@ static int continue_single_pick(struct repository *r, struct replay_opts *opts) * Include --cleanup=strip as well because we don't want the * "# Conflicts:" messages. */ - strvec_pushl(&argv, "--no-edit", "--cleanup=strip", NULL); + strvec_pushl(&cmd.args, "--no-edit", "--cleanup=strip", NULL); - ret = run_command_v_opt(argv.v, RUN_GIT_CMD); - strvec_clear(&argv); - return ret; + return run_command(&cmd); } static int commit_staged_changes(struct repository *r, diff --git a/sha1dc_git.h b/sha1dc_git.h index 41e1c3fd3f..60e3ce8439 100644 --- a/sha1dc_git.h +++ b/sha1dc_git.h @@ -17,6 +17,7 @@ void git_SHA1DCInit(SHA1_CTX *); void git_SHA1DCFinal(unsigned char [20], SHA1_CTX *); void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *data, unsigned long len); +#define platform_SHA_IS_SHA1DC /* used by "test-tool sha1-is-sha1dc" */ #define platform_SHA_CTX SHA1_CTX #define platform_SHA1_Init git_SHA1DCInit #define platform_SHA1_Update git_SHA1DCUpdate @@ -52,21 +52,24 @@ static void cd_to_homedir(void) static void run_shell(void) { int done = 0; - static const char *help_argv[] = { HELP_COMMAND, NULL }; + struct child_process help_cmd = CHILD_PROCESS_INIT; if (!access(NOLOGIN_COMMAND, F_OK)) { /* Interactive login disabled. */ - const char *argv[] = { NOLOGIN_COMMAND, NULL }; + struct child_process nologin_cmd = CHILD_PROCESS_INIT; int status; - status = run_command_v_opt(argv, 0); + strvec_push(&nologin_cmd.args, NOLOGIN_COMMAND); + status = run_command(&nologin_cmd); if (status < 0) exit(127); exit(status); } /* Print help if enabled */ - run_command_v_opt(help_argv, RUN_SILENT_EXEC_FAILURE); + help_cmd.silent_exec_failure = 1; + strvec_push(&help_cmd.args, HELP_COMMAND); + run_command(&help_cmd); do { const char *prog; @@ -125,9 +128,13 @@ static void run_shell(void) !strcmp(prog, "exit") || !strcmp(prog, "bye")) { done = 1; } else if (is_valid_cmd_name(prog)) { + struct child_process cmd = CHILD_PROCESS_INIT; + full_cmd = make_cmd(prog); argv[0] = full_cmd; - code = run_command_v_opt(argv, RUN_SILENT_EXEC_FAILURE); + cmd.silent_exec_failure = 1; + strvec_pushv(&cmd.args, argv); + code = run_command(&cmd); if (code == -1 && errno == ENOENT) { fprintf(stderr, "unrecognized command '%s'\n", prog); } diff --git a/sparse-index.c b/sparse-index.c index e4a54ce194..8c269dab80 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -493,24 +493,42 @@ void clear_skip_worktree_from_present_files(struct index_state *istate) int dir_found = 1; int i; + int path_count[2] = {0, 0}; + int restarted = 0; if (!core_apply_sparse_checkout || sparse_expect_files_outside_of_patterns) return; + trace2_region_enter("index", "clear_skip_worktree_from_present_files", + istate->repo); restart: for (i = 0; i < istate->cache_nr; i++) { struct cache_entry *ce = istate->cache[i]; - if (ce_skip_worktree(ce) && - path_found(ce->name, &last_dirname, &dir_len, &dir_found)) { - if (S_ISSPARSEDIR(ce->ce_mode)) { - ensure_full_index(istate); - goto restart; + if (ce_skip_worktree(ce)) { + path_count[restarted]++; + if (path_found(ce->name, &last_dirname, &dir_len, &dir_found)) { + if (S_ISSPARSEDIR(ce->ce_mode)) { + if (restarted) + BUG("ensure-full-index did not fully flatten?"); + ensure_full_index(istate); + restarted = 1; + goto restart; + } + ce->ce_flags &= ~CE_SKIP_WORKTREE; } - ce->ce_flags &= ~CE_SKIP_WORKTREE; } } + + if (path_count[0]) + trace2_data_intmax("index", istate->repo, + "sparse_path_count", path_count[0]); + if (restarted) + trace2_data_intmax("index", istate->repo, + "sparse_path_count_full", path_count[1]); + trace2_region_leave("index", "clear_skip_worktree_from_present_files", + istate->repo); } /* diff --git a/t/helper/test-cache-tree.c b/t/helper/test-cache-tree.c new file mode 100644 index 0000000000..93051b25f5 --- /dev/null +++ b/t/helper/test-cache-tree.c @@ -0,0 +1,64 @@ +#include "test-tool.h" +#include "cache.h" +#include "tree.h" +#include "cache-tree.h" +#include "parse-options.h" + +static char const * const test_cache_tree_usage[] = { + N_("test-tool cache-tree <options> (control|prime|update)"), + NULL +}; + +int cmd__cache_tree(int argc, const char **argv) +{ + struct object_id oid; + struct tree *tree; + int empty = 0; + int invalidate_qty = 0; + int i; + + struct option options[] = { + OPT_BOOL(0, "empty", &empty, + N_("clear the cache tree before each iteration")), + OPT_INTEGER_F(0, "invalidate", &invalidate_qty, + N_("number of entries in the cache tree to invalidate (default 0)"), + PARSE_OPT_NONEG), + OPT_END() + }; + + setup_git_directory(); + + argc = parse_options(argc, argv, NULL, options, test_cache_tree_usage, 0); + + if (read_cache() < 0) + die(_("unable to read index file")); + + oidcpy(&oid, &the_index.cache_tree->oid); + tree = parse_tree_indirect(&oid); + if (!tree) + die(_("not a tree object: %s"), oid_to_hex(&oid)); + + if (empty) { + /* clear the cache tree & allocate a new one */ + cache_tree_free(&the_index.cache_tree); + the_index.cache_tree = cache_tree(); + } else if (invalidate_qty) { + /* invalidate the specified number of unique paths */ + float f_interval = (float)the_index.cache_nr / invalidate_qty; + int interval = f_interval < 1.0 ? 1 : (int)f_interval; + for (i = 0; i < invalidate_qty && i * interval < the_index.cache_nr; i++) + cache_tree_invalidate_path(&the_index, the_index.cache[i * interval]->name); + } + + if (argc != 1) + usage_with_options(test_cache_tree_usage, options); + else if (!strcmp(argv[0], "prime")) + prime_cache_tree(the_repository, &the_index, tree); + else if (!strcmp(argv[0], "update")) + cache_tree_update(&the_index, WRITE_TREE_SILENT | WRITE_TREE_REPAIR); + /* use "control" subcommand to specify no-op */ + else if (!!strcmp(argv[0], "control")) + die(_("Unhandled subcommand '%s'"), argv[0]); + + return 0; +} diff --git a/t/helper/test-fake-ssh.c b/t/helper/test-fake-ssh.c index 12beee99ad..2e576bcc11 100644 --- a/t/helper/test-fake-ssh.c +++ b/t/helper/test-fake-ssh.c @@ -8,7 +8,7 @@ int cmd_main(int argc, const char **argv) struct strbuf buf = STRBUF_INIT; FILE *f; int i; - const char *child_argv[] = { NULL, NULL }; + struct child_process cmd = CHILD_PROCESS_INIT; /* First, print all parameters into $TRASH_DIRECTORY/ssh-output */ if (!trash_directory) @@ -25,6 +25,7 @@ int cmd_main(int argc, const char **argv) /* Now, evaluate the *last* parameter */ if (argc < 2) return 0; - child_argv[0] = argv[argc - 1]; - return run_command_v_opt(child_argv, RUN_USING_SHELL); + cmd.use_shell = 1; + strvec_push(&cmd.args, argv[argc - 1]); + return run_command(&cmd); } diff --git a/t/helper/test-sha1.c b/t/helper/test-sha1.c index d860c387c3..71fe5c6145 100644 --- a/t/helper/test-sha1.c +++ b/t/helper/test-sha1.c @@ -5,3 +5,11 @@ int cmd__sha1(int ac, const char **av) { return cmd_hash_impl(ac, av, GIT_HASH_SHA1); } + +int cmd__sha1_is_sha1dc(int argc UNUSED, const char **argv UNUSED) +{ +#ifdef platform_SHA_IS_SHA1DC + return 0; +#endif + return 1; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 01cda9358d..7eb1a26a30 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -14,6 +14,7 @@ static struct test_cmd cmds[] = { { "bitmap", cmd__bitmap }, { "bloom", cmd__bloom }, { "bundle-uri", cmd__bundle_uri }, + { "cache-tree", cmd__cache_tree }, { "chmtime", cmd__chmtime }, { "config", cmd__config }, { "crontab", cmd__crontab }, @@ -73,6 +74,7 @@ static struct test_cmd cmds[] = { { "scrap-cache-tree", cmd__scrap_cache_tree }, { "serve-v2", cmd__serve_v2 }, { "sha1", cmd__sha1 }, + { "sha1-is-sha1dc", cmd__sha1_is_sha1dc }, { "sha256", cmd__sha256 }, { "sigchain", cmd__sigchain }, { "simple-ipc", cmd__simple_ipc }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index ca2948066f..da7cd6351a 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -8,6 +8,7 @@ int cmd__advise_if_enabled(int argc, const char **argv); int cmd__bitmap(int argc, const char **argv); int cmd__bloom(int argc, const char **argv); int cmd__bundle_uri(int argc, const char **argv); +int cmd__cache_tree(int argc, const char **argv); int cmd__chmtime(int argc, const char **argv); int cmd__config(int argc, const char **argv); int cmd__crontab(int argc, const char **argv); @@ -66,6 +67,7 @@ int cmd__run_command(int argc, const char **argv); int cmd__scrap_cache_tree(int argc, const char **argv); int cmd__serve_v2(int argc, const char **argv); int cmd__sha1(int argc, const char **argv); +int cmd__sha1_is_sha1dc(int argc, const char **argv); int cmd__oid_array(int argc, const char **argv); int cmd__sha256(int argc, const char **argv); int cmd__sigchain(int argc, const char **argv); diff --git a/t/helper/test-trace2.c b/t/helper/test-trace2.c index 1b092c6071..f374c21ec3 100644 --- a/t/helper/test-trace2.c +++ b/t/helper/test-trace2.c @@ -132,6 +132,7 @@ static int ut_003error(int argc, const char **argv) */ static int ut_004child(int argc, const char **argv) { + struct child_process cmd = CHILD_PROCESS_INIT; int result; /* @@ -141,7 +142,8 @@ static int ut_004child(int argc, const char **argv) if (!argc) return 0; - result = run_command_v_opt(argv, 0); + strvec_pushv(&cmd.args, argv); + result = run_command(&cmd); exit(result); } diff --git a/t/perf/p0006-read-tree-checkout.sh b/t/perf/p0006-read-tree-checkout.sh index c481c012d2..325566e18e 100755 --- a/t/perf/p0006-read-tree-checkout.sh +++ b/t/perf/p0006-read-tree-checkout.sh @@ -49,6 +49,14 @@ test_perf "read-tree br_base br_ballast ($nr_files)" ' git read-tree -n -m br_base br_ballast ' +test_perf "read-tree br_ballast_plus_1 ($nr_files)" ' + # Run read-tree 100 times for clearer performance results & comparisons + for i in $(test_seq 100) + do + git read-tree -n -m br_ballast_plus_1 || return 1 + done +' + test_perf "switch between br_base br_ballast ($nr_files)" ' git checkout -q br_base && git checkout -q br_ballast diff --git a/t/perf/p0090-cache-tree.sh b/t/perf/p0090-cache-tree.sh new file mode 100755 index 0000000000..a8eabca2c4 --- /dev/null +++ b/t/perf/p0090-cache-tree.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +test_description="Tests performance of cache tree update operations" + +. ./perf-lib.sh + +test_perf_large_repo +test_checkout_worktree + +count=100 + +test_expect_success 'setup cache tree' ' + git write-tree +' + +test_cache_tree () { + test_perf "$1, $3" " + for i in \$(test_seq $count) + do + test-tool cache-tree $4 $2 + done + " +} + +test_cache_tree_update_functions () { + test_cache_tree 'no-op' 'control' "$1" "$2" + test_cache_tree 'prime_cache_tree' 'prime' "$1" "$2" + test_cache_tree 'cache_tree_update' 'update' "$1" "$2" +} + +test_cache_tree_update_functions "clean" "" +test_cache_tree_update_functions "invalidate 2" "--invalidate 2" +test_cache_tree_update_functions "invalidate 50" "--invalidate 50" +test_cache_tree_update_functions "empty" "--empty" + +test_done diff --git a/t/perf/p7102-reset.sh b/t/perf/p7102-reset.sh new file mode 100755 index 0000000000..9b039e8691 --- /dev/null +++ b/t/perf/p7102-reset.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +test_description='performance of reset' +. ./perf-lib.sh + +test_perf_default_repo +test_checkout_worktree + +test_perf 'reset --hard with change in tree' ' + base=$(git rev-parse HEAD) && + test_commit --no-tag A && + new=$(git rev-parse HEAD) && + + for i in $(test_seq 10) + do + git reset --hard $new && + git reset --hard $base || return $? + done +' + +test_done diff --git a/t/t0013-sha1dc.sh b/t/t0013-sha1dc.sh index 9ad76080aa..5324047689 100755 --- a/t/t0013-sha1dc.sh +++ b/t/t0013-sha1dc.sh @@ -6,9 +6,11 @@ TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh TEST_DATA="$TEST_DIRECTORY/t0013" -if test -z "$DC_SHA1" +test_lazy_prereq SHA1_IS_SHA1DC 'test-tool sha1-is-sha1dc' + +if ! test_have_prereq SHA1_IS_SHA1DC then - skip_all='skipping sha1 collision tests, DC_SHA1 not set' + skip_all='skipping sha1 collision tests, not using sha1collisiondetection' test_done fi diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 7b5423eebd..e2411f6a9b 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -130,7 +130,8 @@ World EOF test_expect_success 'run_command runs in parallel with more jobs available than tasks' ' - test-tool run-command run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test-tool run-command run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>actual && + test_must_be_empty out && test_cmp expect actual ' @@ -141,7 +142,8 @@ test_expect_success 'run_command runs ungrouped in parallel with more jobs avail ' test_expect_success 'run_command runs in parallel with as many jobs as tasks' ' - test-tool run-command run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test-tool run-command run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>actual && + test_must_be_empty out && test_cmp expect actual ' @@ -152,7 +154,8 @@ test_expect_success 'run_command runs ungrouped in parallel with as many jobs as ' test_expect_success 'run_command runs in parallel with more tasks than jobs available' ' - test-tool run-command run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test-tool run-command run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>actual && + test_must_be_empty out && test_cmp expect actual ' @@ -172,7 +175,8 @@ asking for a quick stop EOF test_expect_success 'run_command is asked to abort gracefully' ' - test-tool run-command run-command-abort 3 false 2>actual && + test-tool run-command run-command-abort 3 false >out 2>actual && + test_must_be_empty out && test_cmp expect actual ' @@ -187,7 +191,8 @@ no further jobs available EOF test_expect_success 'run_command outputs ' ' - test-tool run-command run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test-tool run-command run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>actual && + test_must_be_empty out && test_cmp expect actual ' diff --git a/t/t1022-read-tree-partial-clone.sh b/t/t1022-read-tree-partial-clone.sh index a9953b6a71..da53971635 100755 --- a/t/t1022-read-tree-partial-clone.sh +++ b/t/t1022-read-tree-partial-clone.sh @@ -19,7 +19,7 @@ test_expect_success 'read-tree in partial clone prefetches in one batch' ' git -C server config uploadpack.allowfilter 1 && git -C server config uploadpack.allowanysha1inwant 1 && git clone --bare --filter=blob:none "file://$(pwd)/server" client && - GIT_TRACE_PACKET="$(pwd)/trace" git -C client read-tree $TREE && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client read-tree $TREE $TREE && # "done" marks the end of negotiation (once per fetch). Expect that # only one fetch occurs. diff --git a/t/t1800-hook.sh b/t/t1800-hook.sh index 43fcb7c0bf..2ef3579fa7 100755 --- a/t/t1800-hook.sh +++ b/t/t1800-hook.sh @@ -95,7 +95,7 @@ test_expect_success 'git hook run -- out-of-repo runs excluded' ' test_expect_success 'git -c core.hooksPath=<PATH> hook run' ' mkdir my-hooks && write_script my-hooks/test-hook <<-\EOF && - echo Hook ran $1 >>actual + echo Hook ran $1 EOF cat >expect <<-\EOF && diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 7f605f865b..5a169b68d6 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -279,6 +279,42 @@ test_expect_success 'git branch -M and -C fail on detached HEAD' ' test_cmp expect err ' +test_expect_success 'git branch -d on orphan HEAD (merged)' ' + test_when_finished git checkout main && + git checkout --orphan orphan && + test_when_finished "rm -rf .git/objects/commit-graph*" && + git commit-graph write --reachable && + git branch --track to-delete main && + git branch -d to-delete +' + +test_expect_success 'git branch -d on orphan HEAD (merged, graph)' ' + test_when_finished git checkout main && + git checkout --orphan orphan && + git branch --track to-delete main && + git branch -d to-delete +' + +test_expect_success 'git branch -d on orphan HEAD (unmerged)' ' + test_when_finished git checkout main && + git checkout --orphan orphan && + test_when_finished "git branch -D to-delete" && + git branch to-delete main && + test_must_fail git branch -d to-delete 2>err && + grep "not fully merged" err +' + +test_expect_success 'git branch -d on orphan HEAD (unmerged, graph)' ' + test_when_finished git checkout main && + git checkout --orphan orphan && + test_when_finished "git branch -D to-delete" && + git branch to-delete main && + test_when_finished "rm -rf .git/objects/commit-graph*" && + git commit-graph write --reachable && + test_must_fail git branch -d to-delete 2>err && + grep "not fully merged" err +' + test_expect_success 'git branch -v -d t should work' ' git branch t && git rev-parse --verify refs/heads/t && diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index 4f5abb5ad2..462cefd25d 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -1964,6 +1964,113 @@ test_expect_success 'respect user edits to update-ref steps' ' test_cmp_rev HEAD refs/heads/no-conflict-branch ' +test_expect_success '--update-refs: all update-ref lines removed' ' + git checkout -b test-refs-not-removed no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + git branch -f tip && + + test_commit test-refs-not-removed && + git commit --amend --fixup first && + + git rev-parse first second third tip no-conflict-branch >expect-oids && + + ( + set_cat_todo_editor && + test_must_fail git rebase -i --update-refs base >todo.raw && + sed -e "/^update-ref/d" <todo.raw >todo + ) && + ( + set_replace_editor todo && + git rebase -i --update-refs base + ) && + + # Ensure refs are not deleted and their OIDs have not changed + git rev-parse first second third tip no-conflict-branch >actual-oids && + test_cmp expect-oids actual-oids +' + +test_expect_success '--update-refs: all update-ref lines removed, then some re-added' ' + git checkout -b test-refs-not-removed2 no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + git branch -f tip && + + test_commit test-refs-not-removed2 && + git commit --amend --fixup first && + + git rev-parse first second third >expect-oids && + + ( + set_cat_todo_editor && + test_must_fail git rebase -i \ + --autosquash --update-refs \ + base >todo.raw && + sed -e "/^update-ref/d" <todo.raw >todo + ) && + + # Add a break to the end of the todo so we can edit later + echo "break" >>todo && + + ( + set_replace_editor todo && + git rebase -i --autosquash --update-refs base && + echo "update-ref refs/heads/tip" >todo && + git rebase --edit-todo && + git rebase --continue + ) && + + # Ensure first/second/third are unchanged, but tip is updated + git rev-parse first second third >actual-oids && + test_cmp expect-oids actual-oids && + test_cmp_rev HEAD tip +' + +test_expect_success '--update-refs: --edit-todo with no update-ref lines' ' + git checkout -b test-refs-not-removed3 no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + git branch -f tip && + + test_commit test-refs-not-removed3 && + git commit --amend --fixup first && + + git rev-parse first second third tip no-conflict-branch >expect-oids && + + ( + set_cat_todo_editor && + test_must_fail git rebase -i \ + --autosquash --update-refs \ + base >todo.raw && + sed -e "/^update-ref/d" <todo.raw >todo + ) && + + # Add a break to the beginning of the todo so we can resume with no + # update-ref lines + echo "break" >todo.new && + cat todo >>todo.new && + + ( + set_replace_editor todo.new && + git rebase -i --autosquash --update-refs base && + + # Make no changes when editing so update-refs is still empty + cat todo >todo.new && + git rebase --edit-todo && + git rebase --continue + ) && + + # Ensure refs are not deleted and their OIDs have not changed + git rev-parse first second third tip no-conflict-branch >actual-oids && + test_cmp expect-oids actual-oids +' + test_expect_success '--update-refs: check failed ref update' ' git checkout -B update-refs-error no-conflict-branch && git branch -f base HEAD~4 && diff --git a/t/t5000-tar-tree.sh b/t/t5000-tar-tree.sh index eaa0b22ece..d473048138 100755 --- a/t/t5000-tar-tree.sh +++ b/t/t5000-tar-tree.sh @@ -342,6 +342,13 @@ test_expect_success 'only enabled filters are available remotely' ' test_cmp_bin remote.bar config.bar ' +test_expect_success 'invalid filter is reported only once' ' + test_must_fail git -c tar.invalid.command= archive --format=invalid \ + HEAD >out 2>err && + test_must_be_empty out && + test_line_count = 1 err +' + test_expect_success 'git archive --format=tgz' ' git archive --format=tgz HEAD >j.tgz ' diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 79dc470c01..4f2bfaf005 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -1853,37 +1853,6 @@ test_expect_success 'refuse to push a hidden ref, and make sure do not pollute t test_dir_is_empty testrepo/.git/objects/pack ' -test_expect_success LIBCURL 'fetch warns or fails when using username:password' ' - message="URL '\''https://username:<redacted>@localhost/'\'' uses plaintext credentials" && - test_must_fail git -c transfer.credentialsInUrl=allow fetch https://username:password@localhost 2>err && - ! grep "$message" err && - - test_must_fail git -c transfer.credentialsInUrl=warn fetch https://username:password@localhost 2>err && - grep "warning: $message" err >warnings && - test_line_count = 3 warnings && - - test_must_fail git -c transfer.credentialsInUrl=die fetch https://username:password@localhost 2>err && - grep "fatal: $message" err >warnings && - test_line_count = 1 warnings && - - test_must_fail git -c transfer.credentialsInUrl=die fetch https://username:@localhost 2>err && - grep "fatal: $message" err >warnings && - test_line_count = 1 warnings -' - - -test_expect_success LIBCURL 'push warns or fails when using username:password' ' - message="URL '\''https://username:<redacted>@localhost/'\'' uses plaintext credentials" && - test_must_fail git -c transfer.credentialsInUrl=allow push https://username:password@localhost 2>err && - ! grep "$message" err && - - test_must_fail git -c transfer.credentialsInUrl=warn push https://username:password@localhost 2>err && - grep "warning: $message" err >warnings && - test_must_fail git -c transfer.credentialsInUrl=die push https://username:password@localhost 2>err && - grep "fatal: $message" err >warnings && - test_line_count = 1 warnings -' - test_expect_success 'push with config push.useBitmaps' ' mk_test testrepo heads/main && git checkout main && diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh index 75da8acf8f..b9546ef8e5 100755 --- a/t/t5526-fetch-submodules.sh +++ b/t/t5526-fetch-submodules.sh @@ -178,6 +178,7 @@ test_expect_success "submodule.recurse option triggers recursive fetch" ' ' test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" ' + test_when_finished "rm -f trace.out" && add_submodule_commits && ( cd downstream && @@ -705,15 +706,22 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git test_expect_success 'fetching submodules respects parallel settings' ' git config fetch.recurseSubmodules true && + test_when_finished "rm -f downstream/trace.out" && ( cd downstream && GIT_TRACE=$(pwd)/trace.out git fetch && grep "1 tasks" trace.out && + >trace.out && + GIT_TRACE=$(pwd)/trace.out git fetch --jobs 7 && grep "7 tasks" trace.out && + >trace.out && + git config submodule.fetchJobs 8 && GIT_TRACE=$(pwd)/trace.out git fetch && grep "8 tasks" trace.out && + >trace.out && + GIT_TRACE=$(pwd)/trace.out git fetch --jobs 9 && grep "9 tasks" trace.out && >trace.out && diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index 6a38294a47..64c6c9f59e 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -580,4 +580,81 @@ test_expect_success 'passing hostname resolution information works' ' git -c "http.curloptResolve=$BOGUS_HOST:$LIB_HTTPD_PORT:127.0.0.1" ls-remote "$BOGUS_HTTPD_URL/smart/repo.git" >/dev/null ' +# here user%40host is the URL-encoded version of user@host, +# which is our intentionally-odd username to catch parsing errors +url_user=$HTTPD_URL_USER/auth/smart/repo.git +url_userpass=$HTTPD_URL_USER_PASS/auth/smart/repo.git +url_userblank=$HTTPD_PROTO://user%40host:@$HTTPD_DEST/auth/smart/repo.git +message="URL .*:<redacted>@.* uses plaintext credentials" + +test_expect_success 'clone warns or fails when using username:password' ' + test_when_finished "rm -rf attempt*" && + + git -c transfer.credentialsInUrl=allow \ + clone $url_userpass attempt1 2>err && + ! grep "$message" err && + + git -c transfer.credentialsInUrl=warn \ + clone $url_userpass attempt2 2>err && + grep "warning: $message" err >warnings && + test_line_count -ge 1 warnings && + + test_must_fail git -c transfer.credentialsInUrl=die \ + clone $url_userpass attempt3 2>err && + grep "fatal: $message" err >warnings && + test_line_count -ge 1 warnings && + + test_must_fail git -c transfer.credentialsInUrl=die \ + clone $url_userblank attempt4 2>err && + grep "fatal: $message" err >warnings && + test_line_count -ge 1 warnings +' + +test_expect_success 'clone does not detect username:password when it is https://username@domain:port/' ' + test_when_finished "rm -rf attempt1" && + + # we are relying on lib-httpd for url construction, so document our + # assumptions + case "$HTTPD_URL_USER" in + *:[0-9]*) : ok ;; + *) BUG "httpd url does not have port: $HTTPD_URL_USER" + esac && + + git -c transfer.credentialsInUrl=warn clone $url_user attempt1 2>err && + ! grep "uses plaintext credentials" err +' + +test_expect_success 'fetch warns or fails when using username:password' ' + git -c transfer.credentialsInUrl=allow fetch $url_userpass 2>err && + ! grep "$message" err && + + git -c transfer.credentialsInUrl=warn fetch $url_userpass 2>err && + grep "warning: $message" err >warnings && + test_line_count -ge 1 warnings && + + test_must_fail git -c transfer.credentialsInUrl=die \ + fetch $url_userpass 2>err && + grep "fatal: $message" err >warnings && + test_line_count -ge 1 warnings && + + test_must_fail git -c transfer.credentialsInUrl=die \ + fetch $url_userblank 2>err && + grep "fatal: $message" err >warnings && + test_line_count -ge 1 warnings +' + + +test_expect_success 'push warns or fails when using username:password' ' + git -c transfer.credentialsInUrl=allow push $url_userpass 2>err && + ! grep "$message" err && + + git -c transfer.credentialsInUrl=warn push $url_userpass 2>err && + grep "warning: $message" err >warnings && + + test_must_fail git -c transfer.credentialsInUrl=die \ + push $url_userpass 2>err && + grep "fatal: $message" err >warnings && + test_line_count -ge 1 warnings +' + test_done diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 45f0803ed4..b2524a24c2 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -71,29 +71,6 @@ test_expect_success 'clone respects GIT_WORK_TREE' ' ' -test_expect_success LIBCURL 'clone warns or fails when using username:password' ' - message="URL '\''https://username:<redacted>@localhost/'\'' uses plaintext credentials" && - test_must_fail git -c transfer.credentialsInUrl=allow clone https://username:password@localhost attempt1 2>err && - ! grep "$message" err && - - test_must_fail git -c transfer.credentialsInUrl=warn clone https://username:password@localhost attempt2 2>err && - grep "warning: $message" err >warnings && - test_line_count = 2 warnings && - - test_must_fail git -c transfer.credentialsInUrl=die clone https://username:password@localhost attempt3 2>err && - grep "fatal: $message" err >warnings && - test_line_count = 1 warnings && - - test_must_fail git -c transfer.credentialsInUrl=die clone https://username:@localhost attempt3 2>err && - grep "fatal: $message" err >warnings && - test_line_count = 1 warnings -' - -test_expect_success LIBCURL 'clone does not detect username:password when it is https://username@domain:port/' ' - test_must_fail git -c transfer.credentialsInUrl=warn clone https://username@localhost:8080 attempt3 2>err && - ! grep "uses plaintext credentials" err -' - test_expect_success 'clone from hooks' ' test_create_repo r0 && diff --git a/t/t6300-for-each-ref.sh b/t/t6300-for-each-ref.sh index dcaab7265f..fa38b87441 100755 --- a/t/t6300-for-each-ref.sh +++ b/t/t6300-for-each-ref.sh @@ -1406,4 +1406,44 @@ test_expect_success 'for-each-ref reports broken tags' ' refs/tags/broken-tag-* ' +test_expect_success 'set up tag with signature and no blank lines' ' + git tag -F - fake-sig-no-blanks <<-\EOF + this is the subject + -----BEGIN PGP SIGNATURE----- + not a real signature, but we just care about the + subject/body parsing. It is important here that + there are no blank lines in the signature. + -----END PGP SIGNATURE----- + EOF +' + +test_atom refs/tags/fake-sig-no-blanks contents:subject 'this is the subject' +test_atom refs/tags/fake-sig-no-blanks contents:body '' +test_atom refs/tags/fake-sig-no-blanks contents:signature "$sig" + +test_expect_success 'set up tag with CRLF signature' ' + append_cr <<-\EOF | + this is the subject + -----BEGIN PGP SIGNATURE----- + + not a real signature, but we just care about + the subject/body parsing. It is important here + that there is a blank line separating this + from the signature header. + -----END PGP SIGNATURE----- + EOF + git tag -F - --cleanup=verbatim fake-sig-crlf +' + +test_atom refs/tags/fake-sig-crlf contents:subject 'this is the subject' +test_atom refs/tags/fake-sig-crlf contents:body '' + +# CRLF is retained in the signature, so we have to pass our expected value +# through append_cr. But test_atom requires a shell string, which means command +# substitution, and the shell will strip trailing newlines from the output of +# the substitution. Hack around it by adding and then removing a dummy line. +sig_crlf="$(printf "%s" "$sig" | append_cr; echo dummy)" +sig_crlf=${sig_crlf%dummy} +test_atom refs/tags/fake-sig-crlf contents:signature "$sig_crlf" + test_done diff --git a/t/t6500-gc.sh b/t/t6500-gc.sh index cd6c53360d..d9acb63951 100755 --- a/t/t6500-gc.sh +++ b/t/t6500-gc.sh @@ -202,6 +202,102 @@ test_expect_success 'one of gc.reflogExpire{Unreachable,}=never does not skip "e grep -E "^trace: (built-in|exec|run_command): git reflog expire --" trace.out ' +prepare_cruft_history () { + test_commit base && + + test_commit --no-tag foo && + test_commit --no-tag bar && + git reset HEAD^^ +} + +assert_cruft_packs () { + find .git/objects/pack -name "*.mtimes" >mtimes && + sed -e 's/\.mtimes$/\.pack/g' mtimes >packs && + + test_file_not_empty packs && + while read pack + do + test_path_is_file "$pack" || return 1 + done <packs +} + +assert_no_cruft_packs () { + find .git/objects/pack -name "*.mtimes" >mtimes && + test_must_be_empty mtimes +} + +test_expect_success 'gc --cruft generates a cruft pack' ' + test_when_finished "rm -fr crufts" && + git init crufts && + ( + cd crufts && + + prepare_cruft_history && + git gc --cruft && + assert_cruft_packs + ) +' + +test_expect_success 'gc.cruftPacks=true generates a cruft pack' ' + test_when_finished "rm -fr crufts" && + git init crufts && + ( + cd crufts && + + prepare_cruft_history && + git -c gc.cruftPacks=true gc && + assert_cruft_packs + ) +' + +test_expect_success 'feature.experimental=true generates a cruft pack' ' + git init crufts && + test_when_finished "rm -fr crufts" && + ( + cd crufts && + + prepare_cruft_history && + git -c feature.experimental=true gc && + assert_cruft_packs + ) +' + +test_expect_success 'feature.experimental=false allows explicit cruft packs' ' + git init crufts && + test_when_finished "rm -fr crufts" && + ( + cd crufts && + + prepare_cruft_history && + git -c gc.cruftPacks=true -c feature.experimental=false gc && + assert_cruft_packs + ) +' + +test_expect_success 'feature.experimental=true can be overridden' ' + git init crufts && + test_when_finished "rm -fr crufts" && + ( + cd crufts && + + prepare_cruft_history && + git -c feature.expiremental=true -c gc.cruftPacks=false gc && + assert_no_cruft_packs + ) +' + +test_expect_success 'feature.experimental=false avoids cruft packs by default' ' + git init crufts && + test_when_finished "rm -fr crufts" && + ( + cd crufts && + + prepare_cruft_history && + git -c feature.experimental=false gc && + assert_no_cruft_packs + ) +' + run_and_wait_for_auto_gc () { # We read stdout from gc for the side effect of waiting until the # background gc process exits, closing its fd 9. Furthermore, the diff --git a/t/t7001-mv.sh b/t/t7001-mv.sh index 8c37bceb33..d72cef8826 100755 --- a/t/t7001-mv.sh +++ b/t/t7001-mv.sh @@ -60,8 +60,8 @@ test_expect_success 'checking the commit' ' test_expect_success 'mv --dry-run does not move file' ' git mv -n path0/COPYING MOVED && - test -f path0/COPYING && - test ! -f MOVED + test_path_is_file path0/COPYING && + test_path_is_missing MOVED ' test_expect_success 'checking -k on non-existing file' ' @@ -71,25 +71,25 @@ test_expect_success 'checking -k on non-existing file' ' test_expect_success 'checking -k on untracked file' ' >untracked1 && git mv -k untracked1 path0 && - test -f untracked1 && - test ! -f path0/untracked1 + test_path_is_file untracked1 && + test_path_is_missing path0/untracked1 ' test_expect_success 'checking -k on multiple untracked files' ' >untracked2 && git mv -k untracked1 untracked2 path0 && - test -f untracked1 && - test -f untracked2 && - test ! -f path0/untracked1 && - test ! -f path0/untracked2 + test_path_is_file untracked1 && + test_path_is_file untracked2 && + test_path_is_missing path0/untracked1 && + test_path_is_missing path0/untracked2 ' test_expect_success 'checking -f on untracked file with existing target' ' >path0/untracked1 && test_must_fail git mv -f untracked1 path0 && - test ! -f .git/index.lock && - test -f untracked1 && - test -f path0/untracked1 + test_path_is_missing .git/index.lock && + test_path_is_file untracked1 && + test_path_is_file path0/untracked1 ' # clean up the mess in case bad things happen @@ -215,8 +215,8 @@ test_expect_success 'absolute pathname' ' git add sub/file && git mv sub "$(pwd)/in" && - ! test -d sub && - test -d in && + test_path_is_missing sub && + test_path_is_dir in && git ls-files --error-unmatch in/file ) ' @@ -234,8 +234,8 @@ test_expect_success 'absolute pathname outside should fail' ' git add sub/file && test_must_fail git mv sub "$out/out" && - test -d sub && - ! test -d ../in && + test_path_is_dir sub && + test_path_is_missing ../in && git ls-files --error-unmatch sub/file ) ' @@ -295,8 +295,8 @@ test_expect_success 'git mv should overwrite symlink to a file' ' git add moved && test_must_fail git mv moved symlink && git mv -f moved symlink && - ! test -e moved && - test -f symlink && + test_path_is_missing moved && + test_path_is_file symlink && test "$(cat symlink)" = 1 && git update-index --refresh && git diff-files --quiet @@ -312,13 +312,13 @@ test_expect_success 'git mv should overwrite file with a symlink' ' git add moved && test_must_fail git mv symlink moved && git mv -f symlink moved && - ! test -e symlink && + test_path_is_missing symlink && git update-index --refresh && git diff-files --quiet ' test_expect_success SYMLINKS 'check moved symlink' ' - test -h moved + test_path_is_symlink moved ' rm -f moved symlink @@ -352,7 +352,7 @@ test_expect_success 'git mv moves a submodule with a .git directory and no .gitm ) && mkdir mod && git mv sub mod/sub && - ! test -e sub && + test_path_is_missing sub && test "$entry" = "$(git ls-files --stage mod/sub | cut -f 1)" && git -C mod/sub status && git update-index --refresh && @@ -372,7 +372,7 @@ test_expect_success 'git mv moves a submodule with a .git directory and .gitmodu ) && mkdir mod && git mv sub mod/sub && - ! test -e sub && + test_path_is_missing sub && test "$entry" = "$(git ls-files --stage mod/sub | cut -f 1)" && git -C mod/sub status && echo mod/sub >expected && @@ -389,7 +389,7 @@ test_expect_success 'git mv moves a submodule with gitfile' ' entry="$(git ls-files --stage sub | cut -f 1)" && mkdir mod && git -C mod mv ../sub/ . && - ! test -e sub && + test_path_is_missing sub && test "$entry" = "$(git ls-files --stage mod/sub | cut -f 1)" && git -C mod/sub status && echo mod/sub >expected && @@ -408,7 +408,7 @@ test_expect_success 'mv does not complain when no .gitmodules file is found' ' mkdir mod && git mv sub mod/sub 2>actual.err && test_must_be_empty actual.err && - ! test -e sub && + test_path_is_missing sub && test "$entry" = "$(git ls-files --stage mod/sub | cut -f 1)" && git -C mod/sub status && git update-index --refresh && @@ -423,13 +423,13 @@ test_expect_success 'mv will error out on a modified .gitmodules file unless sta entry="$(git ls-files --stage sub | cut -f 1)" && mkdir mod && test_must_fail git mv sub mod/sub 2>actual.err && - test -s actual.err && - test -e sub && + test_file_not_empty actual.err && + test_path_exists sub && git diff-files --quiet -- sub && git add .gitmodules && git mv sub mod/sub 2>actual.err && test_must_be_empty actual.err && - ! test -e sub && + test_path_is_missing sub && test "$entry" = "$(git ls-files --stage mod/sub | cut -f 1)" && git -C mod/sub status && git update-index --refresh && @@ -447,7 +447,7 @@ test_expect_success 'mv issues a warning when section is not found in .gitmodule mkdir mod && git mv sub mod/sub 2>actual.err && test_cmp expect.err actual.err && - ! test -e sub && + test_path_is_missing sub && test "$entry" = "$(git ls-files --stage mod/sub | cut -f 1)" && git -C mod/sub status && git update-index --refresh && @@ -460,7 +460,7 @@ test_expect_success 'mv --dry-run does not touch the submodule or .gitmodules' ' git submodule update && mkdir mod && git mv -n sub mod/sub 2>actual.err && - test -f sub/.git && + test_path_is_file sub/.git && git diff-index --exit-code HEAD && git update-index --refresh && git diff-files --quiet -- sub .gitmodules @@ -474,10 +474,10 @@ test_expect_success 'checking out a commit before submodule moved needs manual u git status -s sub2 >actual && echo "?? sub2/" >expected && test_cmp expected actual && - ! test -f sub/.git && - test -f sub2/.git && + test_path_is_missing sub/.git && + test_path_is_file sub2/.git && git submodule update && - test -f sub/.git && + test_path_is_file sub/.git && rm -rf sub2 && git diff-index --exit-code HEAD && git update-index --refresh && diff --git a/t/t7700-repack.sh b/t/t7700-repack.sh index 5be483bf88..c630e0d52d 100755 --- a/t/t7700-repack.sh +++ b/t/t7700-repack.sh @@ -543,4 +543,125 @@ test_expect_success '-n overrides repack.updateServerInfo=true' ' test_server_info_missing ' +test_expect_success '--expire-to stores pruned objects (now)' ' + git init expire-to-now && + ( + cd expire-to-now && + + git branch -M main && + + test_commit base && + + git checkout -b cruft && + test_commit --no-tag cruft && + + git rev-list --objects --no-object-names main..cruft >moved.raw && + sort moved.raw >moved.want && + + git rev-list --all --objects --no-object-names >expect.raw && + sort expect.raw >expect && + + git checkout main && + git branch -D cruft && + git reflog expire --all --expire=all && + + git init --bare expired.git && + git repack -d \ + --cruft --cruft-expiration="now" \ + --expire-to="expired.git/objects/pack/pack" && + + expired="$(ls expired.git/objects/pack/pack-*.idx)" && + test_path_is_file "${expired%.idx}.mtimes" && + + # Since the `--cruft-expiration` is "now", the effective + # behavior is to move _all_ unreachable objects out to + # the location in `--expire-to`. + git show-index <$expired >expired.raw && + cut -d" " -f2 expired.raw | sort >expired.objects && + git rev-list --all --objects --no-object-names \ + >remaining.objects && + + # ...in other words, the combined contents of this + # repository and expired.git should be the same as the + # set of objects we started with. + cat expired.objects remaining.objects | sort >actual && + test_cmp expect actual && + + # The "moved" objects (i.e., those in expired.git) + # should be the same as the cruft objects which were + # expired in the previous step. + test_cmp moved.want expired.objects + ) +' + +test_expect_success '--expire-to stores pruned objects (5.minutes.ago)' ' + git init expire-to-5.minutes.ago && + ( + cd expire-to-5.minutes.ago && + + git branch -M main && + + test_commit base && + + # Create two classes of unreachable objects, one which + # is older than 5 minutes (stale), and another which is + # newer (recent). + for kind in stale recent + do + git checkout -b $kind main && + test_commit --no-tag $kind || return 1 + done && + + git rev-list --objects --no-object-names main..stale >in && + stale="$(git pack-objects $objdir/pack/pack <in)" && + mtime="$(test-tool chmtime --get =-600 $objdir/pack/pack-$stale.pack)" && + + # expect holds the set of objects we expect to find in + # this repository after repacking + git rev-list --objects --no-object-names recent >expect.raw && + sort expect.raw >expect && + + # moved.want holds the set of objects we expect to find + # in expired.git + git rev-list --objects --no-object-names main..stale >out && + sort out >moved.want && + + git checkout main && + git branch -D stale recent && + git reflog expire --all --expire=all && + git prune-packed && + + git init --bare expired.git && + git repack -d \ + --cruft --cruft-expiration=5.minutes.ago \ + --expire-to="expired.git/objects/pack/pack" && + + # Some of the remaining objects in this repository are + # unreachable, so use `cat-file --batch-all-objects` + # instead of `rev-list` to get their names + git cat-file --batch-all-objects --batch-check="%(objectname)" \ + >remaining.objects && + sort remaining.objects >actual && + test_cmp expect actual && + + ( + cd expired.git && + + expired="$(ls objects/pack/pack-*.mtimes)" && + test-tool pack-mtimes $(basename $expired) >out && + cut -d" " -f1 out | sort >../moved.got && + + # Ensure that there are as many objects with the + # expected mtime as were moved to expired.git. + # + # In other words, ensure that the recorded + # mtimes of any moved objects was written + # correctly. + grep " $mtime$" out >matching && + test_line_count = $(wc -l <../moved.want) matching + ) && + test_cmp moved.want moved.got + ) +' + test_done diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 29d914a12b..796093a7b3 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -921,10 +921,6 @@ test_path_is_missing () { then echo "Path exists:" ls -ld "$1" - if test $# -ge 1 - then - echo "$*" - fi false fi } diff --git a/tmp-objdir.h b/tmp-objdir.h index 76efc7edee..237d96b660 100644 --- a/tmp-objdir.h +++ b/tmp-objdir.h @@ -10,9 +10,11 @@ * * Example: * + * struct child_process child = CHILD_PROCESS_INIT; * struct tmp_objdir *t = tmp_objdir_create("incoming"); - * if (!run_command_v_opt_cd_env(cmd, 0, NULL, tmp_objdir_env(t)) && - * !tmp_objdir_migrate(t)) + * strvec_push(&child.args, cmd); + * strvec_pushv(&child.env, tmp_objdir_env(t)); + * if (!run_command(&child)) && !tmp_objdir_migrate(t)) * printf("success!\n"); * else * die("failed...tmp_objdir will clean up for us"); diff --git a/unpack-trees.c b/unpack-trees.c index bae812156c..8a762aa077 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -2043,7 +2043,8 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (!ret) { if (git_env_bool("GIT_TEST_CHECK_CACHE_TREE", 0)) cache_tree_verify(the_repository, &o->result); - if (!cache_tree_fully_valid(o->result.cache_tree)) + if (!o->skip_cache_tree_update && + !cache_tree_fully_valid(o->result.cache_tree)) cache_tree_update(&o->result, WRITE_TREE_SILENT | WRITE_TREE_REPAIR); diff --git a/unpack-trees.h b/unpack-trees.h index efb9edfbb2..6ab0d74c84 100644 --- a/unpack-trees.h +++ b/unpack-trees.h @@ -71,7 +71,8 @@ struct unpack_trees_options { quiet, exiting_early, show_all_errors, - dry_run; + dry_run, + skip_cache_tree_update; enum unpack_trees_reset_type reset; const char *prefix; int cache_bottom; |