GIT-FILTER-REPO(1) Git Manual GIT-FILTER-REPO(1) (NAME) git-filter-repo - (SYNOPSIS) git filter-repo --analyze git filter-repo [] [] [] [] [] [] [] [] (DESCRIPTION) . blob ( ) . "DISCUSSION" . ( ) : o ( ) o o ( ) o ( ) o ( ) o o mailmap o (grafts) (replacement refs) o ( ): o ( ) o (pruning) ( ) o (stashes) o refs/replace/ (replace refs) o o (repacking) : o replace-refs ( git-replace(1)) push fetch ( ) : o ( ). "FRESH CLONE SAFETY CHECK AND --FORCE" . o ( ) . "VERSATILITY" "DISCUSSION" "EXAMPLES" "INTERNALS" . (OPTIONS) (Analysis Options) --analyze ( ). . ( --filename-callback ) . git --path olddir/ --path newdir/ --invert-paths --path-{match,glob,regex} . --path-match , --path ( ) . --path . --path-glob glob . --path-glob . --path-regex (Regex) . --path-regex . --use-base-name (base name) . --path-rename . ( --filename-callback ) : . --path-rename , --path-rename-match . --path-rename . (Path shortcuts) --paths-from-file . ==> literal: () glob: regex: . # ( literal: # glob: regex: literal: ). --subdirectory-filter . --path / --path-rename /: --to-subdirectory-filter . --path-rename :/ ( --blob-callback ) --replace-text . (literal) regex: glob: . ==> ***REMOVED*** . --strip-blobs-bigger-than blob () ( 5M 2G ) --strip-blobs-with-ids (object id) ( --refname-callback ) --tag-rename . --tag-rename foo:bar foo-1.2.3 bar-1.2.3 . ( --message-callback ) --replace-message . --replace-text . --preserve-commit-hashes ( ) . . --preserve-commit-encoding UTF-8. UTF-8 . ( --name-callback --email-callback ) --mailmap mailmap ( git-shortlog(1) ) (author) (committer) (tagger). . --use-mailmap : --mailmap .mailmap (Parent rewriting) --replace-refs {delete-no-add, delete-and-add, update-no-add, update-or-add, update-and-add, old-default} (replace refs git-replace(1)). ( git-filter-repo) ( ) . . --replace-refs . update-and-add ( ) refs/replace/ . update-no-add . old-default git-filter-repo-2.45 update-and-add git-filter-repo update-or-add git-filter-repo . --prune-empty {always, auto, never} . auto () ( ). . --prune-degenerate {always, auto, never} . (degenerate) ( ). . (auto) ( ). --no-ff . --prune-degenerate merge --no-ff . (Generic callback code snippets) --filename-callback "CALLBACKS". --message-callback ( ) "CALLBACKS". --name-callback "CALLBACKS". --email-callback "CALLBACKS". --refname-callback (refnames) "CALLBACKS". --file-info-callback (mode) <. --file-info-callback --replace-text --file-info-callback . --blob-callback blob "CALLBACKS". --commit-callback "CALLBACKS". --tag-callback "CALLBACKS". (lightweight tags) . --refname-callback . --reset-callback reset "CALLBACKS". (Sensitive Data Removal) --sensitive-data-removal, --sdr . . : o (fetch) ( push) . --no-fetch . o () o LFS () o --no-fetch " " --sensitive-data-removal . --partial --partial . / (Location to filter from/to) --partial . --partial . --source --target (Miscellaneous options) --help, -h . --force, -f ( reflog ). "FRESH CLONE SAFETY CHECK AND --FORCE". --no-local git clone --force git-filter-repo . --partial . refs/remotes/origin/* refs/heads/* origin (unexported) reflog gc . --tag-rename --refname-callback . . --refs . --partial . --partial ( refs/remotes/origin/* refs/heads/* ) (negative revisions) . --dry-run . git fast-export . fast-import . --debug . ( --dry-run ). --stdin git fast-export fast-export (stdin) . ( original-oid ). --quiet --quiet . (OUTPUT) filter-repo .git/filter-repo/ . (overwrite) . (Commit map) $GIT_DIR/filter-repo/commit-map ( ) . o (header) "old" "new" . o . o ( ). o SHA (null SHA) . "new" . (Reference map) $GIT_DIR/filter-repo/ref-map ( ) . o "old" "new" "ref" . o ref . o SHA . "new" . (Changed References) $GIT_DIR/filter-repo/changed-refs ref . o . o ref ref-map old != new . o ref-map ref . (First Changed Commits) $GIT_DIR/filter-repo/first-changed-commits () . . git-filter-repo A1-B1-C1-D1-E1 A1-B2-C2-D2-E2 First Changed Commits B2 . . (Already Ran) $GIT_DIR/filter-repo/already_ran git-filter-repo . . : * " " (Fresh Clone) git-filter-repo . " " . o commit-map ref-map . filter-repo A B filter-repo B C "B C" "A C" . o () ( --sensitive-data-removal ) . : A1-B1-C1-D1-E1 filter-repo A1-B1-C2-D2-E2 "C1" . filter-repo A1-B1-C2-D3-E3 "C1" "D2" . already_ran git-filter-repo . . $GIT_DIR/filter-repo . filter-repo . LFS (Original LFS Objects) --sensitive-data-removal LFS $GIT_DIR/filter-repo/original_lfs_objects LFS . LFS (Orphaned LFS Objects) --sensitive-data-removal LFS $GIT_DIR/filter-repo/orphaned_lfs_objects LFS git-filter-repo . . --FORCE (FRESH CLONE SAFETY CHECK AND --FORCE) filter-repo . filter repo . . : " " git . ( --no-local ) . . -- filter-repo . filter-repo ( ). filter-repo filter-repo . filter-repo . `--force` filter-repo . --force . --force --force . . (VERSATILITY) filter-repo . : o ( --path --strip-blobs-bigger-than --replace-text --mailmap) o ( --subdirectory-filter (--path) (--path-rename) --paths-from-file --path* ) o ( filename message name email refname) o git ( commit tag blob reset) o filter-repo . . "CALLBACKS" . filter-repo contrib/filter-repo-demos/ . git-filter-branch git-filter-branch BFG Repo Cleaner . (DISCUSSION) filter-repo . (rebase) . "RECOVERING FROM UPSTREAM REBASE" git-rebase(1) ( "The hard case") . git-filter-repo : 1. . --bare --mirror git clone . --no-local . . 2. () git filter-repo --analyze . : () () () /// (blob-id). . --analyze . 3. git-filter-repo . / . --sensitive-data-removal " " . : o . git pull && git push -- -- . . URL . o --tags git fetch git pull ( "On Re-tagging" git-tag(1) ). URL . o ref ( ) git push --mirror --prune ref . . ref fetch --prune . URL . o ref force push . ( refs/changes/ refs/pull/ refs/merge-requests/) ( ) . ( ). 4. filter-repo . "EXAMPLES" . ( filter-repo) . . 5. ( refs/remotes/origin/* filter-repo refs/heads/* ). 6. () o filter-repo --replace-refs (replace refs) ( git-replace(1) ) ( ) git . replace ref URL ( refspec git config --add remote.origin.fetch +refs/replace/*:refs/replace/*). replace ref jgit libgit2 ( Gerrit GitHub GitLab) replace ref . . replace ref git . (commit-graphs) replace ref replace ref commit-graphs . origin (Why is my origin removed?) ( ) . . " " ( ) . . git filter-branch BFG repo cleaner git filter-repo . : o ( ): git pull && git push o : " Pull-Request/Merge-Request/Code-Review merge/submit" origin ( ) . : o git remote add origin $ORIGINAL_CLONE_URL ( git push --force --branches --tags --prune). origin origin . : . . (Sensitive Data Removals) . // . / // (token/password/credential/secret) ( ) (revoke) (rotate) . . / . : o git-filter-repo o : o o o . . git-filter-repo git-filter-repo . : o --sensitive-data-removal . . o --invert-paths --path . o () --replace-text . . : git log --all --name-status -- ${PROBLEMATIC_FILE1} ${PROBLEMATIC_FILE2} git log -S"${PROBLEMATIC_STRING}" --all -p -- git-filter-repo . : force push . ref . ( ): git push --force --mirror origin : o force push . o force push ref (Gerrit GitHub GitLab ) ref ( refs/changes/* refs/pull/* refs/merge-requests/*). ref (garbage collection) . : ( GitLab [1] "Fully removing the data from GitHub" GitHub[2]). o --no-fetch git-filter-repo ( ) () --mirror ref refspec () --mirror ref (fetch ) . ref ( ref ) ref . o --mirror --force . --force . LFS (orphaned) LFS / . LFS / . : . git pull && git push . . git-filter-repo . . git-filter-repo . : o git fetch --prune --tags . fetch . o ( ref ) . "RECOVERING FROM UPSTREAM REBASE" git-rebase(1) ( "The hard case") . o ( reflog stash . ): o git reflog expire --expire=now --all o git gc --prune=now ( ). : git cat-file -t ${HASH_OF_FIRST_CHANGED_COMMIT} ${HASH_OF_FIRST_CHANGED_COMMIT} git-filter-repo ( " " ). (fatal error) . "commit" /ref reflog gc . ref ${HASH_OF_FIRST_CHANGED_COMMIT} reflog gc /ref : git for-each-ref --contains ${HASH_OF_FIRST_CHANGED_COMMIT} cat-file ${HASH_OF_FIRST_CHANGED_COMMIT} . LFS ( git-filter-repo --sensitive-data-removal ) LFS . .git/lfs/objects/ . . . : o ( git pull && git push ) . o () (re-)push . ( ban-commit Gerrit https://gerrit-review.googlesource.com/Documentation/cmd-ban-commit.html ) pre-receive . ( pre-receive: git cat-file -t ${BAD_COMMIT} ${BAD_COMMIT} ) : o git .gitignore . o . Azure Key Vault AWS Secrets Manager HashiCorp Vault . o pre-commit pre-commit git-secrets gitleaks . (EXAMPLES) (Path based filtering) README.md guides tools/releases/: git filter-repo --path README.md --path guides/ --path tools/releases (toplevel) . --invert-paths : git filter-repo --path README.md --path guides/ --path tools/releases --invert-paths (inclusion) (exclusion) filter-repo . src/main src/main data : git filter-repo --path src/main/ git filter-repo --path-glob 'src/*/data' --invert-paths (*) src/main/org/whatever/data . src/not-main/foo/data src/not-main/ . (glob expansion) . (regular expression) ( https://docs.python.org/3/library/re.html#regular-expression-syntax). YYYY-MM-DD.txt : git filter-repo --path-regex '^.*/.*/[0-9]{4}-[0-9]{2}-[0-9]{2}.txt$' ( ) --path-rename cmds/ src/scripts/ tools/: git filter-repo --path-rename cmds:tools --path-rename src/scripts/:tools/ --path --path-rename . --path-rename (overwrite) . ( src/scripts/run_release.sh cmds/run_release.sh ) . : git filter-repo --path-rename cmds/run_release.sh:tools/do_release.sh \ --path-rename cmds/:tools/ \ --path-rename src/scripts/:tools/ --path-rename . git filter-repo --path-rename sources/:src/main/ --path src/main/ ( src/main/ ). cmds/run_release.sh . git filter-repo --path src/main/ --path-rename tools/:scripts/ tools scripts src/main/ . : git filter-repo --path src/main/ --path tools/ --path-rename tools/:scripts/ (basename) --use-base-name ( --path-rename ). README.md Makefile : git filter-repo --use-base-name --path README.md --path Makefile .DS_Store : git filter-repo --invert-paths --path '.DS_Store' --use-base-name : git filter-repo --invert-paths --path-glob '*/.DS_Store' --path '.DS_Store' ( --path-glob .DS_Store --path-glob '*.DS_Store' foo.DS_Store bar/baz.DS_Store ) --filename-callback "CALLBACKS" . (Filtering based on many paths) (globs) --paths-from-file stuff-i-want.txt : # Blank lines and comment lines are ignored. # Examples similar to --path: README.md guides/ tools/releases # An example that is like --path-glob: glob:*.py # An example that is like --path-regex: regex:^.*/.*/[0-9]{4}-[0-9]{2}-[0-9]{2}.txt$ # An example of renaming a path tools/==>scripts/ # An example of using a regex to rename a path regex:(.*)/([^/]*)/([^/]*)\.text$==>\2/\1/\3.txt git filter-repo --paths-from-file stuff-i-want.txt README.md guides/ tools/releases/ YYYY-MM-DD.txt tools/ scripts/ foo/bar/baz.text bar/foo/baz.txt . glob: regex: ==> . . (newline) ( core.quotePath git config --help ) git ls-files ( ) : git ls-files >../paths-i-want.txt git filter-repo --paths-from-file ../paths-i-want.txt --paths-from-file . git filter-repo --analyze .git/filter-repo/analysis/path-deleted-sizes.txt /tmp/files-i-dont-want-anymore.txt git filter-repo --invert-paths --paths-from-file /tmp/files-i-dont-want-anymore.txt . (Directory based shortcuts) : module/ foo.c bar.c otherDir/ blah.config stuff.txt zebra.jpg module/ : foo.c bar.c : git filter-repo --subdirectory-filter module/ my-module/ : my-module/ module/ foo.c bar.c otherDir/ blah.config stuff.txt zebra.jpg : git filter-repo --to-subdirectory-filter my-module/ (Content based filtering) () --strip-blobs-bigger-than ( K M G ) : git filter-repo --strip-blobs-bigger-than 10M () : git filter-repo --strip-blobs-with-ids FILE_WITH_GIT_BLOB_IDS . expressions.txt : p455w0rd foo==>bar glob:*666*==> regex:\bdriver\b==>pilot literal:MM/DD/YYYY==>YYYY-MM-DD regex:([0-9]{2})/([0-9]{2})/([0-9]{4})==>\3-\1-\2 git filter-repo --replace-text expressions.txt p455w0rd ***REMOVED*** foo bar 666 driver pilot ( drivers ) MM/DD/YYYY YYYY-MM-DD MM/DD/YYYY YYYY-MM-DD . : o ==> . ==> ***REMOVED*** . o literal: glob: regex: (literal) (glob - https://docs.python.org/3/library/fnmatch.html) ( https://docs.python.org/3/library/re.html#regular-expression-syntax) . literal: . o . o glob regex . (?m) regex MULTILINE ^ $ . https://docs.python.org/3/library/re.html . --blob-callback "CALLBACKS" . / (Updating commit/tag messages) --replace-text . expressions.txt : foo==>bar git filter-repo --replace-message expressions.txt foo bar . --message-callback "CALLBACKS" . (Refname based filtering) --tag-rename : git filter-repo --tag-rename foo:bar foo bar . : git filter-repo --tag-rename '':'my-module-' --refname-callback "CALLBACKS" . (User and email based filtering) mailmap git-shortlog(1) . my-mailmap : git filter-repo --mailmap my-mailmap ( mailmap ): Name For User New Name And New Name And Old Name And / . --name-callback --email-callback "CALLBACKS" . (Parent rewriting) $commit_A $commit_B ( $commit_A $commit_B ) : git replace $commit_A $commit_B git filter-repo --proceed $commit_A () $commit_A : git replace --graft $commit_A $new_parent_or_parents git filter-repo --proceed --proceed << >> (no arguments specified) . git-filter-repo (graft) --force not-a-fresh-clone --force . (Partial history rewrites) ( ) --refs . extraneous.txt master: git filter-repo --invert-paths --path extraneous.txt --refs master : git filter-repo --invert-paths --path extraneous.txt --refs master~3..master (CALLBACKS) filter-repo . API "API BACKWARD COMPATIBILITY CAVEAT" git-filter-repo . (--file-info-callback ). --foo-callback 'BODY' : def foo_callback(foo): BODY BODY foo . filter-repo (bytestrings - https://docs.python.org/3/library/stdtypes.html#bytes ) . git-fast-import(1) : --blob-callback --commit-callback --tag-callback --reset-callback . ( (refnames) ). : --filename-callback --message-callback --name-callback --email-callback --refname-callback --file-info-callback . git-filter-repo --name-callback 'return name.replace(b"Wiliam", b"William")' : def name_callback(name): return name.replace(b"Wiliam", b"William") : git-filter-repo --email-callback 'return email.replace(b".cm", b".com")' (refname) (fully qualified) ( b"refs/heads/master" b"master" b"refs/tags/v1.0.7" b"1.0.7"): git-filter-repo --refname-callback ' # Change e.g. refs/heads/master to refs/heads/prefix-master rdir,rpath = os.path.split(refname) return rdir + b"/prefix-" + rpath' : git-filter-repo --message-callback ' if b"Signed-off-by:" not in message: message += b"\nSigned-off-by: Me My " return re.sub(b"[Ee]-?[Mm][Aa][Ii][Ll]", b"email", message)' (filename) . None . : git-filter-repo --filename-callback ' if b"/src/" in filename: # Remove all files with a directory named "src" in their path # (except when "src" appears at the toplevel). return None elif filename.startswith(b"tools/"): # Rename tools/ -> scripts/misc/ return b"scripts/misc/" + filename[6:] else: # Keep the filename and do not rename it return filename ' (file-info callback) . ( mode) . ( ). (filename mode blob_id value) (filename mode blob_id). ( None ). (mode) (b"100644" b"100755" / b"120000" b"160000" ). blob_id value . value : value.get_contents_by_identifier(blob_id) -> contents (bytestring) value.get_size_by_identifier(blob_id) -> size_of_blob (int) value.insert_file_with_contents(contents) -> blob_id value.is_binary(contents) -> bool value.apply_replace_text(contents) -> new_contents (bytestring) : value.data (dict). blob_id ( grep(1)) --replace-text ( --file-info-callback --replace-text ). --replace-text : git-filter-repo --file-info-callback ' if not filename.endswith(b".config"): # Make no changes to the file; return as-is return (filename, mode, blob_id) new_filename = filename[0:-7] + b".cfg" contents = value.get_contents_by_identifier(blob_id) new_contents = value.apply_replace_text(contents) new_blob_id = value.insert_file_with_contents(new_contents) return (new_filename, mode, new_blob_id) ( cherry-picked revert ) --file-info-callback . value.data . blob_id : git-filter-repo --file-info-callback ' if not filename.endswith(b".config"): # Make no changes to the file; return as-is return (filename, mode, blob_id) new_filename = filename[0:-7] + b".cfg" if blob_id in value.data: return (new_filename, mode, value.data[blob_id]) contents = value.get_contents_by_identifier(blob_id) new_contents = value.apply_replace_text(contents) new_blob_id = value.insert_file_with_contents(new_contents) value.data[blob_id] = new_blob_id return (new_filename, mode, new_blob_id) --file-info-callback .sh .sh : git-filter-repo --file-info-callback ' if not filename.endswith(b".sh"): # Make no changes to the file; return as-is return (filename, mode, blob_id) # There are only 4 valid modes in git: # - 100644, for regular non-executable files # - 100755, for executable files/scripts # - 120000, for symlinks # - 160000, for submodules new_mode = b"100755" contents = value.get_contents_by_identifier(blob_id) new_contents = contents + b"\n" new_blob_id = value.insert_file_with_contents(new_contents) return (filename, new_mode, new_blob_id) blob reset tag commit . ( API ): o Blob: original_id ( ) data o Reset: ref ( ) from_ref ( ) o Tag: ref from_ref original_id tagger_name tagger_email tagger_date message o Commit: branch original_id author_name author_email author_date committer_name committer_email committer_date message file_changes ( FileChange type filename mode blob_id ) parents ( ) : git filter-repo --blob-callback ' if len(blob.data) > 25: # Mark this blob for removal from all commits blob.skip() else: blob.data = blob.data.replace(b"Hello", b"Goodbye") ' git filter-repo --reset-callback 'reset.ref = reset.ref.replace(b"master", b"dev")' git filter-repo --tag-callback ' if tag.tagger_name == b"Jim Williams": # Omit this tag tag.skip() else: tag.message = tag.message + b"\n\nTag of %s by %s on %s" % (tag.ref, tag.tagger_email, tag.tagger_date)' git filter-repo --commit-callback ' # Remove executable files with three 6s in their name (including # from leading directories). # Also, undo deletion of sources/foo/bar.txt (change types are # either b"D" (deletion) or b"M" (add or modify); renames are # handled by deleting the old file and adding a new one) commit.file_changes = [ change for change in commit.file_changes if not (change.mode == b"100755" and change.filename.count(b"6") == 3) and not (change.type == b"D" and change.filename == b"sources/foo/bar.txt")] # Mark all .sh files as executable; modes in git are always one of # 100644 (normal file), 100755 (executable), 120000 (symlink), or # 160000 (submodule) for change in commit.file_changes: if change.filename.endswith(b".sh"): change.mode = b"100755" ' (INTERNALS) . filter-repo filter-repo git fast-export | filter | git fast-import filter-repo filter . filter-repo . filter-repo : 1. (fresh clone) 2. git fetch -u . refs/remotes/origin/*:refs/heads/* 3. git remote rm origin 4. git fast-export --show-original-ids --reference-excluded-parents --fake-missing-tagger --signed-tags=strip --tag-of-filtered-object=rewrite --use-done-feature --no-data --reencode=yes --mark-tags --all | filter | git -c core.ignorecase=false fast-import --date-format=raw-permissive --force --quiet 5. git update-ref --no-deref --stdin . 6. git reset --hard 7. git reflog expire --expire=now --all 8. git gc --prune=now : 1. . ( --force filter-repo .) 2. git update-ref --force . fetch . 3. push " (DISCUSSION)" . . --mirror push . 4. . --replace-text --blob-callback filter-repo blob --no-data fast-export . blob --no-data . ( --dry-run --debug) 5. (replace refs) . . 6. (working tree) (index) . bare . 7. (Reflogs) . 8. gc . . ( blob --aggressive .) --debug filter-repo . --partial --replace-refs update-no-add nuke-unused-refs --replace-refs . git filter-repo fast-export fast-import : o (extended commit headers) o ( ) o . git-2.24.0 blob (fast-export ). (trees) ( fast-export fast-import ). o refs/tags/ ( ) o fast-import o fast-export UTF-8 ( ). o (author) (committer) o o (commit cycles) ( git-replace(1) ) fast-export : o (stream) fast-export ( ) . ( .) o fast-export fast-import (patches). ( " ") . git- rebase(1) . : o ( .git/filter-repo/suboptimal-issues ): o o (prune) o o . . . ( .git/filter-repo/suboptimal-issues ) filter-repo o (+051800) . . fast-import (+0261). o --path-rename --path-rename (clobber) . --path-rename . ( --path-rename ) o fast-export ( fast-import) . --full-tree filter-repo ( ) -M -C filter-repo . o (Partial-repo filtering) << >> filter-repo . ( --reference-excluded-parents fast-export filter-repo ). --partial --refs . . <<>>. ( ***REMOVED***) . filter-repo : o fast-export fast-import o (grafts) (replace refs) o . o filter-repo ( UTF-8) . o filter-repo . API filter-repo ( ). . (SEE ALSO) git-rebase(1) git-filter-branch(1) (GIT) git(1) (NOTES) 1. (GitLab) https://docs.gitlab.com/ee/user/project/repository/reducing_the_repo_size_using_git.html 2. "Fully removing the data from GitHub" (GitHub) https://docs.github.com/en/authentication/keeping-your-account-and- data-secure/removing-sensitive-data-from-a-repository#fully- removing-the-data-from-github Git 2.47.1.2.g787e9d71 2024-11-27 GIT-FILTER-REPO(1)