Skip to content

GH-89727: Fix shutil.rmtree() recursion error on deep trees #119808

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
barneygale merged 13 commits into from
Jun 1, 2024
+68 −97
Merged

GH-89727: Fix shutil.rmtree() recursion error on deep trees #119808

Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
417b829
Tidy up `_rmtree_safe_fd()` implementation.
barneygale May 30, 2024
8d42ea8
Rewrite using list as stack.
barneygale May 30, 2024
7018370
Add NEWS
barneygale May 30, 2024
9f13194
Enable test
barneygale May 30, 2024
aee87db
Use DirEntry.stat() where possible.
barneygale May 30, 2024
d9d0386
Comments, naming, line limit.
barneygale May 30, 2024
4daf2f2
Restore comment.
barneygale May 30, 2024
a335a63
Simplify overall diff
barneygale May 30, 2024
1d4772a
Address some review feedback
barneygale May 31, 2024
462cfc1
Handle further errors in finally: block
barneygale May 31, 2024
8d7ba83
Comments.
barneygale May 31, 2024
11deeab
Handle os.close explicitly
barneygale May 31, 2024
a53e473
Skip os.unlink() if os.DirEntry.is_dir() raises FileNotFoundError
barneygale May 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use DirEntry.stat() where possible.
  • Loading branch information
@barneygale
barneygale committed May 30, 2024
commit aee87db34504dd05682fcbe3ad5d0661337f27ab
18 changes: 11 additions & 7 deletions Lib/shutil.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -636,24 +636,28 @@ def onerror(err):

# Version using fd-based APIs to protect against races
def _rmtree_safe_fd(stack, onexc):
func, dir_fd, name, path = stack.pop()
func, dir_fd, name, path, entry = stack.pop()
try:
if func is os.close:
os.close(dir_fd)
elif func is os.rmdir:
os.rmdir(name, dir_fd=dir_fd)
else:
assert func is os.lstat
orig_st = os.lstat(name, dir_fd=dir_fd)
if entry is None:
orig_st = os.lstat(name, dir_fd=dir_fd)
else:
orig_st = entry.stat(follow_symlinks=False)

func = os.open # For error reporting.
fd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd)
try:
func = os.path.islink # For error reporting.
if not os.path.samestat(orig_st, os.fstat(fd)):
raise OSError("Cannot call rmtree on a symbolic link")
stack.append((os.rmdir, dir_fd, name, path))
stack.append((os.rmdir, dir_fd, name, path, None))
finally:
stack.append((os.close, fd, name, path))
stack.append((os.close, fd, name, path, None))

func = os.scandir # For error reporting.
with os.scandir(fd) as scandir_it:
Expand All @@ -665,7 +669,7 @@ def _rmtree_safe_fd(stack, onexc):
except OSError:
is_dir = False
if is_dir:
stack.append((os.lstat, fd, entry.name, fullname))
stack.append((os.lstat, fd, entry.name, fullname, entry))
else:
try:
os.unlink(entry.name, dir_fd=fd)
Expand Down Expand Up @@ -733,13 +737,13 @@ def onexc(*args):
path = os.fsdecode(path)
# Note: To guard against symlink races, we use the standard
# lstat()/open()/fstat() trick.
Comment thread
barneygale marked this conversation as resolved.
Outdated
stack = [(os.lstat, dir_fd, path, path)]
stack = [(os.lstat, dir_fd, path, path, None)]
try:
while stack:
_rmtree_safe_fd(stack, onexc)
finally:
while stack:
func, dir_fd, name, path = stack.pop()
func, dir_fd, name, path, entry = stack.pop()
if func is os.close:
os.close(dir_fd)
else:
Expand Down