337 lines
12 KiB
Python
337 lines
12 KiB
Python
# Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
import os
|
|
import unreal
|
|
import flow.cmd
|
|
from peafour import P4
|
|
import subprocess as sp
|
|
from pathlib import Path
|
|
|
|
#-------------------------------------------------------------------------------
|
|
def _running_exes_powershell():
|
|
ps_cmd = (
|
|
"powershell.exe",
|
|
"-NoProfile",
|
|
"-NoLogo",
|
|
"-Command",
|
|
r'(Get-CimInstance -ClassName Win32_Process).ExecutablePath',
|
|
)
|
|
startupinfo = sp.STARTUPINFO()
|
|
startupinfo.dwFlags = sp.STARTF_USESHOWWINDOW
|
|
startupinfo.wShowWindow = sp.SW_HIDE
|
|
proc = sp.Popen(
|
|
ps_cmd,
|
|
stdout=sp.PIPE, stderr=sp.DEVNULL, stdin=sp.DEVNULL,
|
|
startupinfo=startupinfo, creationflags=sp.CREATE_NEW_CONSOLE
|
|
)
|
|
with proc:
|
|
yield from (x.strip().decode() for x in proc.stdout)
|
|
proc.stdout.close()
|
|
|
|
#-------------------------------------------------------------------------------
|
|
def _running_exes_wmic():
|
|
proc = sp.Popen(
|
|
("wmic.exe", "process", "get", "executablepath"),
|
|
stdout=sp.PIPE, stderr=sp.DEVNULL, stdin=sp.DEVNULL
|
|
)
|
|
with proc:
|
|
for line in proc.stdout:
|
|
line = line.strip().decode()
|
|
if line:
|
|
yield line
|
|
proc.stdout.close()
|
|
|
|
#-------------------------------------------------------------------------------
|
|
def _read_running_exes():
|
|
try:
|
|
gen = _running_exes_powershell()
|
|
except:
|
|
gen = _running_exes_wmic()
|
|
yield from gen
|
|
|
|
#-------------------------------------------------------------------------------
|
|
def _detect_locked_files(root_dir):
|
|
for line in _read_running_exes():
|
|
if line and Path(line).parent.is_relative_to(root_dir):
|
|
return Path(line)
|
|
|
|
|
|
|
|
#-------------------------------------------------------------------------------
|
|
class _Collector(object):
|
|
def __init__(self):
|
|
self._dirs = []
|
|
self._careful_dirs = []
|
|
self._pinned_files = set()
|
|
|
|
def get_pinned_count(self):
|
|
return len(self._pinned_files)
|
|
|
|
def add_dir(self, path, *, remove_carefully=False):
|
|
(self._careful_dirs if remove_carefully else self._dirs).append(path)
|
|
|
|
def pin_file(self, path):
|
|
path = str(path.resolve())
|
|
self._pinned_files.add(path.lower())
|
|
|
|
def read_careful_dirs(self):
|
|
return (x for x in self._careful_dirs)
|
|
|
|
def dispatch(self, actuator):
|
|
actuator.begin()
|
|
|
|
for dir in self._dirs:
|
|
actuator.remove_dir(dir)
|
|
|
|
for dir in self._careful_dirs:
|
|
for item in (x for x in dir.rglob("*") if x.is_file()):
|
|
key = str(item.resolve()).lower()
|
|
if key not in self._pinned_files:
|
|
actuator.remove_file(item)
|
|
|
|
actuator.end()
|
|
|
|
#-------------------------------------------------------------------------------
|
|
class _DryRun(object):
|
|
def _spam(self):
|
|
mb_size = format(self._bytes // 1024 // 1024, ",") + "MB"
|
|
print("%d files to remove (%s)" % (self._count, mb_size), end="\r")
|
|
|
|
def begin(self):
|
|
self._bytes = 0
|
|
self._count = 0
|
|
self._breakdown = {}
|
|
|
|
def remove_dir(self, path):
|
|
for item in (x for x in path.rglob("*") if x.is_file()):
|
|
self.remove_file(item)
|
|
|
|
def remove_file(self, path):
|
|
self._bytes += path.stat().st_size
|
|
self._count += 1
|
|
if self._count % 493:
|
|
self._spam()
|
|
|
|
def end(self):
|
|
self._spam()
|
|
print()
|
|
|
|
#-------------------------------------------------------------------------------
|
|
class _Cleaner(object):
|
|
def __init__(self, work_dir):
|
|
self._work_dir = work_dir
|
|
|
|
def __del__(self):
|
|
if self._dir_count:
|
|
print(f"Launching background rmdir for {self._rubbish_dir}")
|
|
else:
|
|
print("Skipping background rmdir. No major directories to remove")
|
|
|
|
if os.name == "nt":
|
|
sp.Popen(
|
|
("cmd.exe", "/d/c", "rd", "/q/s", str(self._rubbish_dir)),
|
|
stdout=sp.DEVNULL,
|
|
stderr=sp.DEVNULL)
|
|
else:
|
|
if self._rubbish_dir.is_dir() and self._rubbish_dir != "/": # paranoia!
|
|
sp.Popen(
|
|
("rm", "-rf", str(self._rubbish_dir)),
|
|
stdout=sp.DEVNULL,
|
|
stderr=sp.DEVNULL)
|
|
|
|
def _spam(self):
|
|
print(f"Moved {self._dir_count} directories, {self._file_count} files removed", end="\r", flush=True)
|
|
|
|
def begin(self):
|
|
rubbish_dir = self._work_dir / ".ushell_clean"
|
|
rubbish_dir.mkdir(parents=True, exist_ok=True)
|
|
self._rubbish_dir = rubbish_dir
|
|
|
|
self._dir_count = 0
|
|
self._file_count = 0
|
|
print(f"Moving directories to {self._rubbish_dir.name} and removing unversioned files")
|
|
|
|
def remove_dir(self, path):
|
|
path = path.absolute()
|
|
dest_name = "%08x_%016x_%s_%s" % (os.getpid(), hash(path), path.parent.name, path.name)
|
|
try:
|
|
path.rename(str(self._rubbish_dir / dest_name))
|
|
self._dir_count += 1
|
|
self._spam()
|
|
except OSError as e:
|
|
print("WARNING:", e)
|
|
|
|
def remove_file(self, item):
|
|
item.chmod(0o666)
|
|
item.unlink()
|
|
self._file_count += 1
|
|
self._spam()
|
|
|
|
def end(self):
|
|
if self._file_count or self._dir_count:
|
|
print()
|
|
|
|
|
|
|
|
#-------------------------------------------------------------------------------
|
|
class Clean(flow.cmd.Cmd):
|
|
""" Cleans intermediate and temporary files from an Unreal Engine branch. The
|
|
following sub-directories of .uproject, .uplugin, and Engine/ are cleaned up;
|
|
|
|
Intermediate - Removed entirely
|
|
DerivedDataCache - Removed entirely
|
|
Binaries - Unversioned files are removed
|
|
Saved - All sub-directories except --savedkeeps=... (see below)
|
|
|
|
Only a subset of sub-directories of Saved/ are removed. Any directories that
|
|
match --savedkeeps's comma-separated list are not removed. For example, to
|
|
clean everything excepted Saved/StagedBuilds/ and Saved/Profiling/;
|
|
|
|
.p4 clean --savedkeeps=StagedBuilds,Profiling
|
|
|
|
"StagedBuilds,Profiling" is the default value for --savedkeeps. If --allsaved
|
|
is given then all of Saved/ will be removed.
|
|
|
|
Note that the removal happens in two stages. First directories are moved into
|
|
.ushell_clean/ in the root of the branch. This directory is then removed in
|
|
the background after `.p4 clean` exits."""
|
|
dryrun = flow.cmd.Opt(False, "Do nothing except reports statistics")
|
|
allsaved = flow.cmd.Opt(False, "Completely clean Saved/ directories")
|
|
savedkeeps = flow.cmd.Opt("Profiling,StagedBuilds", "Comma-separated list of Saved/ sub-directories to keep")
|
|
|
|
def _append_saved(self, collector, saved_dir):
|
|
if self.args.allsaved:
|
|
for item in saved_dir.glob("*"):
|
|
if item.is_dir():
|
|
collector.add_dir(item)
|
|
return
|
|
|
|
for sub_dir in (x for x in saved_dir.glob("*") if x.is_dir()):
|
|
if sub_dir.name.lower() not in self._saved_keeps:
|
|
collector.add_dir(sub_dir)
|
|
|
|
def main(self):
|
|
self._saved_keeps = {x.strip().lower() for x in self.args.savedkeeps.split(",")}
|
|
|
|
ue_context = unreal.Context(os.getcwd())
|
|
branch = ue_context.get_branch(must_exist=True)
|
|
engine_dir = ue_context.get_engine().get_dir()
|
|
|
|
if os.name == "nt":
|
|
self.print_info("Checking running processes")
|
|
for _ in range(2):
|
|
running_exe = _detect_locked_files(branch.get_dir())
|
|
if not running_exe:
|
|
break
|
|
|
|
if running_exe.name == "dotnet.exe":
|
|
print("'dotnet.exe' is playing gooseberry")
|
|
print("...lets terminate them")
|
|
sp.run(("taskkill", "/f", "/im", str(running_exe.name)))
|
|
continue
|
|
|
|
raise RuntimeError(f"Not cleaning because '{running_exe}' is running")
|
|
|
|
self.print_info("Finding directories and files to clean")
|
|
root_dirs = [
|
|
engine_dir,
|
|
*(x for x in engine_dir.glob("Platforms/*") if x.is_dir()),
|
|
*(x for x in engine_dir.glob("Programs/*") if x.is_dir()),
|
|
*(x for x in engine_dir.glob("Restricted/*") if x.is_dir()),
|
|
]
|
|
|
|
print("Enumerating...", end="")
|
|
rg_args = (
|
|
"rg",
|
|
"--files",
|
|
"--path-separator=/",
|
|
"--no-ignore",
|
|
"-g*.uplugin",
|
|
"-g*.uproject",
|
|
str(branch.get_dir()),
|
|
)
|
|
rg = sp.Popen(rg_args, stdout=sp.PIPE, stderr=sp.DEVNULL)
|
|
for line in rg.stdout.readlines():
|
|
path = Path(line.decode().rstrip()).parent
|
|
root_dirs.append(path)
|
|
rg.wait()
|
|
print("\r", len(root_dirs), " uproject/uplugin roots found", sep="")
|
|
|
|
collector = _Collector()
|
|
clean_handlers = {
|
|
"Intermediate" : lambda x,y: x.add_dir(y),
|
|
"DerivedDataCache" : lambda x,y: x.add_dir(y),
|
|
"Binaries" : lambda x,y: x.add_dir(y, remove_carefully=True),
|
|
"Saved" : self._append_saved,
|
|
}
|
|
for root_dir in root_dirs:
|
|
for dir_name, clean_handler in clean_handlers.items():
|
|
candidate = root_dir / dir_name
|
|
if candidate.is_dir():
|
|
clean_handler(collector, candidate)
|
|
|
|
# Ask Perforce which files shouldn't be deleted
|
|
print("Asking Perforce what's synced...", end="", flush=True)
|
|
specs = (str(x) + "/..." for x in collector.read_careful_dirs())
|
|
p4_have = P4.have(specs)
|
|
for item in p4_have.read(on_error=False):
|
|
path = Path(item.path)
|
|
collector.pin_file(path)
|
|
print("done (", collector.get_pinned_count(), " files)", sep="")
|
|
|
|
if self.args.dryrun:
|
|
collector.dispatch(_DryRun())
|
|
return
|
|
|
|
self.print_info("Cleaning")
|
|
actuator = _Cleaner(branch.get_dir())
|
|
collector.dispatch(actuator)
|
|
|
|
|
|
|
|
#-------------------------------------------------------------------------------
|
|
class Reset(flow.cmd.Cmd):
|
|
""" Reconciles a branch to make it match the depot. Use with caution; this is
|
|
a destructive action and involes removing and rewriting files! """
|
|
thorough = flow.cmd.Opt(False, "Compare digests instead of file-modified time")
|
|
|
|
def _get_reset_paths(self):
|
|
try:
|
|
ue_context = unreal.Context(os.getcwd())
|
|
if not (branch := ue_context.get_branch()):
|
|
return None
|
|
except EnvironmentError:
|
|
return None
|
|
|
|
root_dir = branch.get_dir()
|
|
ret = {
|
|
# root_dir / "*", # to much scope to go wrong here and it's so few files
|
|
root_dir / "Template/...",
|
|
root_dir / "Engine/...",
|
|
}
|
|
for uproj_path in branch.read_projects():
|
|
ret.add(uproj_path.parent.absolute() / "...")
|
|
|
|
return ret
|
|
|
|
def main(self):
|
|
# Confirm the user wants to really do a reset
|
|
self.print_warning("Destructive action!")
|
|
if self.is_interactive():
|
|
while True:
|
|
c = input("Are you sure you want to continue [yn] ?")
|
|
if c.lower() == "y": break
|
|
if c.lower() == "n": return False
|
|
|
|
# Run the reconcile
|
|
args = ("reconcile", "-wade",)
|
|
if not self.args.thorough:
|
|
args = (*args, "--modtime")
|
|
|
|
if reset_paths := self._get_reset_paths():
|
|
args = (*args, *(str(x) for x in reset_paths))
|
|
|
|
exec_context = self.get_exec_context()
|
|
cmd = exec_context.create_runnable("p4", *args)
|
|
return cmd.run()
|