diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/servo/bootstrap_commands.py | 175 | ||||
| -rw-r--r-- | python/servo/build_commands.py | 20 |
2 files changed, 175 insertions, 20 deletions
diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index a3016014792..da182198ac4 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -18,6 +18,7 @@ import shutil import subprocess import sys import urllib2 +import glob from mach.decorators import ( CommandArgument, @@ -26,7 +27,7 @@ from mach.decorators import ( ) import servo.bootstrap as bootstrap -from servo.command_base import CommandBase, BIN_SUFFIX +from servo.command_base import CommandBase, BIN_SUFFIX, cd from servo.util import delete, download_bytes, download_file, extract, host_triple @@ -346,3 +347,175 @@ class MachCommands(CommandBase): elif not force: print("Nothing done. " "Run `./mach clean-nightlies -f` to actually remove.") + + @Command('clean-cargo-cache', + description='Clean unused Cargo packages', + category='bootstrap') + @CommandArgument('--force', '-f', + action='store_true', + help='Actually remove stuff') + @CommandArgument('--show-size', '-s', + action='store_true', + help='Show packages size') + @CommandArgument('--keep', + default='1', + help='Keep up to this many most recent dependencies') + @CommandArgument('--custom-path', '-c', + action='store_true', + help='Get Cargo path from CARGO_HOME environment variable') + def clean_cargo_cache(self, force=False, show_size=False, keep=None, custom_path=False): + def get_size(path): + if os.path.isfile(path): + return os.path.getsize(path) / (1024 * 1024.0) + total_size = 0 + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size / (1024 * 1024.0) + + removing_anything = False + packages = { + 'crates': {}, + 'git': {}, + } + import toml + if os.environ.get("CARGO_HOME", "") and custom_path: + cargo_dir = os.environ.get("CARGO_HOME") + else: + cargo_dir = path.join(self.context.topdir, ".cargo") + cargo_file = open(path.join(self.context.topdir, "Cargo.lock")) + content = toml.load(cargo_file) + + for package in content.get("package", []): + source = package.get("source", "") + version = package["version"] + if source == u"registry+https://github.com/rust-lang/crates.io-index": + crate_name = "{}-{}".format(package["name"], version) + if not packages["crates"].get(crate_name, False): + packages["crates"][package["name"]] = { + "current": [], + "exist": [], + } + packages["crates"][package["name"]]["current"].append(crate_name) + elif source.startswith("git+"): + name = source.split("#")[0].split("/")[-1].replace(".git", "") + branch = "" + crate_name = "{}-{}".format(package["name"], source.split("#")[1]) + crate_branch = name.split("?") + if len(crate_branch) > 1: + branch = crate_branch[1].replace("branch=", "") + name = crate_branch[0] + + if not packages["git"].get(name, False): + packages["git"][name] = { + "current": [], + "exist": [], + } + packages["git"][name]["current"].append(source.split("#")[1][:7]) + if branch: + packages["git"][name]["current"].append(branch) + + crates_dir = path.join(cargo_dir, "registry") + crates_cache_dir = "" + crates_src_dir = "" + if os.path.isdir(path.join(crates_dir, "cache")): + for p in os.listdir(path.join(crates_dir, "cache")): + crates_cache_dir = path.join(crates_dir, "cache", p) + crates_src_dir = path.join(crates_dir, "src", p) + + git_dir = path.join(cargo_dir, "git") + git_db_dir = path.join(git_dir, "db") + git_checkout_dir = path.join(git_dir, "checkouts") + git_db_list = filter(lambda f: not f.startswith('.'), os.listdir(git_db_dir)) + git_checkout_list = os.listdir(git_checkout_dir) + + for d in list(set(git_db_list + git_checkout_list)): + crate_name = d.replace("-{}".format(d.split("-")[-1]), "") + if not packages["git"].get(crate_name, False): + packages["git"][crate_name] = { + "current": [], + "exist": [], + } + if os.path.isdir(path.join(git_checkout_dir, d)): + for d2 in os.listdir(path.join(git_checkout_dir, d)): + dep_path = path.join(git_checkout_dir, d, d2) + if os.path.isdir(dep_path): + packages["git"][crate_name]["exist"].append((path.getmtime(dep_path), d, d2)) + elif os.path.isdir(path.join(git_db_dir, d)): + packages["git"][crate_name]["exist"].append(("db", d, "")) + + for d in os.listdir(crates_src_dir): + crate_name = re.sub(r"\-\d+(\.\d+){1,3}.+", "", d) + if not packages["crates"].get(crate_name, False): + packages["crates"][crate_name] = { + "current": [], + "exist": [], + } + packages["crates"][crate_name]["exist"].append(d) + + total_size = 0 + for packages_type in ["git", "crates"]: + sorted_packages = sorted(packages[packages_type]) + for crate_name in sorted_packages: + crate_count = 0 + existed_crates = packages[packages_type][crate_name]["exist"] + for exist in sorted(existed_crates, reverse=True): + current_crate = packages[packages_type][crate_name]["current"] + size = 0 + exist_name = exist + exist_item = exist[2] if packages_type == "git" else exist + if exist_item not in current_crate: + crate_count += 1 + removing_anything = True + if int(crate_count) >= int(keep) or not current_crate: + crate_paths = [] + if packages_type == "git": + exist_checkout_path = path.join(git_checkout_dir, exist[1]) + exist_db_path = path.join(git_db_dir, exist[1]) + exist_name = path.join(exist[1], exist[2]) + exist_path = path.join(git_checkout_dir, exist_name) + + if exist[0] == "db": + crate_paths.append(exist_db_path) + crate_count += -1 + else: + crate_paths.append(exist_path) + + # remove crate from checkout if doesn't exist in db directory + if not os.path.isdir(exist_db_path): + crate_count += -1 + + with cd(path.join(exist_path, ".git", "objects", "pack")): + for pack in glob.glob("*"): + pack_path = path.join(exist_db_path, "objects", "pack", pack) + if os.path.exists(pack_path): + crate_paths.append(pack_path) + + if len(os.listdir(exist_checkout_path)) <= 1: + crate_paths.append(exist_checkout_path) + if os.path.isdir(exist_db_path): + crate_paths.append(exist_db_path) + else: + crate_paths.append(path.join(crates_cache_dir, "{}.crate".format(exist))) + crate_paths.append(path.join(crates_src_dir, exist)) + + size = sum(get_size(p) for p in crate_paths) if show_size else 0 + total_size += size + print_msg = (exist_name, " ({}MB)".format(round(size, 2)) if show_size else "", cargo_dir) + if force: + print("Removing `{}`{} package from {}".format(*print_msg)) + for crate_path in crate_paths: + if os.path.exists(crate_path): + delete(crate_path) + else: + print("Would remove `{}`{} package from {}".format(*print_msg)) + + if removing_anything and show_size: + print("\nTotal size of {} MB".format(round(total_size, 2))) + + if not removing_anything: + print("Nothing to remove.") + elif not force: + print("\nNothing done. " + "Run `./mach clean-cargo-cache -f` to actually remove.") diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 792b6071145..3a33f5f3b0d 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -24,7 +24,7 @@ from mach.decorators import ( Command, ) -from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest +from servo.command_base import CommandBase, cd, call, BIN_SUFFIX from servo.util import host_triple @@ -413,7 +413,6 @@ class MachCommands(CommandBase): self.ensure_bootstrapped() env = self.build_env(is_build=True, geckolib=True) - geckolib_build_path = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") ret = None opts = [] @@ -433,14 +432,6 @@ class MachCommands(CommandBase): if features: opts += ["--features", ' '.join(features)] - if with_gecko is not None: - print("Generating atoms data...") - run_file = path.join(self.context.topdir, "components", - "style", "binding_tools", "regen_atoms.py") - run_globals = {"__file__": run_file} - execfile(run_file, run_globals) - run_globals["generate_atoms"](env["MOZ_DIST"]) - build_start = time() with cd(path.join("ports", "geckolib")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) @@ -451,15 +442,6 @@ class MachCommands(CommandBase): print("GeckoLib build completed in %s" % format_duration(elapsed)) - if with_gecko is not None: - print("Copying binding files to style/gecko_bindings...") - build_path = path.join(geckolib_build_path, "release" if release else "debug", "") - target_style_path = find_dep_path_newest("style", build_path) - out_gecko_path = path.join(target_style_path, "out", "gecko") - bindings_path = path.join(self.context.topdir, "components", "style", "gecko_bindings") - for f in ["bindings.rs", "structs_debug.rs", "structs_release.rs"]: - shutil.copy(path.join(out_gecko_path, f), bindings_path) - return ret @Command('clean', |