ceph/src/arrow/dev/tasks/conda-recipes/clean.py

   1 from subprocess import check_output, check_call
   2 from typing import List
   3
   4 import json
   5 import os
   6 import pandas as pd
   7 import sys
   8
   9 from packaging.version import Version
  10
  11
  12 VERSIONS_TO_KEEP = 5
  13 PACKAGES = [
  14     "arrow-cpp",
  15     "arrow-cpp-proc",
  16     "parquet-cpp",
  17     "pyarrow",
  18     "pyarrow-tests",
  19     "r-arrow",
  20 ]
  21 PLATFORMS = [
  22     "linux-64",
  23     "linux-aarch64",
  24     "osx-64",
  25     "win-64",
  26 ]
  27 EXCLUDED_PATTERNS = [
  28     ["r-arrow", "linux-aarch64"],
  29 ]
  30
  31
  32 def packages_to_delete(package_name: str, platform: str) -> List[str]:
  33     env = os.environ.copy()
  34     env["CONDA_SUBDIR"] = platform
  35     pkgs_json = check_output(
  36         [
  37             "conda",
  38             "search",
  39             "--json",
  40             "-c",
  41             "arrow-nightlies",
  42             "--override-channels",
  43             package_name,
  44         ],
  45         env=env,
  46     )
  47     pkgs = pd.DataFrame(json.loads(pkgs_json)[package_name])
  48     pkgs["version"] = pkgs["version"].map(Version)
  49     pkgs["py_version"] = pkgs["build"].str.slice(0, 4)
  50
  51     to_delete = []
  52
  53     for (subdir, python), group in pkgs.groupby(["subdir", "py_version"]):
  54         group = group.sort_values(by="version", ascending=False)
  55
  56         if len(group) > VERSIONS_TO_KEEP:
  57             del_candidates = group[VERSIONS_TO_KEEP:]
  58             to_delete += (
  59                 f"arrow-nightlies/{package_name}/"
  60                 + del_candidates["version"].astype(str)
  61                 + del_candidates["url"].str.replace(
  62                     "https://conda.anaconda.org/arrow-nightlies", ""
  63                 )
  64             ).to_list()
  65
  66     return to_delete
  67
  68
  69 if __name__ == "__main__":
  70     to_delete = []
  71     for package in PACKAGES:
  72         for platform in PLATFORMS:
  73             if [package, platform] in EXCLUDED_PATTERNS:
  74                 continue
  75             to_delete += packages_to_delete(package, platform)
  76
  77     for name in to_delete:
  78         print(f"Deleting {name} …")
  79         if "FORCE" in sys.argv:
  80             check_call(["anaconda", "remove", "-f", name])