ceph/src/fmt/support/manage.py

   1 #!/usr/bin/env python
   2
   3 """Manage site and releases.
   4
   5 Usage:
   6   manage.py release [<branch>]
   7   manage.py site
   8 """
   9
  10 from __future__ import print_function
  11 import datetime, docopt, errno, fileinput, json, os
  12 import re, requests, shutil, sys, tempfile
  13 from contextlib import contextmanager
  14 from distutils.version import LooseVersion
  15 from subprocess import check_call
  16
  17
  18 class Git:
  19     def __init__(self, dir):
  20         self.dir = dir
  21
  22     def call(self, method, args, **kwargs):
  23         return check_call(['git', method] + list(args), **kwargs)
  24
  25     def add(self, *args):
  26         return self.call('add', args, cwd=self.dir)
  27
  28     def checkout(self, *args):
  29         return self.call('checkout', args, cwd=self.dir)
  30
  31     def clean(self, *args):
  32         return self.call('clean', args, cwd=self.dir)
  33
  34     def clone(self, *args):
  35         return self.call('clone', list(args) + [self.dir])
  36
  37     def commit(self, *args):
  38         return self.call('commit', args, cwd=self.dir)
  39
  40     def pull(self, *args):
  41         return self.call('pull', args, cwd=self.dir)
  42
  43     def push(self, *args):
  44         return self.call('push', args, cwd=self.dir)
  45
  46     def reset(self, *args):
  47         return self.call('reset', args, cwd=self.dir)
  48
  49     def update(self, *args):
  50         clone = not os.path.exists(self.dir)
  51         if clone:
  52             self.clone(*args)
  53         return clone
  54
  55
  56 def clean_checkout(repo, branch):
  57     repo.clean('-f', '-d')
  58     repo.reset('--hard')
  59     repo.checkout(branch)
  60
  61
  62 class Runner:
  63     def __init__(self, cwd):
  64         self.cwd = cwd
  65
  66     def __call__(self, *args, **kwargs):
  67         kwargs['cwd'] = kwargs.get('cwd', self.cwd)
  68         check_call(args, **kwargs)
  69
  70
  71 def create_build_env():
  72     """Create a build environment."""
  73     class Env:
  74         pass
  75     env = Env()
  76
  77     # Import the documentation build module.
  78     env.fmt_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  79     sys.path.insert(0, os.path.join(env.fmt_dir, 'doc'))
  80     import build
  81
  82     env.build_dir = 'build'
  83     env.versions = build.versions
  84
  85     # Virtualenv and repos are cached to speed up builds.
  86     build.create_build_env(os.path.join(env.build_dir, 'virtualenv'))
  87
  88     env.fmt_repo = Git(os.path.join(env.build_dir, 'fmt'))
  89     return env
  90
  91
  92 @contextmanager
  93 def rewrite(filename):
  94     class Buffer:
  95         pass
  96     buffer = Buffer()
  97     if not os.path.exists(filename):
  98         buffer.data = ''
  99         yield buffer
 100         return
 101     with open(filename) as f:
 102         buffer.data = f.read()
 103     yield buffer
 104     with open(filename, 'w') as f:
 105         f.write(buffer.data)
 106
 107
 108 fmt_repo_url = 'git@github.com:fmtlib/fmt'
 109
 110
 111 def update_site(env):
 112     env.fmt_repo.update(fmt_repo_url)
 113
 114     doc_repo = Git(os.path.join(env.build_dir, 'fmtlib.github.io'))
 115     doc_repo.update('git@github.com:fmtlib/fmtlib.github.io')
 116
 117     for version in env.versions:
 118         clean_checkout(env.fmt_repo, version)
 119         target_doc_dir = os.path.join(env.fmt_repo.dir, 'doc')
 120         # Remove the old theme.
 121         for entry in os.listdir(target_doc_dir):
 122             path = os.path.join(target_doc_dir, entry)
 123             if os.path.isdir(path):
 124                 shutil.rmtree(path)
 125         # Copy the new theme.
 126         for entry in ['_static', '_templates', 'basic-bootstrap', 'bootstrap',
 127                       'conf.py', 'fmt.less']:
 128             src = os.path.join(env.fmt_dir, 'doc', entry)
 129             dst = os.path.join(target_doc_dir, entry)
 130             copy = shutil.copytree if os.path.isdir(src) else shutil.copyfile
 131             copy(src, dst)
 132         # Rename index to contents.
 133         contents = os.path.join(target_doc_dir, 'contents.rst')
 134         if not os.path.exists(contents):
 135             os.rename(os.path.join(target_doc_dir, 'index.rst'), contents)
 136         # Fix issues in reference.rst/api.rst.
 137         for filename in ['reference.rst', 'api.rst']:
 138             pattern = re.compile('doxygenfunction.. (bin|oct|hexu|hex)$', re.M)
 139             with rewrite(os.path.join(target_doc_dir, filename)) as b:
 140                 b.data = b.data.replace('std::ostream &', 'std::ostream&')
 141                 b.data = re.sub(pattern, r'doxygenfunction:: \1(int)', b.data)
 142                 b.data = b.data.replace('std::FILE*', 'std::FILE *')
 143                 b.data = b.data.replace('unsigned int', 'unsigned')
 144                 b.data = b.data.replace('operator""_', 'operator"" _')
 145         # Fix a broken link in index.rst.
 146         index = os.path.join(target_doc_dir, 'index.rst')
 147         with rewrite(index) as b:
 148             b.data = b.data.replace(
 149                 'doc/latest/index.html#format-string-syntax', 'syntax.html')
 150         # Build the docs.
 151         html_dir = os.path.join(env.build_dir, 'html')
 152         if os.path.exists(html_dir):
 153             shutil.rmtree(html_dir)
 154         include_dir = env.fmt_repo.dir
 155         if LooseVersion(version) >= LooseVersion('5.0.0'):
 156             include_dir = os.path.join(include_dir, 'include', 'fmt')
 157         elif LooseVersion(version) >= LooseVersion('3.0.0'):
 158             include_dir = os.path.join(include_dir, 'fmt')
 159         import build
 160         build.build_docs(version, doc_dir=target_doc_dir,
 161                          include_dir=include_dir, work_dir=env.build_dir)
 162         shutil.rmtree(os.path.join(html_dir, '.doctrees'))
 163         # Create symlinks for older versions.
 164         for link, target in {'index': 'contents', 'api': 'reference'}.items():
 165             link = os.path.join(html_dir, link) + '.html'
 166             target += '.html'
 167             if os.path.exists(os.path.join(html_dir, target)) and \
 168                not os.path.exists(link):
 169                 os.symlink(target, link)
 170         # Copy docs to the website.
 171         version_doc_dir = os.path.join(doc_repo.dir, version)
 172         try:
 173             shutil.rmtree(version_doc_dir)
 174         except OSError as e:
 175             if e.errno != errno.ENOENT:
 176                 raise
 177         shutil.move(html_dir, version_doc_dir)
 178
 179
 180 def release(args):
 181     env = create_build_env()
 182     fmt_repo = env.fmt_repo
 183
 184     branch = args.get('<branch>')
 185     if branch is None:
 186         branch = 'master'
 187     if not fmt_repo.update('-b', branch, fmt_repo_url):
 188         clean_checkout(fmt_repo, branch)
 189
 190     # Convert changelog from RST to GitHub-flavored Markdown and get the
 191     # version.
 192     changelog = 'ChangeLog.rst'
 193     changelog_path = os.path.join(fmt_repo.dir, changelog)
 194     import rst2md
 195     changes, version = rst2md.convert(changelog_path)
 196     cmakelists = 'CMakeLists.txt'
 197     for line in fileinput.input(os.path.join(fmt_repo.dir, cmakelists),
 198                                 inplace=True):
 199         prefix = 'set(FMT_VERSION '
 200         if line.startswith(prefix):
 201             line = prefix + version + ')\n'
 202         sys.stdout.write(line)
 203
 204     # Update the version in the changelog.
 205     title_len = 0
 206     for line in fileinput.input(changelog_path, inplace=True):
 207         if line.decode('utf-8').startswith(version + ' - TBD'):
 208             line = version + ' - ' + datetime.date.today().isoformat()
 209             title_len = len(line)
 210             line += '\n'
 211         elif title_len:
 212             line = '-' * title_len + '\n'
 213             title_len = 0
 214         sys.stdout.write(line)
 215
 216     # Add the version to the build script.
 217     script = os.path.join('doc', 'build.py')
 218     script_path = os.path.join(fmt_repo.dir, script)
 219     for line in fileinput.input(script_path, inplace=True):
 220       m = re.match(r'( *versions = )\[(.+)\]', line)
 221       if m:
 222         line = '{}[{}, \'{}\']\n'.format(m.group(1), m.group(2), version)
 223       sys.stdout.write(line)
 224
 225     fmt_repo.checkout('-B', 'release')
 226     fmt_repo.add(changelog, cmakelists, script)
 227     fmt_repo.commit('-m', 'Update version')
 228
 229     # Build the docs and package.
 230     run = Runner(fmt_repo.dir)
 231     run('cmake', '.')
 232     run('make', 'doc', 'package_source')
 233     update_site(env)
 234
 235     # Create a release on GitHub.
 236     fmt_repo.push('origin', 'release')
 237     params = {'access_token': os.getenv('FMT_TOKEN')}
 238     r = requests.post('https://api.github.com/repos/fmtlib/fmt/releases',
 239                       params=params,
 240                       data=json.dumps({'tag_name': version,
 241                                        'target_commitish': 'release',
 242                                        'body': changes, 'draft': True}))
 243     if r.status_code != 201:
 244         raise Exception('Failed to create a release ' + str(r))
 245     id = r.json()['id']
 246     uploads_url = 'https://uploads.github.com/repos/fmtlib/fmt/releases'
 247     package = 'fmt-{}.zip'.format(version)
 248     r = requests.post(
 249         '{}/{}/assets?name={}'.format(uploads_url, id, package),
 250         headers={'Content-Type': 'application/zip'},
 251         params=params, data=open('build/fmt/' + package, 'rb'))
 252     if r.status_code != 201:
 253         raise Exception('Failed to upload an asset ' + str(r))
 254
 255
 256 if __name__ == '__main__':
 257     args = docopt.docopt(__doc__)
 258     if args.get('release'):
 259         release(args)
 260     elif args.get('site'):
 261         update_site(create_build_env())