ceph/src/fmt/support/manage.py

   1 #!/usr/bin/env python
   2
   3 """Manage site and releases.
   4
   5 Usage:
   6   manage.py release [<branch>]
   7   manage.py site
   8
   9 For the release command $FMT_TOKEN should contain a GitHub personal access token
  10 obtained from https://github.com/settings/tokens.
  11 """
  12
  13 from __future__ import print_function
  14 import datetime, docopt, errno, fileinput, json, os
  15 import re, requests, shutil, sys, tempfile
  16 from contextlib import contextmanager
  17 from distutils.version import LooseVersion
  18 from subprocess import check_call
  19
  20
  21 class Git:
  22     def __init__(self, dir):
  23         self.dir = dir
  24
  25     def call(self, method, args, **kwargs):
  26         return check_call(['git', method] + list(args), **kwargs)
  27
  28     def add(self, *args):
  29         return self.call('add', args, cwd=self.dir)
  30
  31     def checkout(self, *args):
  32         return self.call('checkout', args, cwd=self.dir)
  33
  34     def clean(self, *args):
  35         return self.call('clean', args, cwd=self.dir)
  36
  37     def clone(self, *args):
  38         return self.call('clone', list(args) + [self.dir])
  39
  40     def commit(self, *args):
  41         return self.call('commit', args, cwd=self.dir)
  42
  43     def pull(self, *args):
  44         return self.call('pull', args, cwd=self.dir)
  45
  46     def push(self, *args):
  47         return self.call('push', args, cwd=self.dir)
  48
  49     def reset(self, *args):
  50         return self.call('reset', args, cwd=self.dir)
  51
  52     def update(self, *args):
  53         clone = not os.path.exists(self.dir)
  54         if clone:
  55             self.clone(*args)
  56         return clone
  57
  58
  59 def clean_checkout(repo, branch):
  60     repo.clean('-f', '-d')
  61     repo.reset('--hard')
  62     repo.checkout(branch)
  63
  64
  65 class Runner:
  66     def __init__(self, cwd):
  67         self.cwd = cwd
  68
  69     def __call__(self, *args, **kwargs):
  70         kwargs['cwd'] = kwargs.get('cwd', self.cwd)
  71         check_call(args, **kwargs)
  72
  73
  74 def create_build_env():
  75     """Create a build environment."""
  76     class Env:
  77         pass
  78     env = Env()
  79
  80     # Import the documentation build module.
  81     env.fmt_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  82     sys.path.insert(0, os.path.join(env.fmt_dir, 'doc'))
  83     import build
  84
  85     env.build_dir = 'build'
  86     env.versions = build.versions
  87
  88     # Virtualenv and repos are cached to speed up builds.
  89     build.create_build_env(os.path.join(env.build_dir, 'virtualenv'))
  90
  91     env.fmt_repo = Git(os.path.join(env.build_dir, 'fmt'))
  92     return env
  93
  94
  95 @contextmanager
  96 def rewrite(filename):
  97     class Buffer:
  98         pass
  99     buffer = Buffer()
 100     if not os.path.exists(filename):
 101         buffer.data = ''
 102         yield buffer
 103         return
 104     with open(filename) as f:
 105         buffer.data = f.read()
 106     yield buffer
 107     with open(filename, 'w') as f:
 108         f.write(buffer.data)
 109
 110
 111 fmt_repo_url = 'git@github.com:fmtlib/fmt'
 112
 113
 114 def update_site(env):
 115     env.fmt_repo.update(fmt_repo_url)
 116
 117     doc_repo = Git(os.path.join(env.build_dir, 'fmtlib.github.io'))
 118     doc_repo.update('git@github.com:fmtlib/fmtlib.github.io')
 119
 120     for version in env.versions:
 121         clean_checkout(env.fmt_repo, version)
 122         target_doc_dir = os.path.join(env.fmt_repo.dir, 'doc')
 123         # Remove the old theme.
 124         for entry in os.listdir(target_doc_dir):
 125             path = os.path.join(target_doc_dir, entry)
 126             if os.path.isdir(path):
 127                 shutil.rmtree(path)
 128         # Copy the new theme.
 129         for entry in ['_static', '_templates', 'basic-bootstrap', 'bootstrap',
 130                       'conf.py', 'fmt.less']:
 131             src = os.path.join(env.fmt_dir, 'doc', entry)
 132             dst = os.path.join(target_doc_dir, entry)
 133             copy = shutil.copytree if os.path.isdir(src) else shutil.copyfile
 134             copy(src, dst)
 135         # Rename index to contents.
 136         contents = os.path.join(target_doc_dir, 'contents.rst')
 137         if not os.path.exists(contents):
 138             os.rename(os.path.join(target_doc_dir, 'index.rst'), contents)
 139         # Fix issues in reference.rst/api.rst.
 140         for filename in ['reference.rst', 'api.rst', 'index.rst']:
 141             pattern = re.compile('doxygenfunction.. (bin|oct|hexu|hex)$', re.M)
 142             with rewrite(os.path.join(target_doc_dir, filename)) as b:
 143                 b.data = b.data.replace('std::ostream &', 'std::ostream&')
 144                 b.data = re.sub(pattern, r'doxygenfunction:: \1(int)', b.data)
 145                 b.data = b.data.replace('std::FILE*', 'std::FILE *')
 146                 b.data = b.data.replace('unsigned int', 'unsigned')
 147                 #b.data = b.data.replace('operator""_', 'operator"" _')
 148                 b.data = b.data.replace(', size_t', ', std::size_t')
 149                 b.data = b.data.replace('aa long', 'a long')
 150                 if version.startswith('6.2.'):
 151                     b.data = b.data.replace(
 152                         'vformat(const S&, basic_format_args<' +
 153                         'buffer_context<Char>>)',
 154                         'vformat(const S&, basic_format_args<' +
 155                         'buffer_context<type_identity_t<Char>>>)')
 156         # Fix a broken link in index.rst.
 157         index = os.path.join(target_doc_dir, 'index.rst')
 158         with rewrite(index) as b:
 159             b.data = b.data.replace(
 160                 'doc/latest/index.html#format-string-syntax', 'syntax.html')
 161         # Build the docs.
 162         html_dir = os.path.join(env.build_dir, 'html')
 163         if os.path.exists(html_dir):
 164             shutil.rmtree(html_dir)
 165         include_dir = env.fmt_repo.dir
 166         if LooseVersion(version) >= LooseVersion('5.0.0'):
 167             include_dir = os.path.join(include_dir, 'include', 'fmt')
 168         elif LooseVersion(version) >= LooseVersion('3.0.0'):
 169             include_dir = os.path.join(include_dir, 'fmt')
 170         import build
 171         build.build_docs(version, doc_dir=target_doc_dir,
 172                          include_dir=include_dir, work_dir=env.build_dir)
 173         shutil.rmtree(os.path.join(html_dir, '.doctrees'))
 174         # Create symlinks for older versions.
 175         for link, target in {'index': 'contents', 'api': 'reference'}.items():
 176             link = os.path.join(html_dir, link) + '.html'
 177             target += '.html'
 178             if os.path.exists(os.path.join(html_dir, target)) and \
 179                not os.path.exists(link):
 180                 os.symlink(target, link)
 181         # Copy docs to the website.
 182         version_doc_dir = os.path.join(doc_repo.dir, version)
 183         try:
 184             shutil.rmtree(version_doc_dir)
 185         except OSError as e:
 186             if e.errno != errno.ENOENT:
 187                 raise
 188         shutil.move(html_dir, version_doc_dir)
 189
 190
 191 def release(args):
 192     env = create_build_env()
 193     fmt_repo = env.fmt_repo
 194
 195     branch = args.get('<branch>')
 196     if branch is None:
 197         branch = 'master'
 198     if not fmt_repo.update('-b', branch, fmt_repo_url):
 199         clean_checkout(fmt_repo, branch)
 200
 201     # Convert changelog from RST to GitHub-flavored Markdown and get the
 202     # version.
 203     changelog = 'ChangeLog.rst'
 204     changelog_path = os.path.join(fmt_repo.dir, changelog)
 205     import rst2md
 206     changes, version = rst2md.convert(changelog_path)
 207     cmakelists = 'CMakeLists.txt'
 208     for line in fileinput.input(os.path.join(fmt_repo.dir, cmakelists),
 209                                 inplace=True):
 210         prefix = 'set(FMT_VERSION '
 211         if line.startswith(prefix):
 212             line = prefix + version + ')\n'
 213         sys.stdout.write(line)
 214
 215     # Update the version in the changelog.
 216     title_len = 0
 217     for line in fileinput.input(changelog_path, inplace=True):
 218         if line.decode('utf-8').startswith(version + ' - TBD'):
 219             line = version + ' - ' + datetime.date.today().isoformat()
 220             title_len = len(line)
 221             line += '\n'
 222         elif title_len:
 223             line = '-' * title_len + '\n'
 224             title_len = 0
 225         sys.stdout.write(line)
 226
 227     # Add the version to the build script.
 228     script = os.path.join('doc', 'build.py')
 229     script_path = os.path.join(fmt_repo.dir, script)
 230     for line in fileinput.input(script_path, inplace=True):
 231       m = re.match(r'( *versions = )\[(.+)\]', line)
 232       if m:
 233         line = '{}[{}, \'{}\']\n'.format(m.group(1), m.group(2), version)
 234       sys.stdout.write(line)
 235
 236     fmt_repo.checkout('-B', 'release')
 237     fmt_repo.add(changelog, cmakelists, script)
 238     fmt_repo.commit('-m', 'Update version')
 239
 240     # Build the docs and package.
 241     run = Runner(fmt_repo.dir)
 242     run('cmake', '.')
 243     run('make', 'doc', 'package_source')
 244     update_site(env)
 245
 246     # Create a release on GitHub.
 247     fmt_repo.push('origin', 'release')
 248     params = {'access_token': os.getenv('FMT_TOKEN')}
 249     r = requests.post('https://api.github.com/repos/fmtlib/fmt/releases',
 250                       params=params,
 251                       data=json.dumps({'tag_name': version,
 252                                        'target_commitish': 'release',
 253                                        'body': changes, 'draft': True}))
 254     if r.status_code != 201:
 255         raise Exception('Failed to create a release ' + str(r))
 256     id = r.json()['id']
 257     uploads_url = 'https://uploads.github.com/repos/fmtlib/fmt/releases'
 258     package = 'fmt-{}.zip'.format(version)
 259     r = requests.post(
 260         '{}/{}/assets?name={}'.format(uploads_url, id, package),
 261         headers={'Content-Type': 'application/zip'},
 262         params=params, data=open('build/fmt/' + package, 'rb'))
 263     if r.status_code != 201:
 264         raise Exception('Failed to upload an asset ' + str(r))
 265
 266
 267 if __name__ == '__main__':
 268     args = docopt.docopt(__doc__)
 269     if args.get('release'):
 270         release(args)
 271     elif args.get('site'):
 272         update_site(create_build_env())