[ceph.git] / ceph / src / fmt / support / manage.py

#!/usr/bin/env python3

"""Manage site and releases.

Usage:
  manage.py release [<branch>]
  manage.py site

For the release command $FMT_TOKEN should contain a GitHub personal access token
obtained from https://github.com/settings/tokens.
"""

from __future__ import print_function
import datetime, docopt, errno, fileinput, json, os
import re, requests, shutil, sys, tempfile
from contextlib import contextmanager
from distutils.version import LooseVersion
from subprocess import check_call


class Git:
    def __init__(self, dir):
        self.dir = dir

    def call(self, method, args, **kwargs):
        return check_call(['git', method] + list(args), **kwargs)

    def add(self, *args):
        return self.call('add', args, cwd=self.dir)

    def checkout(self, *args):
        return self.call('checkout', args, cwd=self.dir)

    def clean(self, *args):
        return self.call('clean', args, cwd=self.dir)

    def clone(self, *args):
        return self.call('clone', list(args) + [self.dir])

    def commit(self, *args):
        return self.call('commit', args, cwd=self.dir)

    def pull(self, *args):
        return self.call('pull', args, cwd=self.dir)

    def push(self, *args):
        return self.call('push', args, cwd=self.dir)

    def reset(self, *args):
        return self.call('reset', args, cwd=self.dir)

    def update(self, *args):
        clone = not os.path.exists(self.dir)
        if clone:
            self.clone(*args)
        return clone


def clean_checkout(repo, branch):
    repo.clean('-f', '-d')
    repo.reset('--hard')
    repo.checkout(branch)


class Runner:
    def __init__(self, cwd):
        self.cwd = cwd

    def __call__(self, *args, **kwargs):
        kwargs['cwd'] = kwargs.get('cwd', self.cwd)
        check_call(args, **kwargs)


def create_build_env():
    """Create a build environment."""
    class Env:
        pass
    env = Env()

    # Import the documentation build module.
    env.fmt_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    sys.path.insert(0, os.path.join(env.fmt_dir, 'doc'))
    import build

    env.build_dir = 'build'
    env.versions = build.versions

    # Virtualenv and repos are cached to speed up builds.
    build.create_build_env(os.path.join(env.build_dir, 'virtualenv'))

    env.fmt_repo = Git(os.path.join(env.build_dir, 'fmt'))
    return env


@contextmanager
def rewrite(filename):
    class Buffer:
        pass
    buffer = Buffer()
    if not os.path.exists(filename):
        buffer.data = ''
        yield buffer
        return
    with open(filename) as f:
        buffer.data = f.read()
    yield buffer
    with open(filename, 'w') as f:
        f.write(buffer.data)


fmt_repo_url = 'git@github.com:fmtlib/fmt'


def update_site(env):
    env.fmt_repo.update(fmt_repo_url)

    doc_repo = Git(os.path.join(env.build_dir, 'fmtlib.github.io'))
    doc_repo.update('git@github.com:fmtlib/fmtlib.github.io')

    for version in env.versions:
        clean_checkout(env.fmt_repo, version)
        target_doc_dir = os.path.join(env.fmt_repo.dir, 'doc')
        # Remove the old theme.
        for entry in os.listdir(target_doc_dir):
            path = os.path.join(target_doc_dir, entry)
            if os.path.isdir(path):
                shutil.rmtree(path)
        # Copy the new theme.
        for entry in ['_static', '_templates', 'basic-bootstrap', 'bootstrap',
                      'conf.py', 'fmt.less']:
            src = os.path.join(env.fmt_dir, 'doc', entry)
            dst = os.path.join(target_doc_dir, entry)
            copy = shutil.copytree if os.path.isdir(src) else shutil.copyfile
            copy(src, dst)
        # Rename index to contents.
        contents = os.path.join(target_doc_dir, 'contents.rst')
        if not os.path.exists(contents):
            os.rename(os.path.join(target_doc_dir, 'index.rst'), contents)
        # Fix issues in reference.rst/api.rst.
        for filename in ['reference.rst', 'api.rst', 'index.rst']:
            pattern = re.compile('doxygenfunction.. (bin|oct|hexu|hex)$', re.M)
            with rewrite(os.path.join(target_doc_dir, filename)) as b:
                b.data = b.data.replace('std::ostream &', 'std::ostream&')
                b.data = re.sub(pattern, r'doxygenfunction:: \1(int)', b.data)
                b.data = b.data.replace('std::FILE*', 'std::FILE *')
                b.data = b.data.replace('unsigned int', 'unsigned')
                #b.data = b.data.replace('operator""_', 'operator"" _')
                b.data = b.data.replace(
                    'format_to_n(OutputIt, size_t, string_view, Args&&',
                    'format_to_n(OutputIt, size_t, const S&, const Args&')
                b.data = b.data.replace(
                    'format_to_n(OutputIt, std::size_t, string_view, Args&&',
                    'format_to_n(OutputIt, std::size_t, const S&, const Args&')
                if version == ('3.0.2'):
                    b.data = b.data.replace(
                        'fprintf(std::ostream&', 'fprintf(std::ostream &')
                if version == ('5.3.0'):
                    b.data = b.data.replace(
                        'format_to(OutputIt, const S&, const Args&...)',
                        'format_to(OutputIt, const S &, const Args &...)')
                if version.startswith('5.') or version.startswith('6.'):
                    b.data = b.data.replace(', size_t', ', std::size_t')
                if version.startswith('7.'):
                    b.data = b.data.replace(', std::size_t', ', size_t')
                    b.data = b.data.replace('join(It, It', 'join(It, Sentinel')
                if version.startswith('7.1.'):
                    b.data = b.data.replace(', std::size_t', ', size_t')
                    b.data = b.data.replace('join(It, It', 'join(It, Sentinel')
                    b.data = b.data.replace(
                        'fmt::format_to(OutputIt, const S&, Args&&...)',
                        'fmt::format_to(OutputIt, const S&, Args&&...) -> ' +
                        'typename std::enable_if<enable, OutputIt>::type')
                b.data = b.data.replace('aa long', 'a long')
                b.data = b.data.replace('serveral', 'several')
                if version.startswith('6.2.'):
                    b.data = b.data.replace(
                        'vformat(const S&, basic_format_args<' +
                        'buffer_context<Char>>)',
                        'vformat(const S&, basic_format_args<' +
                        'buffer_context<type_identity_t<Char>>>)')
        # Fix a broken link in index.rst.
        index = os.path.join(target_doc_dir, 'index.rst')
        with rewrite(index) as b:
            b.data = b.data.replace(
                'doc/latest/index.html#format-string-syntax', 'syntax.html')
        # Fix issues in syntax.rst.
        index = os.path.join(target_doc_dir, 'syntax.rst')
        with rewrite(index) as b:
            b.data = b.data.replace(
                '..productionlist:: sf\n', '.. productionlist:: sf\n ')
            b.data = b.data.replace('Examples:\n', 'Examples::\n')
        # Build the docs.
        html_dir = os.path.join(env.build_dir, 'html')
        if os.path.exists(html_dir):
            shutil.rmtree(html_dir)
        include_dir = env.fmt_repo.dir
        if LooseVersion(version) >= LooseVersion('5.0.0'):
            include_dir = os.path.join(include_dir, 'include', 'fmt')
        elif LooseVersion(version) >= LooseVersion('3.0.0'):
            include_dir = os.path.join(include_dir, 'fmt')
        import build
        build.build_docs(version, doc_dir=target_doc_dir,
                         include_dir=include_dir, work_dir=env.build_dir)
        shutil.rmtree(os.path.join(html_dir, '.doctrees'))
        # Create symlinks for older versions.
        for link, target in {'index': 'contents', 'api': 'reference'}.items():
            link = os.path.join(html_dir, link) + '.html'
            target += '.html'
            if os.path.exists(os.path.join(html_dir, target)) and \
               not os.path.exists(link):
                os.symlink(target, link)
        # Copy docs to the website.
        version_doc_dir = os.path.join(doc_repo.dir, version)
        try:
            shutil.rmtree(version_doc_dir)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
        shutil.move(html_dir, version_doc_dir)


def release(args):
    env = create_build_env()
    fmt_repo = env.fmt_repo

    branch = args.get('<branch>')
    if branch is None:
        branch = 'master'
    if not fmt_repo.update('-b', branch, fmt_repo_url):
        clean_checkout(fmt_repo, branch)

    # Convert changelog from RST to GitHub-flavored Markdown and get the
    # version.
    changelog = 'ChangeLog.rst'
    changelog_path = os.path.join(fmt_repo.dir, changelog)
    import rst2md
    changes, version = rst2md.convert(changelog_path)
    cmakelists = 'CMakeLists.txt'
    for line in fileinput.input(os.path.join(fmt_repo.dir, cmakelists),
                                inplace=True):
        prefix = 'set(FMT_VERSION '
        if line.startswith(prefix):
            line = prefix + version + ')\n'
        sys.stdout.write(line)

    # Update the version in the changelog.
    title_len = 0
    for line in fileinput.input(changelog_path, inplace=True):
        if line.startswith(version + ' - TBD'):
            line = version + ' - ' + datetime.date.today().isoformat()
            title_len = len(line)
            line += '\n'
        elif title_len:
            line = '-' * title_len + '\n'
            title_len = 0
        sys.stdout.write(line)

    # Add the version to the build script.
    script = os.path.join('doc', 'build.py')
    script_path = os.path.join(fmt_repo.dir, script)
    for line in fileinput.input(script_path, inplace=True):
      m = re.match(r'( *versions = )\[(.+)\]', line)
      if m:
        line = '{}[{}, \'{}\']\n'.format(m.group(1), m.group(2), version)
      sys.stdout.write(line)

    fmt_repo.checkout('-B', 'release')
    fmt_repo.add(changelog, cmakelists, script)
    fmt_repo.commit('-m', 'Update version')

    # Build the docs and package.
    run = Runner(fmt_repo.dir)
    run('cmake', '.')
    run('make', 'doc', 'package_source')
    update_site(env)

    # Create a release on GitHub.
    fmt_repo.push('origin', 'release')
    auth_headers = {'Authorization': 'token ' + os.getenv('FMT_TOKEN')}
    r = requests.post('https://api.github.com/repos/fmtlib/fmt/releases',
                      headers=auth_headers,
                      data=json.dumps({'tag_name': version,
                                       'target_commitish': 'release',
                                       'body': changes, 'draft': True}))
    if r.status_code != 201:
        raise Exception('Failed to create a release ' + str(r))
    id = r.json()['id']
    uploads_url = 'https://uploads.github.com/repos/fmtlib/fmt/releases'
    package = 'fmt-{}.zip'.format(version)
    r = requests.post(
        '{}/{}/assets?name={}'.format(uploads_url, id, package),
        headers={'Content-Type': 'application/zip'} | auth_headers,
        data=open('build/fmt/' + package, 'rb'))
    if r.status_code != 201:
        raise Exception('Failed to upload an asset ' + str(r))


if __name__ == '__main__':
    args = docopt.docopt(__doc__)
    if args.get('release'):
        release(args)
    elif args.get('site'):
        update_site(create_build_env())
Commit	Line	Data
1e59de90	1	#!/usr/bin/env python3
11fdf7f2 TL	2
	3	"""Manage site and releases.
	4
	5	Usage:
	6	manage.py release [<branch>]
	7	manage.py site
f67539c2 TL	8
	9	For the release command $FMT_TOKEN should contain a GitHub personal access token
	10	obtained from https://github.com/settings/tokens.
11fdf7f2 TL	11	"""
	12
	13	from __future__ import print_function
	14	import datetime, docopt, errno, fileinput, json, os
	15	import re, requests, shutil, sys, tempfile
	16	from contextlib import contextmanager
	17	from distutils.version import LooseVersion
	18	from subprocess import check_call
	19
	20
	21	class Git:
	22	def __init__(self, dir):
	23	self.dir = dir
	24
	25	def call(self, method, args, **kwargs):
	26	return check_call(['git', method] + list(args), **kwargs)
	27
	28	def add(self, *args):
	29	return self.call('add', args, cwd=self.dir)
	30
	31	def checkout(self, *args):
	32	return self.call('checkout', args, cwd=self.dir)
	33
	34	def clean(self, *args):
	35	return self.call('clean', args, cwd=self.dir)
	36
	37	def clone(self, *args):
	38	return self.call('clone', list(args) + [self.dir])
	39
	40	def commit(self, *args):
	41	return self.call('commit', args, cwd=self.dir)
	42
	43	def pull(self, *args):
	44	return self.call('pull', args, cwd=self.dir)
	45
	46	def push(self, *args):
	47	return self.call('push', args, cwd=self.dir)
	48
	49	def reset(self, *args):
	50	return self.call('reset', args, cwd=self.dir)
	51
	52	def update(self, *args):
	53	clone = not os.path.exists(self.dir)
	54	if clone:
	55	self.clone(*args)
	56	return clone
	57
	58
	59	def clean_checkout(repo, branch):
	60	repo.clean('-f', '-d')
	61	repo.reset('--hard')
	62	repo.checkout(branch)
	63
	64
	65	class Runner:
	66	def __init__(self, cwd):
	67	self.cwd = cwd
	68
	69	def __call__(self, args, *kwargs):
	70	kwargs['cwd'] = kwargs.get('cwd', self.cwd)
	71	check_call(args, **kwargs)
	72
	73
	74	def create_build_env():
75	"""Create a build environment."""
76	class Env:
77	pass
78	env = Env()
79
80	# Import the documentation build module.
81	env.fmt_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
82	sys.path.insert(0, os.path.join(env.fmt_dir, 'doc'))
83	import build
84
85	env.build_dir = 'build'
86	env.versions = build.versions
87
88	# Virtualenv and repos are cached to speed up builds.
89	build.create_build_env(os.path.join(env.build_dir, 'virtualenv'))
90
91	env.fmt_repo = Git(os.path.join(env.build_dir, 'fmt'))
92	return env
93
94
95	@contextmanager
96	def rewrite(filename):
97	class Buffer:
98	pass
99	buffer = Buffer()
100	if not os.path.exists(filename):
101	buffer.data = ''
102	yield buffer
103	return
104	with open(filename) as f:
105	buffer.data = f.read()
106	yield buffer
107	with open(filename, 'w') as f:
108	f.write(buffer.data)
109
110
111	fmt_repo_url = 'git@github.com:fmtlib/fmt'
112
113
114	def update_site(env):
115	env.fmt_repo.update(fmt_repo_url)
116
117	doc_repo = Git(os.path.join(env.build_dir, 'fmtlib.github.io'))
118	doc_repo.update('git@github.com:fmtlib/fmtlib.github.io')
119
120	for version in env.versions:
121	clean_checkout(env.fmt_repo, version)
122	target_doc_dir = os.path.join(env.fmt_repo.dir, 'doc')
123	# Remove the old theme.
124	for entry in os.listdir(target_doc_dir):
125	path = os.path.join(target_doc_dir, entry)
126	if os.path.isdir(path):
127	shutil.rmtree(path)
128	# Copy the new theme.
129	for entry in ['_static', '_templates', 'basic-bootstrap', 'bootstrap',
130	'conf.py', 'fmt.less']:
131	src = os.path.join(env.fmt_dir, 'doc', entry)
132	dst = os.path.join(target_doc_dir, entry)
133	copy = shutil.copytree if os.path.isdir(src) else shutil.copyfile
134	copy(src, dst)
135	# Rename index to contents.
136	contents = os.path.join(target_doc_dir, 'contents.rst')
137	if not os.path.exists(contents):
138	os.rename(os.path.join(target_doc_dir, 'index.rst'), contents)
139	# Fix issues in reference.rst/api.rst.
f67539c2	140	for filename in ['reference.rst', 'api.rst', 'index.rst']:
11fdf7f2 TL	141	pattern = re.compile('doxygenfunction.. (bin\|oct\|hexu\|hex)$', re.M)
	142	with rewrite(os.path.join(target_doc_dir, filename)) as b:
	143	b.data = b.data.replace('std::ostream &', 'std::ostream&')
	144	b.data = re.sub(pattern, r'doxygenfunction:: \1(int)', b.data)
	145	b.data = b.data.replace('std::FILE', 'std::FILE ')
	146	b.data = b.data.replace('unsigned int', 'unsigned')
f67539c2	147	#b.data = b.data.replace('operator""_', 'operator"" _')
20effc67 TL	148	b.data = b.data.replace(
	149	'format_to_n(OutputIt, size_t, string_view, Args&&',
	150	'format_to_n(OutputIt, size_t, const S&, const Args&')
	151	b.data = b.data.replace(
	152	'format_to_n(OutputIt, std::size_t, string_view, Args&&',
	153	'format_to_n(OutputIt, std::size_t, const S&, const Args&')
	154	if version == ('3.0.2'):
	155	b.data = b.data.replace(
	156	'fprintf(std::ostream&', 'fprintf(std::ostream &')
	157	if version == ('5.3.0'):
	158	b.data = b.data.replace(
	159	'format_to(OutputIt, const S&, const Args&...)',
	160	'format_to(OutputIt, const S &, const Args &...)')
	161	if version.startswith('5.') or version.startswith('6.'):
	162	b.data = b.data.replace(', size_t', ', std::size_t')
	163	if version.startswith('7.'):
	164	b.data = b.data.replace(', std::size_t', ', size_t')
	165	b.data = b.data.replace('join(It, It', 'join(It, Sentinel')
1e59de90 TL	166	if version.startswith('7.1.'):
	167	b.data = b.data.replace(', std::size_t', ', size_t')
	168	b.data = b.data.replace('join(It, It', 'join(It, Sentinel')
	169	b.data = b.data.replace(
	170	'fmt::format_to(OutputIt, const S&, Args&&...)',
	171	'fmt::format_to(OutputIt, const S&, Args&&...) -> ' +
	172	'typename std::enable_if<enable, OutputIt>::type')
f67539c2	173	b.data = b.data.replace('aa long', 'a long')
20effc67	174	b.data = b.data.replace('serveral', 'several')
f67539c2 TL	175	if version.startswith('6.2.'):
	176	b.data = b.data.replace(
	177	'vformat(const S&, basic_format_args<' +
	178	'buffer_context<Char>>)',
	179	'vformat(const S&, basic_format_args<' +
	180	'buffer_context<type_identity_t<Char>>>)')
11fdf7f2 TL	181	# Fix a broken link in index.rst.
	182	index = os.path.join(target_doc_dir, 'index.rst')
	183	with rewrite(index) as b:
	184	b.data = b.data.replace(
	185	'doc/latest/index.html#format-string-syntax', 'syntax.html')
1e59de90 TL	186	# Fix issues in syntax.rst.
	187	index = os.path.join(target_doc_dir, 'syntax.rst')
	188	with rewrite(index) as b:
	189	b.data = b.data.replace(
	190	'..productionlist:: sf\n', '.. productionlist:: sf\n ')
	191	b.data = b.data.replace('Examples:\n', 'Examples::\n')
11fdf7f2 TL	192	# Build the docs.
	193	html_dir = os.path.join(env.build_dir, 'html')
	194	if os.path.exists(html_dir):
	195	shutil.rmtree(html_dir)
	196	include_dir = env.fmt_repo.dir
	197	if LooseVersion(version) >= LooseVersion('5.0.0'):
	198	include_dir = os.path.join(include_dir, 'include', 'fmt')
	199	elif LooseVersion(version) >= LooseVersion('3.0.0'):
	200	include_dir = os.path.join(include_dir, 'fmt')
	201	import build
	202	build.build_docs(version, doc_dir=target_doc_dir,
	203	include_dir=include_dir, work_dir=env.build_dir)
	204	shutil.rmtree(os.path.join(html_dir, '.doctrees'))
	205	# Create symlinks for older versions.
	206	for link, target in {'index': 'contents', 'api': 'reference'}.items():
	207	link = os.path.join(html_dir, link) + '.html'
	208	target += '.html'
	209	if os.path.exists(os.path.join(html_dir, target)) and \
	210	not os.path.exists(link):
	211	os.symlink(target, link)
	212	# Copy docs to the website.
	213	version_doc_dir = os.path.join(doc_repo.dir, version)
	214	try:
	215	shutil.rmtree(version_doc_dir)
	216	except OSError as e:
	217	if e.errno != errno.ENOENT:
	218	raise
	219	shutil.move(html_dir, version_doc_dir)
	220
	221
	222	def release(args):
	223	env = create_build_env()
	224	fmt_repo = env.fmt_repo
	225
	226	branch = args.get('<branch>')
	227	if branch is None:
	228	branch = 'master'
	229	if not fmt_repo.update('-b', branch, fmt_repo_url):
	230	clean_checkout(fmt_repo, branch)
	231
	232	# Convert changelog from RST to GitHub-flavored Markdown and get the
	233	# version.
	234	changelog = 'ChangeLog.rst'
	235	changelog_path = os.path.join(fmt_repo.dir, changelog)
	236	import rst2md
	237	changes, version = rst2md.convert(changelog_path)
	238	cmakelists = 'CMakeLists.txt'
	239	for line in fileinput.input(os.path.join(fmt_repo.dir, cmakelists),
	240	inplace=True):
	241	prefix = 'set(FMT_VERSION '
	242	if line.startswith(prefix):
	243	line = prefix + version + ')\n'
	244	sys.stdout.write(line)
	245
	246	# Update the version in the changelog.
	247	title_len = 0
	248	for line in fileinput.input(changelog_path, inplace=True):
1e59de90	249	if line.startswith(version + ' - TBD'):
11fdf7f2 TL	250	line = version + ' - ' + datetime.date.today().isoformat()
	251	title_len = len(line)
	252	line += '\n'
	253	elif title_len:
	254	line = '-' * title_len + '\n'
	255	title_len = 0
	256	sys.stdout.write(line)
	257
	258	# Add the version to the build script.
	259	script = os.path.join('doc', 'build.py')
	260	script_path = os.path.join(fmt_repo.dir, script)
	261	for line in fileinput.input(script_path, inplace=True):
	262	m = re.match(r'( *versions = )\[(.+)\]', line)
	263	if m:
	264	line = '{}[{}, \'{}\']\n'.format(m.group(1), m.group(2), version)
	265	sys.stdout.write(line)
	266
	267	fmt_repo.checkout('-B', 'release')
	268	fmt_repo.add(changelog, cmakelists, script)
	269	fmt_repo.commit('-m', 'Update version')
	270
	271	# Build the docs and package.
	272	run = Runner(fmt_repo.dir)
	273	run('cmake', '.')
	274	run('make', 'doc', 'package_source')
	275	update_site(env)
	276
	277	# Create a release on GitHub.
	278	fmt_repo.push('origin', 'release')
1e59de90	279	auth_headers = {'Authorization': 'token ' + os.getenv('FMT_TOKEN')}
11fdf7f2	280	r = requests.post('https://api.github.com/repos/fmtlib/fmt/releases',
1e59de90	281	headers=auth_headers,
11fdf7f2 TL	282	data=json.dumps({'tag_name': version,
	283	'target_commitish': 'release',
	284	'body': changes, 'draft': True}))
	285	if r.status_code != 201:
	286	raise Exception('Failed to create a release ' + str(r))
	287	id = r.json()['id']
	288	uploads_url = 'https://uploads.github.com/repos/fmtlib/fmt/releases'
	289	package = 'fmt-{}.zip'.format(version)
	290	r = requests.post(
	291	'{}/{}/assets?name={}'.format(uploads_url, id, package),
1e59de90 TL	292	headers={'Content-Type': 'application/zip'} \| auth_headers,
1e59de90 TL	293	data=open('build/fmt/' + package, 'rb'))
11fdf7f2 TL	294	if r.status_code != 201:
	295	raise Exception('Failed to upload an asset ' + str(r))
	296
	297
	298	if __name__ == '__main__':
	299	args = docopt.docopt(__doc__)
	300	if args.get('release'):
	301	release(args)
	302	elif args.get('site'):
	303	update_site(create_build_env())