#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# About this script:
# This script is intended to simplify creating Subversion releases, by
# automating as much as is possible. It works well with our Apache
# infrastructure, and should make rolling, posting, and announcing
# releases dirt simple.
#
# This script may be run on a number of platforms, but it is intended to
# be run on people.apache.org. As such, it may have dependencies (such
# as Python version) which may not be common, but are guaranteed to be
# available on people.apache.org.
# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed
# Stuff we need
import os
import re
import sys
import glob
import shutil
import urllib2
import hashlib
import tarfile
import logging
import datetime
import operator
import itertools
import subprocess
import argparse # standard in Python 2.7
# Find ezt, using Subversion's copy, if there isn't one on the system.
try:
import ezt
except ImportError:
ezt_path = os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0])))
ezt_path = os.path.join(ezt_path, 'build', 'generator')
sys.path.append(ezt_path)
import ezt
# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'
# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'
people_host = 'minotaur.apache.org'
people_dist_dir = '/www/www.apache.org/dist/subversion'
#----------------------------------------------------------------------
# Utility functions
class Version(object):
regex = re.compile('(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
def __init__(self, ver_str):
match = self.regex.search(ver_str)
if not match:
raise RuntimeError("Bad version string '%s'" % ver_str)
self.major = int(match.group(1))
self.minor = int(match.group(2))
self.patch = int(match.group(3))
if match.group(4):
self.pre = match.group(4)
self.pre_num = int(match.group(5))
else:
self.pre = None
self.pre_num = None
self.base = '%d.%d.%d' % (self.major, self.minor, self.patch)
def is_prerelease(self):
return self.pre != None
def __lt__(self, that):
if self.major < that.major: return True
if self.major > that.major: return False
if self.minor < that.minor: return True
if self.minor > that.minor: return False
if self.patch < that.patch: return True
if self.patch > that.patch: return False
if not self.pre and not that.pre: return False
if not self.pre and that.pre: return False
if self.pre and not that.pre: return True
# We are both pre-releases
if self.pre != that.pre:
return self.pre < that.pre
else:
return self.pre_num < that.pre_num
def __str(self):
if self.pre:
extra = '-%s%d' % (self.pre, self.pre_num)
else:
extra = ''
return self.base + extra
def __repr__(self):
return "Version('%s')" % self.__str()
def __str__(self):
return self.__str()
def get_prefix(base_dir):
return os.path.join(base_dir, 'prefix')
def get_tempdir(base_dir):
return os.path.join(base_dir, 'tempdir')
def get_deploydir(base_dir):
return os.path.join(base_dir, 'deploy')
def get_tmpldir():
return os.path.join(os.path.abspath(sys.path[0]), 'templates')
def get_tmplfile(filename):
try:
return open(os.path.join(get_tmpldir(), filename))
except IOError:
# Hmm, we had a problem with the local version, let's try the repo
return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename)
def get_nullfile():
# This is certainly not cross platform
return open('/dev/null', 'w')
def run_script(verbose, script):
if verbose:
stdout = None
stderr = None
else:
stdout = get_nullfile()
stderr = subprocess.STDOUT
for l in script.split('\n'):
subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)
def download_file(url, target):
response = urllib2.urlopen(url)
target_file = open(target, 'w')
target_file.write(response.read())
def assert_people():
if os.uname()[1] != people_host:
raise RuntimeError('Not running on expected host "%s"' % people_host)
#----------------------------------------------------------------------
# Cleaning up the environment
def cleanup(args):
'Remove generated files and folders.'
logging.info('Cleaning')
shutil.rmtree(get_prefix(args.base_dir), True)
shutil.rmtree(get_tempdir(args.base_dir), True)
shutil.rmtree(get_deploydir(args.base_dir), True)
#----------------------------------------------------------------------
# Creating an environment to roll the release
class RollDep(object):
'The super class for each of the build dependencies.'
def __init__(self, base_dir, use_existing, verbose):
self._base_dir = base_dir
self._use_existing = use_existing
self._verbose = verbose
def _test_version(self, cmd):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
(stdout, stderr) = proc.communicate()
rc = proc.wait()
if rc: return ''
return stdout.split('\n')
def build(self):
if not hasattr(self, '_extra_configure_flags'):
self._extra_configure_flags = ''
cwd = os.getcwd()
tempdir = get_tempdir(self._base_dir)
tarball = os.path.join(tempdir, self._filebase + '.tar.gz')
if os.path.exists(tarball):
if not self._use_existing:
raise RuntimeError('autoconf tarball "%s" already exists'
% tarball)
logging.info('Using existing %s.tar.gz' % self._filebase)
else:
logging.info('Fetching %s' % self._filebase)
download_file(self._url, tarball)
# Extract tarball
tarfile.open(tarball).extractall(tempdir)
logging.info('Building ' + self.label)
os.chdir(os.path.join(tempdir, self._filebase))
run_script(self._verbose,
'''./configure --prefix=%s %s
make
make install''' % (get_prefix(self._base_dir),
self._extra_configure_flags))
os.chdir(cwd)
class AutoconfDep(RollDep):
def __init__(self, base_dir, use_existing, verbose):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'autoconf'
self._filebase = 'autoconf-' + autoconf_ver
self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase
def have_usable(self):
output = self._test_version(['autoconf', '-V'])
if not output: return False
version = output[0].split()[-1:][0]
return version == autoconf_ver
def use_system(self):
if not self._use_existing: return False
return self.have_usable()
class LibtoolDep(RollDep):
def __init__(self, base_dir, use_existing, verbose):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'libtool'
self._filebase = 'libtool-' + libtool_ver
self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase
def have_usable(self):
output = self._test_version(['libtool', '--version'])
if not output: return False
version = output[0].split()[-1:][0]
return version == libtool_ver
def use_system(self):
# We unconditionally return False here, to avoid using a borked
# system libtool (I'm looking at you, Debian).
return False
class SwigDep(RollDep):
def __init__(self, base_dir, use_existing, verbose, sf_mirror):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'swig'
self._filebase = 'swig-' + swig_ver
self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
{ 'swig' : self._filebase,
'sf_mirror' : sf_mirror }
self._extra_configure_flags = '--without-pcre'
def have_usable(self):
output = self._test_version(['swig', '-version'])
if not output: return False
version = output[1].split()[-1:][0]
return version == swig_ver
def use_system(self):
if not self._use_existing: return False
return self.have_usable()
def build_env(args):
'Download prerequisites for a release and prepare the environment.'
logging.info('Creating release environment')
try:
os.mkdir(get_prefix(args.base_dir))
os.mkdir(get_tempdir(args.base_dir))
except OSError:
if not args.use_existing:
raise
autoconf = AutoconfDep(args.base_dir, args.use_existing, args.verbose)
libtool = LibtoolDep(args.base_dir, args.use_existing, args.verbose)
swig = SwigDep(args.base_dir, args.use_existing, args.verbose,
args.sf_mirror)
# iterate over our rolling deps, and build them if needed
for dep in [autoconf, libtool, swig]:
if dep.use_system():
logging.info('Using system %s' % dep.label)
else:
dep.build()
#----------------------------------------------------------------------
# Create release artifacts
def roll_tarballs(args):
'Create the release artifacts.'
extns = ['zip', 'tar.gz', 'tar.bz2']
if args.branch:
branch = args.branch
else:
branch = args.version.base[:-1] + 'x'
logging.info('Rolling release %s from branch %s@%d' % (args.version,
branch, args.revnum))
# Ensure we've got the appropriate rolling dependencies available
autoconf = AutoconfDep(args.base_dir, False, args.verbose)
libtool = LibtoolDep(args.base_dir, False, args.verbose)
swig = SwigDep(args.base_dir, False, args.verbose, None)
for dep in [autoconf, libtool, swig]:
if not dep.have_usable():
raise RuntimeError('Cannot find usable %s' % dep.label)
# Make sure CHANGES is sync'd
if branch != 'trunk':
trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
args.revnum)
proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
trunk_CHANGES],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
(stdout, stderr) = proc.communicate()
proc.wait()
if stdout:
raise RuntimeError('CHANGES not synced between trunk and branch')
# Create the output directory
if not os.path.exists(get_deploydir(args.base_dir)):
os.mkdir(get_deploydir(args.base_dir))
# For now, just delegate to dist.sh to create the actual artifacts
extra_args = ''
if args.version.is_prerelease():
extra_args = '-%s %d' % (args.version.pre, args.version.pre_num)
logging.info('Building UNIX tarballs')
run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
% (sys.path[0], args.version.base, branch, args.revnum,
extra_args) )
logging.info('Buildling Windows tarballs')
run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
% (sys.path[0], args.version.base, branch, args.revnum,
extra_args) )
# Move the results to the deploy directory
logging.info('Moving artifacts and calculating checksums')
for e in extns:
if args.version.pre == 'nightly':
filename = 'subversion-trunk.%s' % e
else:
filename = 'subversion-%s.%s' % (args.version, e)
shutil.move(filename, get_deploydir(args.base_dir))
filename = os.path.join(get_deploydir(args.base_dir), filename)
m = hashlib.sha1()
m.update(open(filename, 'r').read())
open(filename + '.sha1', 'w').write(m.hexdigest())
shutil.move('svn_version.h.dist', get_deploydir(args.base_dir))
# And we're done!
#----------------------------------------------------------------------
# Post the candidate release artifacts
def post_candidates(args):
'Post the generated tarballs to web-accessible directory.'
if args.target:
target = args.target
else:
target = os.path.join(os.getenv('HOME'), 'public_html', 'svn',
args.version)
if args.code_name:
dirname = args.code_name
else:
dirname = 'deploy'
if not os.path.exists(target):
os.makedirs(target)
data = { 'version' : args.version,
'revnum' : args.revnum,
'dirname' : dirname,
}
# Choose the right template text
if args.version.is_prerelease():
if args.version.pre == 'nightly':
template_filename = 'nightly-candidates.ezt'
else:
template_filename = 'rc-candidates.ezt'
else:
template_filename = 'stable-candidates.ezt'
template = ezt.Template()
template.parse(get_tmplfile(template_filename).read())
template.generate(open(os.path.join(target, 'index.html'), 'w'), data)
logging.info('Moving tarballs to %s' % os.path.join(target, dirname))
if os.path.exists(os.path.join(target, dirname)):
shutil.rmtree(os.path.join(target, dirname))
shutil.copytree(get_deploydir(args.base_dir), os.path.join(target, dirname))
#----------------------------------------------------------------------
# Clean dist
def clean_dist(args):
'Clean the distribution directory of all but the most recent artifacts.'
regex = re.compile('subversion-(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
if not args.dist_dir:
assert_people()
args.dist_dir = people_dist_dir
logging.info('Cleaning dist dir \'%s\'' % args.dist_dir)
filenames = glob.glob(os.path.join(args.dist_dir, 'subversion-*.tar.gz'))
versions = []
for filename in filenames:
versions.append(Version(filename))
for k, g in itertools.groupby(sorted(versions),
lambda x: (x.major, x.minor)):
releases = list(g)
logging.info("Saving release '%s'", releases[-1])
for r in releases[:-1]:
for filename in glob.glob(os.path.join(args.dist_dir,
'subversion-%s.*' % r)):
logging.info("Removing '%s'" % filename)
os.remove(filename)
#----------------------------------------------------------------------
# Write announcements
def write_news(args):
'Write text for the Subversion website.'
data = { 'date' : datetime.date.today().strftime('%Y%m%d'),
'date_pres' : datetime.date.today().strftime('%Y-%m-%d'),
'version' : str(args.version),
'version_base' : args.version.base,
}
if args.version.is_prerelease():
template_filename = 'rc-news.ezt'
else:
template_filename = 'stable-news.ezt'
template = ezt.Template()
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
def get_sha1info(args):
'Return a list of sha1 info for the release'
sha1s = glob.glob(os.path.join(get_deploydir(args.base_dir), '*.sha1'))
class info(object):
pass
sha1info = []
for s in sha1s:
i = info()
i.filename = os.path.basename(s)[:-5]
i.sha1 = open(s, 'r').read()
sha1info.append(i)
return sha1info
def write_announcement(args):
'Write the release announcement.'
sha1info = get_sha1info(args)
data = { 'version' : args.version,
'sha1info' : sha1info,
'siginfo' : open('getsigs-output', 'r').read(),
'major-minor' : args.version.base[:3],
'major-minor-patch' : args.version.base,
}
if args.version.is_prerelease():
template_filename = 'rc-release-ann.ezt'
else:
template_filename = 'stable-release-ann.ezt'
template = ezt.Template(compress_whitespace = False)
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
#----------------------------------------------------------------------
# Main entry point for argument parsing and handling
def main():
'Parse arguments, and drive the appropriate subcommand.'
# Setup our main parser
parser = argparse.ArgumentParser(
description='Create an Apache Subversion release.')
parser.add_argument('--clean', action='store_true', default=False,
help='Remove any directories previously created by %(prog)s')
parser.add_argument('--verbose', action='store_true', default=False,
help='Increase output verbosity')
parser.add_argument('--base-dir', default=os.getcwd(),
help='''The directory in which to create needed files and
folders. The default is the current working
directory.''')
subparsers = parser.add_subparsers(title='subcommands')
# Setup the parser for the build-env subcommand
subparser = subparsers.add_parser('build-env',
help='''Download release prerequisistes, including autoconf,
libtool, and swig.''')
subparser.set_defaults(func=build_env)
subparser.add_argument('--sf-mirror', default='softlayer',
help='''The mirror to use for downloading files from
SourceForge. If in the EU, you may want to use
'kent' for this value.''')
subparser.add_argument('--use-existing', action='store_true', default=False,
help='''Attempt to use existing build dependencies before
downloading and building a private set.''')
# Setup the parser for the roll subcommand
subparser = subparsers.add_parser('roll',
help='''Create the release artifacts.''')
subparser.set_defaults(func=roll_tarballs)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('revnum', type=int,
help='''The revision number to base the release on.''')
subparser.add_argument('--branch',
help='''The branch to base the release on.''')
# Setup the parser for the post-candidates subcommand
subparser = subparsers.add_parser('post-candidates',
help='''Build the website to host the candidate tarballs.
The default location is somewhere in ~/public_html.
''')
subparser.set_defaults(func=post_candidates)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('revnum', type=int,
help='''The revision number to base the release on.''')
subparser.add_argument('--target',
help='''The full path to the destination.''')
subparser.add_argument('--code-name',
help='''A whimsical name for the release, used only for
naming the download directory.''')
# The clean-dist subcommand
subparser = subparsers.add_parser('clean-dist',
help='''Clean the distribution directory (and mirrors) of
all but the most recent MAJOR.MINOR release. If no
dist-dir is given, this command will assume it is
running on people.apache.org.''')
subparser.set_defaults(func=clean_dist)
subparser.add_argument('--dist-dir',
help='''The directory to clean.''')
# The write-news subcommand
subparser = subparsers.add_parser('write-news',
help='''Output to stdout template text for use in the news
section of the Subversion website.''')
subparser.set_defaults(func=write_news)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser = subparsers.add_parser('write-announcement',
help='''Output to stdout template text for the emailed
release announcement.''')
subparser.set_defaults(func=write_announcement)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
# A meta-target
subparser = subparsers.add_parser('clean',
help='''The same as the '--clean' switch, but as a
separate subcommand.''')
subparser.set_defaults(func=cleanup)
# Parse the arguments
args = parser.parse_args()
# first, process any global operations
if args.clean:
cleanup(args)
# Set up logging
logger = logging.getLogger()
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# Fix up our path so we can use our installed versions
os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
+ os.environ['PATH']
# finally, run the subcommand, and give it the parsed arguments
args.func(args)
if __name__ == '__main__':
main()