#!/usr/bin/python
#
# rmerge2 is a robust version of "emerge -e" which supports
# resumption of builds. It was inspired by the original rmerge
# bash script by Dan Doel <dolio@zoomtown.com> which did basically
# the same thing (but not quite).
#
# Copyright (C) 2002 Bardur Arantsson <bardur@imada.sdu.dk>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# See http://www.gnu.org/copyleft/gpl.html for further information.
#
VERSION = "0.9.7"
# system configuration (there should be no need to change these)
STATEDIR="/var/lib/rmerge2"
EMERGE="/usr/bin/emerge"
# default option values (DO NOT CHANGE UNLESS YOU KNOW WHAT YOU ARE DOING)
opt_empty = 0
opt_pretend = 0
opt_skip = 0
opt_resume = 1
opt_force = 0
opt_quiet = 0
opt_statefile = None
# This is an option which is purely for testing.
# It causes merge_pkg to not do anything dangerous,
# and in addition randomly fail emerge runs. Useful
# for testing if resumption of builds works properly.
opt_test = 0
#
# Start of the actual code
#
import getopt
import os
import md5
import re
import whrandom
from popen2 import *
import cPickle
pickle = cPickle
import portage
# read portage configuration to avoid relying on
# the environment
portage_config = portage.config()
# fatal error
def fatal(message):
os.sys.stderr.write("rmerge2: %s\n" % (message,))
os.sys.stderr.flush()
os.sys.exit(1)
class MergeError(Exception):
pass
class State:
"""The state object represents the state of the
compilation. It is pickled/unpickled to preserve
state information across invocations."""
def __init__(self, todo, _failed = [], skipped = []):
"""Create an initially empty state."""
self.todo = todo
self._failed = _failed
self.skipped = skipped
def skip(self, n):
"""Skip n packages."""
self.skipped.extend(self.todo[:n])
self.todo = self.todo[n:]
def any_left(self):
"""Are there any packages left to do?"""
return len(self.todo) != 0
def get_next(self):
"""Return the next package from the to do list."""
return self.todo[0]
def get_failed(self):
"""Return the list of failed packages."""
return self._failed
def get_todo(self):
"""Return the list of packages left to do."""
return self.todo
def done(self):
"""Mark the first package as done."""
del self.todo[0]
def failed(self):
"""The current package failed (completely)."""
self._failed.append(self.todo[0])
del self.todo[0]
def dump(self, fname):
"""Write state to file named fname in a crashproof way."""
f = open("%s.tmp" % fname,"w")
try:
pickle.dump(self, f)
os.rename("%s.tmp" % fname, fname)
except:
os.unlink("%s.tmp" % fname) # don't need anymore
raise
def load_state(fname):
return pickle.load(open(fname,"r"))
def split_pkgname(pkgname):
"""Chops off the version number and returns a tuple
of (basename, version) for the given package name."""
m = re.search("(.*?)(-[.0-9a-zA-Z]+(-r[0-9]+)?)$",pkgname)
if m:
pkg_ver = m.group(2)[1:]
pkgname = m.group(1)
else:
pkg_ver = None
return (pkgname,pkg_ver)
def get_state_filename(pkgnames, create=1):
"""Compute the status directory name for the
given list of packages."""
if opt_statefile:
return opt_statefile
else:
# Generate a state file name if none is given
pkgnames = pkgnames[:] # sorting is destructive
pkgnames.sort() # we want to ignore ordering
# hash each of the package names
h = md5.new()
for pkgname in pkgnames:
h.update(pkgname.strip())
return os.path.join(STATEDIR,h.hexdigest())
def get_prereqs(pkgnames):
"""Compute prerequisite packages for the given packages and
return the list. The list contains full package names (category,
package name and version number). If opt_empty is NOT set,
only packages which were originally listed are returned. However,
this is still useful for determining the proper ordering of those
packages."""
if not opt_empty:
# build a dictionary of package names for quick lookup.
# this is used to determine if we should merge a given
# package. this is a very crude way of doing this, but
# portage doesn't allow us to find out (inexpensively)
# whether a given package is part of another which is
# really what we need if pkgnames contains virtual
# packages. Oh, well.
pkgnames_dict = {}
for pkgname in pkgnames:
pkgnames_dict[pkgname] = None # value doesn't matter
# If any input packages contain version numbers, we should
# repesct them. Do this by adding a '=' in front so that
# emerge knows that we want a specific version
qpkgnames = pkgnames[:] # copy to avoid overwriting
for i in xrange(0,len(pkgnames)):
pkgname = pkgnames[i]
if split_pkgname(pkgname)[1]!=None: # fixed version?
qpkgnames[i] = ("=%s" % pkgname)
# Evil hack to retrieve list of package names (and versions)
r = re.compile("\\[ebuild......\\]\\s+([^\\s]+)")
# Use emerge to get the canonical package list.
# NOTE: If "-e" is not present, emerge will not
# compute interdependencies between packages properly
emerge = Popen3([EMERGE,"-p","-e"] + qpkgnames)
prereqs = [] # current list of prerequisite packages
lines = emerge.fromchild.readlines()
for line in lines:
m = r.match(line)
if m:
if not opt_empty: # only want requested packages to be merged?
# full package name(s)
fpkgname = m.group(1)
fpkgname_nocat = fpkgname.split("/")[1] # without category
# get base name of package
pkgname = split_pkgname(fpkgname)[0]
pkgname_nocat = pkgname.split("/")[1] # without category
if not fpkgname in pkgnames_dict and \
not fpkgname_nocat in pkgnames_dict and \
not pkgname in pkgnames_dict and \
not pkgname_nocat in pkgnames_dict:
continue # move on to next package
# append package to list of prereqs
prereqs.append(m.group(1))
# status?
status = emerge.wait()
if not (os.WIFEXITED(status) and os.WEXITSTATUS(status)==0):
os.sys.stderr.write("emerge output:\n\n")
os.sys.stderr.writelines(lines)
fatal("emerge returned bad exit status. cannot continue.")
return prereqs
def merge_pkg(pkgname):
"""This function merges a single package. The package name
must be given as a full package name, i.e. with a category
and version number."""
if opt_test:
if whrandom.choice([0,1,1]): # "fail" random packages
pass # fake successful merge
else:
raise MergeError("Fake merge error")
else:
print "Merging %s..." % pkgname
os.sys.stdout.flush()
# This is a bit more complicated than using Popen,
# but we really don't have any choice if we want
# to "forward" the output
pid = os.fork()
if pid==0: # child?
if opt_quiet:
try: # pipe to the great void
devnull = open("/dev/null","w")
os.dup2(devnull.fileno(),0)
os.dup2(devnull.fileno(),1)
devnull.close()
except os.error,e:
pass # not that important
os.execv(EMERGE,[EMERGE,"--oneshot","=%s" % pkgname])
fatal("execv() failed: cannot continue")
# wait for emerge to finish
try:
status = os.waitpid(pid,0)[1]
except os.error,e:
status = 0 # child finished already?
# abnormal exit?
if not (os.WIFEXITED(status) and os.WEXITSTATUS(status)==0):
raise MergeError("Failed to merge %s..." % pkgname)
def merge_pkgs(pkgnames):
"""Merge given packages using emerge. The input list of package names
contains package names *without* version numbers. This function
maintains state between each merge to avoid having to remerge
previously merged packages."""
# state file
state_filename = get_state_filename(pkgnames)
# start environment (note that the current environment
# overrides Portage variables as usual).
start_env = { "CFLAGS" : os.environ.get("CFLAGS",
portage_config["CFLAGS"]),
"CXXFLAGS" : os.environ.get("CXXFLAGS",
portage_config["CXXFLAGS"]) }
# environment to use when a package fails to merge
BCFLAGS = os.environ.get("BCFLAGS",None)
if BCFLAGS==None and portage_config.has_key("BCFLAGS"):
BCFLAGS=portage_config["BCFLAGS"]
BCXXFLAGS = os.environ.get("BCXXFLAGS",None)
if BCXXFLAGS==None and portage_config.has_key("BCXXFLAGS"):
BCXXFLAGS=portage_config["BCXXFLAGS"]
# backup environment to use when emerge fails
if BCFLAGS==None and BCXXFLAGS==None:
backup_env = None
elif BCFLAGS!=None and BCXXFLAGS!=None:
backup_env = { "CFLAGS": BCFLAGS, "CXXFLAGS": BCXXFLAGS }
else:
raise MergeError("Must either set both BCFLAGS and BCXXFLAGS or neither.")
# if we want to continue and a saved state exists
if opt_resume and os.path.exists(state_filename):
state = load_state(state_filename)
else:
state = State(get_prereqs(pkgnames))
# skip opt_skip packages
state.skip(opt_skip)
# if we're just pretending, display
# a list of packages that would be merged
if opt_pretend:
print "I would merge the following packages (in order):\n\n"
print "\n".join(state.get_todo())
else:
# merge each package in turn
while state.any_left():
# get the name of the first package left to do
pkgname = state.get_next()
tries=0 # how many times have we tried this package?
while (tries <= 1):
try:
# merge and restore original flags
try:
merge_pkg(pkgname)
finally:
os.environ.update(start_env)
# update the list of packages left to do.
state.done()
break # stop trying this package
except MergeError,e:
# Don't retry unless there is a backup_env
if (tries == 0) and backup_env:
# Let's retry with different flags
tries += 1
os.environ.update(backup_env)
else:
if opt_force:
state.failed() # register failure and continue
tries += 1
else:
raise # give up
# dump state each time through the loop
state.dump(state_filename)
# list the packages that failed to merge
if len(state.get_failed())>0:
print
print "The following packages failed to merge:"
print
print "\n".join(state.get_failed())
#
# Main program
#
try:
# parse options
opts,args = getopt.getopt(os.sys.argv[1:],"epsfqV",
["emptytree","pretend","skip=","scratch",
"force","quiet","help","statefile=","version"])
for (opt,val) in opts:
if opt in ["-e","--emptytree"]:
opt_empty = 1
elif opt in ["-p","--pretend"]:
opt_pretend = 1
elif opt in ["--skip"]:
opt_skip = int(val)
elif opt in ["-s","--scratch"]:
opt_resume = 0
elif opt in ["-f","--force"]:
opt_force = 1
elif opt in ["-q","--quiet"]:
opt_quiet = 1
elif opt in ["-h","--help"]:
raise getopt.error("",None)
elif opt in ["-V","--version"]:
print "rmerge2 %s" % (VERSION,)
os.sys.exit(0)
elif opt in ["--statefile"]:
opt_statefile = val
# require explicit package names
if len(args)==0:
raise getopt.error("Please specify a package name.","")
# check that CFLAGS and CXXFLAGS are set
if not os.environ.has_key("CFLAGS"):
fatal("must set CFLAGS in /etc/make.conf or environment")
if not os.environ.has_key("CXXFLAGS"):
fatal("must set CXXFLAGS in /etc/make.conf or environment")
# anything other than a "pretend" run requires
# root privileges.
if (not opt_pretend) and (os.getuid() != 0):
fatal("root privileges required")
# merge packages
merge_pkgs(args)
except getopt.error,e:
# any reason for exception or do we just want
# to display usage?
if e:
print "rmerge2: %s" % (e,)
print
# print the standard usage
print """Usage:
rmerge2 [options] package1 ...
Synopsis:
Robustly merges all the given packages using emerge.
Options:
-h, --help
Prints this message.
-e, --emptytree
Pretend that no packages (besides glibc) are installed.
This will cause all the packages the given package(s)
depend on to be remerged. Useful for rebuilding the
entire distribution from scratch.
-p, --pretend
Do not merge any packages, just display which packages
would be merged.
--skip=N
Skip the given number of packages. Useful if you are
doing a manual merge of a lot of packages and just want
to skip a package that is giving you trouble.
-s, --scratch
Start a new merge of the given packages from scratch.
Forces rmerge2 to ignore any previous progress information.
Useful if you want to be absolutely certain that
ALL packages are remerged, even after having merged some
of them previously. NOTE: This only applies to the given
set of packages, you could (in theory at least) have
other package batches which could be continued independently
of this run.
-f, --force
Keep running even after packages fail to merge. The
list of packages which failed to merge is displayed
at the end of a complete run. Very useful for a
completely automated rebuild.
NOTE: If a package fails to merge with the normal
CFLAGS/CXXFLAGS the rmerge2 script will retry merging
using BCFLAGS/BCXXFLAGS from your environment or porage
configuration files.
-q, --quiet
Do not display output from emerge runs. This makes the
run almost completely noise-free, but errors from portage
will NOT be displayed. Also, watching these compiles can
be VERY tedious if the individual packages take a long time
to merge.
--statefile
Save the program state in this file instead of the standard
location (which is determined from package names and versions
given on the command line).
-V, --version
Show program version.
Examples:
Remerge all installed packages (except glibc):
rmerge2 -f -s -e world
Remerge given packages in the appropriate order
(as determined by their dependency lists):
rmerge2 package1 package2 ...
Note that this does not work properly on "virtual"
packages (e.g. "kde") because of the way portage
currently works.
Remerge given package(s) and all the packages it
depend on:
rmerge2 -e package1 [package2 ...]
"""