coriolis/bootstrap/buildMangrove.py

577 lines
24 KiB
Python
Executable File

#!/usr/bin/env python
import sys
import re
import os
import os.path
import time
import socket
import subprocess
import optparse
class Project:
def __init__ ( self, name, tools, repository ):
self._name = name
self._tools = tools
self._repository = repository
self._actives = []
return
def getName ( self ): return self._name
def getTools ( self ): return self._tools
def getRepository ( self ): return self._repository
def getActives ( self ): return self._actives
def hasTool ( self, tool ): return tool in self._tools
def desactivate ( self ):
self._active = []
return
def activateAll ( self ):
self._actives = self._tools
return
def activate ( self, tools ):
# Build the ordered list.
for tool in self._tools:
if (tool in tools) and not (tool in self._actives):
self._actives += [ tool ]
# Find the tools not part of the project.
rejecteds = []
for tool in tools:
if not (tool in self._tools) and (not tool in rejecteds):
rejecteds += [ tool ]
return rejecteds
class ProjectBuilder:
def __init__ ( self ):
self._projects = []
self._standalones = []
self._svnTag = "x"
self._svnMethod = "svn+ssh://coriolis.soc.lip6.fr"
self._rootDir = os.path.join ( os.environ["HOME"], "mangrove" )
self._quiet = False
self._buildMode = "Release"
self._rmBuild = False
self._doBuild = True
self._noCache = False
self._enableShared = "ON"
self._enableDoc = "OFF"
self._verboseMakefile = "OFF"
self._libMode = "Shared"
self._makeArguments = []
self._environment = os.environ
self._guessOs ()
self._updateSecondary ()
return
def __setattr__ ( self, attribute, value ):
if attribute[0] == "_":
self.__dict__[attribute] = value
return
if attribute == "svnTag": self._svnTag = value
elif attribute == "svnMethod": self._svnMethod = value
elif attribute == "rootDir": self._rootDir = os.path.expanduser(value)
elif attribute == "quiet": self._quiet = value
elif attribute == "buildMode": self._buildMode = value
elif attribute == "rmBuild": self._rmBuild = value
elif attribute == "doBuild": self._doBuild = value
elif attribute == "noCache": self._noCache = value
elif attribute == "enableShared": self._enableShared = value
elif attribute == "enableDoc": self._enableDoc = value
elif attribute == "enableShared": self._enableShared = value
elif attribute == "verboseMakefile": self._verboseMakefile = value
elif attribute == "makeArguments": self._makeArguments = value.split ()
elif attribute == "libMode": self._libMode = value
self._updateSecondary ()
return
def _updateSecondary ( self ):
self._rpmbuildDir = os.path.join ( self._rootDir, "rpmbuild" )
self._tmppathDir = os.path.join ( self._rpmbuildDir, "tmp" )
self._tarballDir = os.path.join ( self._rootDir, "tarball" )
self._archiveDir = os.path.join ( self._tarballDir, "mangrove-1.0.%s" % self._svnTag )
self._sourceDir = os.path.join ( self._rootDir, "src" )
self._osDir = os.path.join ( self._rootDir
, self._osType
, "%s.%s" % (self._buildMode,self._libMode) )
self._buildDir = os.path.join ( self._osDir, "build" )
self._installDir = os.path.join ( self._osDir, "install" )
if self._enableShared == "ON": self._libMode = "Shared"
else: self._libMode = "Static"
self._specFileIn = os.path.join ( self._sourceDir, "goodies", "mangrove.spec.in" )
self._specFile = os.path.join ( self._sourceDir, "goodies", "mangrove.spec" )
self._sourceTarBz2 = "mangrove-1.0.%s.tar.bz2" % self._svnTag
self._binaryTarBz2 = "mangrove-binary-1.0.%s-1.el5_soc.tar.bz2" % self._svnTag
self._distribPatch = os.path.join ( self._sourceDir, "goodies", "mangrove-for-distribution.patch" )
return
def _guessOs ( self ):
self._libSuffix = None
self._osSLSoC5x_64 = re.compile (".*Linux.*el5.*x86_64.*")
self._osSLSoC5x = re.compile (".*Linux.*(el5|2.6.23.13.*SoC).*")
self._osLinux_64 = re.compile (".*Linux.*x86_64.*")
self._osLinux = re.compile (".*Linux.*")
self._osDarwin = re.compile (".*Darwin.*")
uname = subprocess.Popen ( ["uname", "-srm"], stdout=subprocess.PIPE )
lines = uname.stdout.readlines()
if self._osSLSoC5x_64.match(lines[0]):
self._osType = "Linux.SLSoC5x_64"
self._libSuffix = "64"
elif self._osSLSoC5x .match(lines[0]): self._osType = "Linux.SLSoC5x"
elif self._osLinux_64 .match(lines[0]):
self._osType = "Linux.x86_64"
self._libSuffix = "64"
elif self._osLinux .match(lines[0]): self._osType = "Linux.i386"
elif self._osDarwin .match(lines[0]): self._osType = "Darwin"
else:
uname = subprocess.Popen ( ["uname", "-sr"], stdout=subprocess.PIPE )
self._osType = uname.stdout.readlines()[0][:-1]
print "[WARNING] Unrecognized OS: \"%s\"." % lines[0][:-1]
print " (using: \"%s\")" % self._osType
return
def _guessSvnTag ( self, project ):
revisionPattern = re.compile ( r"^Revision:\s*(?P<revision>\d+)" )
projectSvnDir = os.path.join ( self._svnMethod+project.getRepository() )
command = [ "svn", "info", projectSvnDir ]
svnInfo = subprocess.Popen ( command, stdout=subprocess.PIPE )
for line in svnInfo.stdout.readlines():
m = revisionPattern.match ( line )
if m:
self._svnTag = m.group("revision")
print "Latest revision of project %s is %s." % (project.getName(),self._svnTag)
self._updateSecondary ()
return
print "[WARNING] Cannot guess revision for project \"%s\"." % project.getName()
print " (using: \"x\")"
return
def _doSpec ( self ):
fdSpecFileIn = open ( self._specFileIn, "r" )
fdSpecFile = open ( self._specFile , "w" )
for line in fdSpecFileIn.readlines():
stable = False
substituted0 = line
while not stable:
substituted1 = re.sub ( r"@svntag@" , self._svnTag , substituted0 )
substituted1 = re.sub ( r"@mangroveTop@", "/opt/mangrove", substituted1 )
if substituted0 == substituted1: stable = True
else: substituted0 = substituted1
fdSpecFile.write ( substituted0 )
fdSpecFileIn.close ()
fdSpecFile.close ()
return
def _execute ( self, command, error ):
sys.stdout.flush ()
sys.stderr.flush ()
child = subprocess.Popen ( command, env=self._environment, stdout=None )
(pid,status) = os.waitpid ( child.pid, 0 )
status >>= 8
if status != 0:
print "[ERROR] %s (status:%d)." % (error,status)
sys.exit ( status )
return
def _build ( self, tool ):
toolSourceDir = os.path.join ( self._sourceDir, tool )
toolBuildDir = os.path.join ( self._buildDir , tool )
# Supplied directly in the CMakeLists.txt.
#cmakeModules = os.path.join ( self._installDir, "share", "cmake", "Modules" )
if not os.path.isdir(toolSourceDir):
print "[ERROR] Missing tool source directory: \"%s\" (skipped)." % toolSourceDir
return
if self._rmBuild:
print "Removing tool build directory: \"%s\"." % toolBuildDir
command = [ "/bin/rm", "-rf", toolBuildDir ]
self._execute ( command, "Removing tool build directory" )
if not os.path.isdir(toolBuildDir):
print "Creating tool build directory: \"%s\"." % toolBuildDir
os.makedirs ( toolBuildDir )
os.chdir ( toolBuildDir )
command = ["cmake", "-D", "CMAKE_BUILD_TYPE:STRING=%s" % self._buildMode
, "-D", "BUILD_SHARED_LIBS:STRING=%s" % self._enableShared
#, "-D", "CMAKE_MODULE_PATH:STRING=%s" % cmakeModules
, toolSourceDir ]
self._execute ( command, "First CMake failed" )
os.chdir ( toolBuildDir )
if self._noCache:
cmakeCache = os.path.join(toolBuildDir,"CMakeCache.txt")
if os.path.isfile ( cmakeCache ): os.unlink ( cmakeCache )
command = ["cmake", "-D", "CMAKE_BUILD_TYPE:STRING=%s" % self._buildMode
, "-D", "BUILD_SHARED_LIBS:STRING=%s" % self._enableShared
, "-D", "BUILD_DOC:STRING=%s" % self._enableDoc
, "-D", "CMAKE_VERBOSE_MAKEFILE:STRING=%s" % self._verboseMakefile
, "-D", "CMAKE_INSTALL_PREFIX:STRING=%s" % self._installDir
]
if self._libSuffix:
command += [ "-D", "LIB_SUFFIX:STRING=%s" % self._libSuffix ]
command += [ toolSourceDir ]
self._execute ( command, "Second CMake failed" )
if self._doBuild:
command = [ "make" ]
#command += [ "DESTDIR=%s" % self._installDir ]
if tool == "crlcore" and self._enableDoc == "ON":
command += [ "dvi", "safepdf" ]
command += self._makeArguments
print "Make command:", command
sys.stdout.flush ()
self._execute ( command, "Build failed" )
return
def _svnStatus ( self, tool ):
toolSourceDir = os.path.join ( self._sourceDir , tool )
if not os.path.isdir(toolSourceDir):
if not self._quiet:
print "[ERROR] Missing tool source directory: \"%s\" (skipped)." % toolSourceDir
return
os.chdir ( toolSourceDir )
print "Checking SVN status of tool: ", tool
command = [ "svn", "status", "-u", "-q" ]
self._execute ( command, "svn status -u -q" )
print
return
def _svnUpdate ( self, tool ):
toolSourceDir = os.path.join ( self._sourceDir , tool )
if not os.path.isdir(toolSourceDir):
if not self._quiet:
print "[ERROR] Missing tool source directory: \"%s\" (skipped)." % toolSourceDir
return
os.chdir ( toolSourceDir )
print "Doing a SVN update of tool: ", tool
command = [ "svn", "update" ]
self._execute ( command, "svn update" )
print
return
def _svnCheckout ( self, tool ):
project = self.getToolProject ( tool )
if not project:
print "[ERROR] Tool \"%s\" is not part of any project." % tool
print " Cannot guess the SVN repository."
return
if not project.getRepository ():
print "[ERROR] Project \"%s\" isn't associated to a repository." % project.getName()
return
toolSvnTrunkDir = os.path.join ( self._svnMethod+project.getRepository(), tool, "trunk" )
os.chdir ( self._sourceDir )
print "Doing a SVN checkout of tool: ", tool
command = [ "svn", "co", toolSvnTrunkDir, tool ]
self._execute ( command, "svn checkout %s" % tool )
print
return
def _svnExport ( self, tool ):
project = self.getToolProject ( tool )
if not project:
print "[ERROR] Tool \"%s\" is not part of any project." % tool
print " Cannot guess the SVN repository."
return
if not project.getRepository ():
print "[ERROR] Project \"%s\" isn't associated to a repository." % project.getName()
return
toolSvnTrunkDir = os.path.join ( self._svnMethod+project.getRepository(), tool, "trunk" )
if not os.path.isdir ( self._archiveDir ):
os.mkdir ( self._archiveDir )
os.chdir ( self._archiveDir )
toolExportDir = os.path.join ( self._archiveDir, tool )
if os.path.isdir ( toolExportDir ):
print "Removing tool export (tarball) directory: \"%s\"." % toolExportDir
command = [ "/bin/rm", "-r", toolExportDir ]
self._execute ( command, "Removing tool export (tarball) directory" )
print "Doing a SVN export of tool: ", tool
command = [ "svn", "export", toolSvnTrunkDir, toolExportDir ]
if self._svnTag != "x":
command += [ "--revision", self._svnTag ]
self._execute ( command, "svn export %s" % toolExportDir )
print
return
def getProject ( self, name ):
for project in self._projects:
if project.getName() == name:
return project
return None
def getToolProject ( self, name ):
for project in self._projects:
if project.hasTool(name):
return project
return None
def register ( self, project ):
for registered in self._projects:
if registered.getName() == project.getName():
print "[ERROR] Project \"%s\" is already registered (ignored)."
return
self._projects += [ project ]
return
def _setEnvironment ( self, variable, userVariable ):
if not self._environment.has_key(variable):
self._environment[ variable ] = self._installDir
#if not self._environment.has_key(userVariable):
# self._environment[ userVariable ] = self._installDir
if not self._quiet:
print "Setting %s = \"%s\"." % (variable ,self._environment[variable])
if self._environment.has_key(userVariable):
print "Transmitting %s = \"%s\"." % (userVariable,self._environment[userVariable])
return
def _commandTemplate ( self, tools, projects, command ):
# Set or guess the various projects TOP environment variables.
self._setEnvironment ( "BOOTSTRAP_TOP", "BOOTSTRAP_USER_TOP" )
self._setEnvironment ( "CORIOLIS_TOP" , "CORIOLIS_USER_TOP" )
for project in self._projects:
topVariable = "%s_TOP" % project.getName().upper()
topUserVariable = "%s_USER_TOP" % project.getName().upper()
self._setEnvironment ( topVariable, topUserVariable )
if tools:
# Checks if the requested tools are in the various projects.
self._standalones = tools
for project in self._projects:
self._standalones = project.activate ( self._standalones )
for tool in self._standalones:
print "[WARNING] Tool \"%s\" is not part of any project." % tool
if projects:
for projectName in projects:
project = self.getProject ( projectName )
if not project:
print "[ERROR] No project of name \"%s\"." % projectName
sys.exit ( 1 )
project.activateAll()
if not tools and not projects:
for project in self._projects:
project.activateAll ()
for project in self._projects:
for tool in project.getActives():
print "\nProcessing tool: \"%s\"." % tool
getattr(self,command) ( tool )
for tool in self._standalones:
print "\nProcessing tool: \"%s\"." % tool
getattr(self,command) ( tool )
return
def build ( self, tools, projects ):
self._commandTemplate ( tools, projects, "_build" )
return
def svnStatus ( self, tools, projects ):
self._commandTemplate ( tools, projects, "_svnStatus" )
return
def svnUpdate ( self, tools, projects ):
self._commandTemplate ( tools, projects, "_svnUpdate" )
return
def svnCheckout ( self, tools, projects ):
self._commandTemplate ( tools, projects, "_svnCheckout" )
return
def svnExport ( self, tools, projects ):
self._commandTemplate ( tools, projects, "_svnExport" )
return
def tarball ( self, tools, projects ):
if self._svnTag == "x":
self._guessSvnTag ( self.getProject(projects[0]) )
self._doSpec ()
if os.path.isdir(self._tarballDir):
print "Removing previous tarball directory: \"%s\"." % self._tarballDir
command = [ "/bin/rm", "-r", self._tarballDir ]
self._execute ( command, "Removing top export (tarball) directory" )
print "Creating tarball directory: \"%s\"." % self._tarballDir
os.makedirs ( self._tarballDir )
self.svnExport ( tools, projects )
removeds = [ os.path.join("vlsisapd","openChams")
, os.path.join("vlsisapd","dtr")
]
# Remove unpublisheds (yet) tools/files.
for item in removeds:
command = [ "/bin/rm", "-r", os.path.join(self._archiveDir,item) ]
self._execute ( command, "rm of %s failed" % item)
os.chdir ( self._archiveDir )
command = [ "/usr/bin/patch", "--remove-empty-files"
, "--no-backup-if-mismatch"
, "-p0", "-i", self._distribPatch ]
self._execute ( command, "patch for distribution command failed" )
os.chdir ( self._tarballDir )
command = [ "/bin/tar", "jcvf", self._sourceTarBz2, os.path.basename(self._archiveDir) ]
self._execute ( command, "tar command failed" )
print "Cleanup SVN export tarball archive directory: \"%s\"." % self._archiveDir
command = [ "/bin/rm", "-r", self._archiveDir ]
self._execute ( command, "Removing archive export (tarball) directory" )
return
def doRpm ( self, tools, projects ):
self.tarball ( tools, projects )
for rpmDir in [ "SOURCES", "SPECS", "BUILD", "tmp"
, "SRPMS", "RPMS/i386", "RPMS/i686", "RPMS/x86_64" ]:
rpmFullDir = os.path.join ( self._rpmbuildDir, rpmDir )
if not os.path.isdir(rpmFullDir):
os.makedirs ( rpmFullDir )
rpmSpecFile = os.path.join ( self._rpmbuildDir, "SPECS" , "mangrove.spec" )
rpmSourceFile = os.path.join ( self._rpmbuildDir, "SOURCES", self._sourceTarBz2 )
sourceFile = os.path.join ( self._tarballDir, self._sourceTarBz2 )
if os.path.isfile ( rpmSpecFile ):
os.unlink ( rpmSpecFile )
os.symlink ( self._specFile , rpmSpecFile )
if not os.path.islink ( rpmSourceFile ):
os.symlink ( sourceFile, rpmSourceFile )
os.chdir ( self._rpmbuildDir )
command = [ "/usr/bin/rpmbuild"
, "--define", "_topdir %s" % self._rpmbuildDir
, "--define", "_tmppath %s" % self._tmppathDir
, "--define", "_enable_debug_packages 0"
, "-ba", "--with", "binarytar", rpmSpecFile ]
self._execute ( command, "Rebuild rpm packages" )
return
if __name__ == "__main__":
mangrove = Project ( name = "mangrove"
, tools =[ "mangrove" ]
, repository="/users/outil/mangrove/svn"
)
caiman = Project ( name = "caiman"
, tools =[ "caiman" ]
, repository="/users/outil/caiman/svn"
)
parser = optparse.OptionParser ()
# Build relateds.
parser.add_option ( "-q", "--quiet" , action="store_true" , dest="quiet" )
parser.add_option ( "-r", "--release" , action="store_true" , dest="release" )
parser.add_option ( "-d", "--debug" , action="store_true" , dest="debug" )
parser.add_option ( "-s", "--static" , action="store_true" , dest="static" )
parser.add_option ( "--doc" , action="store_true" , dest="doc" )
parser.add_option ( "-v", "--verbose" , action="store_true" , dest="verboseMakefile" )
parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" )
parser.add_option ( "--no-build" , action="store_true" , dest="noBuild" )
parser.add_option ( "--no-cache" , action="store_true" , dest="noCache" )
parser.add_option ( "--rm-build" , action="store_true" , dest="rmBuild" )
parser.add_option ( "--svn-tag" , action="store" , type="string", dest="svnTag" )
parser.add_option ( "--svn-method" , action="store" , type="string", dest="svnMethod")
parser.add_option ( "--make" , action="store" , type="string", dest="makeArguments")
parser.add_option ( "--project" , action="append" , type="string", dest="projects" )
parser.add_option ( "-t", "--tool" , action="append" , type="string", dest="tools" )
# SVN repository relateds.
parser.add_option ( "--svn-status" , action="store_true", dest="svnStatus" )
parser.add_option ( "--svn-update" , action="store_true", dest="svnUpdate" )
parser.add_option ( "--svn-checkout" , action="store_true", dest="svnCheckout" )
# Miscellaneous.
parser.add_option ( "--tarball" , action="store_true", dest="tarball" )
parser.add_option ( "--do-rpm" , action="store_true", dest="doRpm" )
( options, args ) = parser.parse_args ()
builder = ProjectBuilder ()
builder.register ( mangrove )
builder.register ( caiman )
if options.quiet: builder.quiet = True
if options.release: builder.buildMode = "Release"
if options.debug: builder.buildMode = "Debug"
if options.static: builder.enableShared = "OFF"
if options.doc: builder.enableDoc = "ON"
if options.verboseMakefile: builder.verboseMakefile = "ON"
if options.rootDir: builder.rootDir = options.rootDir
if options.noBuild: builder.doBuild = False
if options.noCache: builder.noCache = True
if options.rmBuild: builder.rmBuild = True
if options.makeArguments: builder.makeArguments = options.makeArguments
if options.svnMethod: builder.svnMethod = options.svnMethod
if options.svnTag: builder.svnTag = options.svnTag
if options.svnStatus: builder.svnStatus ( tools=options.tools, projects=options.projects )
elif options.svnUpdate: builder.svnUpdate ( tools=options.tools, projects=options.projects )
elif options.svnCheckout: builder.svnCheckout ( tools=options.tools, projects=options.projects )
elif options.tarball: builder.tarball ( tools=options.tools, projects=options.projects )
elif options.doRpm: builder.doRpm ( tools=options.tools, projects=options.projects )
else: builder.build ( tools=options.tools, projects=options.projects )
sys.exit ( 0 )