# -*- mode:Python -*- # # This file is part of the Coriolis Software. # Copyright (c) UPMC/LIP6 2008-2012, All Rights Reserved # # +-----------------------------------------------------------------+ # | C O R I O L I S | # | C o r i o l i s / C h a m s B u i l d e r | # | | # | Author : Jean-Paul Chaput | # | E-mail : Jean-Paul.Chaput@asim.lip6.fr | # | =============================================================== | # | Python : "./builder/Builder.py" | # +-----------------------------------------------------------------+ import sys import re import os import os.path import datetime import subprocess from . import ErrorMessage from Project import Project from Configuration import Configuration class Builder: def __init__ ( self ): self._conf = Configuration() self._quiet = False self._rmBuild = False self._doBuild = True self._noCache = False self._enableShared = "ON" self._enableDoc = "OFF" self._checkDatabase = "OFF" self._checkDeterminism = "OFF" self._verboseMakefile = "OFF" self._makeArguments = [] self._environment = os.environ return def __setattr__ ( self, attribute, value ): if attribute[0] == "_": self.__dict__[attribute] = value return if attribute in self._conf.getAllIds(): setattr( self._conf, attribute, value ) if attribute == "quiet": self._quiet = value elif attribute == "rmBuild": self._rmBuild = value elif attribute == "doBuild": self._doBuild = value elif attribute == "noCache": self._noCache = value elif attribute == "enableDoc": self._enableDoc = value elif attribute == "enableShared": self._enableShared = value elif attribute == "checkDatabase": self._checkDatabase = value elif attribute == "checkDeterminism": self._checkDeterminism = value elif attribute == "verboseMakefile": self._verboseMakefile = value elif attribute == "makeArguments": self._makeArguments = value.split () return def __getattr__ ( self, attribute ): if attribute[0] != "_": if attribute == 'conf': return self._conf if attribute in self._conf.getAllIds(): return getattr( self._conf, attribute ) if not self.__dict__.has_key(attribute): raise ErrorMessage( 1, 'Builder has no attribute <%s>.'%attribute ) return self.__dict__[attribute] def _guessSvnTag ( self, project ): revisionPattern = re.compile ( r"^Revision:\s*(?P\d+)" ) projectSvnDir = os.path.join ( self.svnMethod+project.getRepository() ) command = [ "svn", "info", projectSvnDir ] svnInfo = subprocess.Popen ( command, stdout=subprocess.PIPE ) for line in svnInfo.stdout.readlines(): m = revisionPattern.match ( line ) if m: self.svnTag = m.group("revision") print "Latest revision of project %s is %s." % (project.getName(),self.svnTag) return print "[WARNING] Cannot guess revision for project \"%s\"." % project.getName() print " (using: \"x\")" return def _configure ( self, fileIn, file ): fdFileIn = open ( fileIn, "r" ) fdFile = open ( file , "w" ) for line in fdFileIn.readlines(): stable = False substituted0 = line while not stable: substituted1 = re.sub ( r"@svntag@" , self.svnTag, substituted0 ) substituted1 = re.sub ( r"@coriolisTop@", "/opt/coriolis2" , substituted1 ) if substituted0 == substituted1: stable = True else: substituted0 = substituted1 fdFile.write ( substituted0 ) fdFileIn.close () fdFile.close () return def _doSpec ( self ): self._configure ( self.specFileIn, self.specFile ) return def _doDebChangelog ( self ): self._configure ( self.debChangelogIn, self.debChangelog ) return def _execute ( self, command, error ): sys.stdout.flush () sys.stderr.flush () child = subprocess.Popen ( command, env=self._environment, stdout=None ) (pid,status) = os.waitpid ( child.pid, 0 ) status >>= 8 if status != 0: ErrorMessage( status, "%s (status:%d)."%(error,status) ).terminate() return def _enableTool ( self, tool ): return def _build ( self, tool ): toolSourceDir = os.path.join ( self.sourceDir, tool ) toolBuildDir = os.path.join ( self.buildDir , tool ) # Supplied directly in the CMakeLists.txt. #cmakeModules = os.path.join ( self.installDir, "share", "cmake", "Modules" ) if not os.path.isdir(toolSourceDir): print ErrorMessage( 0, "Missing tool source directory: \"%s\" (skipped)."%toolSourceDir ) return if self._rmBuild: print "Removing tool build directory: \"%s\"." % toolBuildDir command = [ "/bin/rm", "-rf", toolBuildDir ] self._execute ( command, "Removing tool build directory" ) if not os.path.isdir(toolBuildDir): print "Creating tool build directory: \"%s\"." % toolBuildDir os.makedirs ( toolBuildDir ) os.chdir ( toolBuildDir ) command = ["cmake", "-D", "CMAKE_BUILD_TYPE:STRING=%s" % self.buildMode , "-D", "BUILD_SHARED_LIBS:STRING=%s" % self.enableShared #, "-D", "CMAKE_MODULE_PATH:STRING=%s" % cmakeModules , toolSourceDir ] self._execute ( command, "First CMake failed" ) os.chdir ( toolBuildDir ) if self._noCache: cmakeCache = os.path.join(toolBuildDir,"CMakeCache.txt") if os.path.isfile ( cmakeCache ): os.unlink ( cmakeCache ) command = ["cmake", "-D", "CMAKE_BUILD_TYPE:STRING=%s" % self.buildMode , "-D", "BUILD_SHARED_LIBS:STRING=%s" % self.enableShared , "-D", "BUILD_DOC:STRING=%s" % self._enableDoc , "-D", "CHECK_DATABASE:STRING=%s" % self._checkDatabase , "-D", "CHECK_DETERMINISM:STRING=%s" % self._checkDeterminism , "-D", "CMAKE_VERBOSE_MAKEFILE:STRING=%s" % self._verboseMakefile , "-D", "CMAKE_INSTALL_PREFIX:STRING=%s" % self.installDir ] if self.libSuffix: command += [ "-D", "LIB_SUFFIX:STRING=%s" % self.libSuffix ] command += [ toolSourceDir ] self._execute ( command, "Second CMake failed" ) if self._doBuild: command = [ "make" ] #command += [ "DESTDIR=%s" % self.installDir ] if self._enableDoc == "ON": #if tool == "crlcore" or tool == "stratus1": if tool == "stratus1": command += [ "dvi", "safepdf", "html" ] command += self._makeArguments print "Make command:", command sys.stdout.flush () self._execute ( command, "Build failed" ) return def _svnStatus ( self, tool ): toolSourceDir = os.path.join ( self.sourceDir , tool ) if not os.path.isdir(toolSourceDir): if not self._quiet: print ErrorMessage( 0, "Missing tool source directory: \"%s\" (skipped)."%toolSourceDir ) return os.chdir ( toolSourceDir ) print "Checking SVN status of tool: ", tool command = [ "svn", "status", "-u", "-q" ] self._execute ( command, "svn status -u -q" ) print return def _svnDiff ( self, tool ): toolSourceDir = os.path.join ( self.sourceDir , tool ) if not os.path.isdir(toolSourceDir): if not self._quiet: print ErrorMessage( 0, "Missing tool source directory: \"%s\" (skipped)."%toolSourceDir ) return os.chdir ( toolSourceDir ) print "Doing a SVN diff of tool: ", tool command = [ "svn", "diff" ] if self.svnTag != "x": command += [ "--revision", self.svnTag ] self._execute ( command, "svn diff %s" % tool ) print return def _svnUpdate ( self, tool ): toolSourceDir = os.path.join ( self.sourceDir , tool ) if not os.path.isdir(toolSourceDir): if not self._quiet: print ErrorMessage( 0, "Missing tool source directory: \"%s\" (skipped)."%toolSourceDir) return os.chdir ( toolSourceDir ) print "Doing a SVN update of tool: ", tool command = [ "svn", "update" ] self._execute ( command, "svn update" ) print return def _svnCheckout ( self, tool ): project = self._conf.getToolProject ( tool ) if not project: print ErrorMessage( 0, "Tool \"%s\" is not part of any project."%tool ,"Cannot guess the SVN repository." ) return if not project.getRepository (): print ErrorMessage( 0, "Project \"%s\" isn't associated to a repository."%project.getName() ) return if not os.path.isdir(self.sourceDir): print ErrorMessage( 0, "Source directory <%s> doesn't exists. Creating."%self.sourceDir ) os.makedirs( self.sourceDir ) toolSvnTrunkDir = os.path.join ( self.svnMethod+project.getRepository(), tool, "trunk" ) os.chdir ( self.sourceDir ) print "Doing a SVN checkout of tool: ", tool command = [ "svn", "co", toolSvnTrunkDir, tool ] if self.svnTag != "x": command += [ "--revision", self.svnTag ] self._execute ( command, "svn checkout %s" % tool ) print return def _svnExport ( self, tool ): project = self._conf.getToolProject ( tool ) if not project: print ErrorMessage( 0, "Tool \"%s\" is not part of any project."%tool , "Cannot guess the SVN repository.") return if not project.getRepository (): print ErrorMessage( 0, "Project \"%s\" isn't associated to a repository."%project.getName() ) return toolSvnTrunkDir = os.path.join ( self.svnMethod+project.getRepository(), tool, "trunk" ) if not os.path.isdir ( self.archiveDir ): os.mkdir ( self.archiveDir ) os.chdir ( self.archiveDir ) toolExportDir = os.path.join ( self.archiveDir, tool ) if os.path.isdir ( toolExportDir ): print "Removing tool export (tarball) directory: \"%s\"." % toolExportDir command = [ "/bin/rm", "-r", toolExportDir ] self._execute ( command, "Removing tool export (tarball) directory" ) print "Doing a SVN export of tool: ", tool command = [ "svn", "export", toolSvnTrunkDir, toolExportDir ] if self.svnTag != "x": command += [ "--revision", self.svnTag ] self._execute ( command, "svn export %s" % toolExportDir ) print return def _setEnvironment ( self, systemVariable, userVariable ): if not self._environment.has_key(systemVariable) or self._environment[systemVariable] == "": if not self._environment.has_key(userVariable) or self._environment[userVariable] == "" : self._environment[ systemVariable ] = self.installDir print "[WARNING] Neither <%s> nor <%s> environment variables are sets." \ % (systemVariable,userVariable) print " Setting <%s> to <%s>." % (systemVariable,self.installDir) else: self._environment[ systemVariable ] = self._environment[ userVariable ] if not self._quiet: print "Setting <%s> to <%s>." % (systemVariable,self._environment[systemVariable]) if self._environment.has_key(userVariable): print "Transmitting <%s> as <%s>." % (userVariable,self._environment[userVariable]) return def _commandTemplate ( self, tools, projects, command ): # Set or guess the various projects TOP environment variables. for project in self.projects: topVariable = "%s_TOP" % project.getName().upper() topUserVariable = "%s_USER_TOP" % project.getName().upper() self._setEnvironment ( topVariable, topUserVariable ) if tools: # Checks if the requested tools are in the various projects. self.standalones = tools for project in self.projects: self.standalones = project.activate ( self.standalones ) for tool in self.standalones: print "[WARNING] Tool \"%s\" is not part of any project." % tool if projects: for projectName in projects: project = self.getProject ( projectName ) if not project: ErrorMessage( 1, "No project of name \"%s\"."%projectName ).terminate() project.activateAll() if not tools and not projects: for project in self.projects: project.activateAll () for project in self.projects: for tool in project.getActives(): print "\nProcessing tool: \"%s\"." % tool getattr(self,command) ( tool ) for tool in self.standalones: print "\nProcessing tool: \"%s\"." % tool getattr(self,command) ( tool ) return def enable ( self, tools, projects ): self._commandTemplate ( tools, projects, "_enableTool" ) return def enabledTools ( self ): tools = [] for project in self.projects: tools += project.getActives() return tools def build ( self, tools, projects ): self._commandTemplate ( tools, projects, "_build" ) return def svnStatus ( self, tools, projects ): self._commandTemplate ( tools, projects, "_svnStatus" ) return def svnUpdate ( self, tools, projects ): self._commandTemplate ( tools, projects, "_svnUpdate" ) return def svnCheckout ( self, tools, projects ): self._commandTemplate ( tools, projects, "_svnCheckout" ) return def svnDiff ( self, tools, projects ): self._commandTemplate ( tools, projects, "_svnDiff" ) return def svnExport ( self, tools, projects ): self._commandTemplate ( tools, projects, "_svnExport" ) return def svnTarball ( self, tools, projects ): if self.svnTag == "x": self._guessSvnTag ( self.getProject(projects[0]) ) self._doSpec () self._doDebChangelog () if os.path.isdir(self.tarballDir): print "Removing previous tarball directory: \"%s\"." % self.tarballDir command = [ "/bin/rm", "-rf", self.tarballDir ] self._execute ( command, "Removing top export (tarball) directory" ) print "Creating tarball directory: \"%s\"." % self.tarballDir os.makedirs ( self.tarballDir ) self.svnExport ( tools, projects ) # Remove unpublisheds (yet) tools/files. for item in self.packageExcludes: command = [ "/bin/rm", "-rf", os.path.join(self.archiveDir,item) ] self._execute ( command, "rm of %s failed" % item) # Adds files neededs only for packaging purpose. command = [ "/bin/cp", "-r", os.path.join(self.sourceDir ,"bootstrap","Makefile.package") , os.path.join(self.archiveDir,"Makefile") ] self._execute ( command, "copy of %s failed" % "boostrap/Makefile.package") command = [ "/bin/cp", self.specFile, self.archiveDir ] self._execute ( command, "Copying RPM spec file" ) command = [ "/bin/cp", "-r", self.debianDir, self.archiveDir ] self._execute ( command, "Copying Debian/Ubuntu package control files" ) os.chdir ( self.archiveDir ) #command = [ "/usr/bin/patch", "--remove-empty-files" # , "--no-backup-if-mismatch" # , "-p0", "-i", self.distribPatch ] #self._execute ( command, "patch for distribution command failed" ) os.chdir ( self.tarballDir ) command = [ "/bin/tar" , "--exclude-backups" , "--exclude-vcs" , "-jcvf", self.sourceTarBz2, os.path.basename(self.archiveDir) ] self._execute ( command, "tar command failed" ) print "Cleanup SVN export tarball archive directory: \"%s\"." % self.archiveDir command = [ "/bin/rm", "-rf", self.archiveDir ] self._execute ( command, "Removing archive export (tarball) directory" ) return def userTarball ( self, tools, projects ): self.enable( tools, projects ) userSourceTarBz2 = os.path.join ( self.tarballDir , datetime.date.today().strftime('%s-%s-%%Y%%m%%d.tar.bz2'% (self.packageName ,self.packageVersion)) ) excludes = [] for exclude in self.packageExcludes: excludes += [ '--exclude='+exclude ] os.chdir ( self.sourceDir ) command = [ "/bin/tar" , "--exclude-backups" , "--exclude-vcs" , "--transform=s,^,%s/src/,"%self.projectDir ] \ + excludes \ + [ "-jcvf", userSourceTarBz2 ] \ + self.enabledTools() self._execute ( command, "tar command failed" ) return def doRpm ( self ): self.svnTarball ( [], self.packageProjects ) for rpmDir in [ "SOURCES", "SPECS", "BUILD", "tmp" , "SRPMS", "RPMS/i386", "RPMS/i686", "RPMS/x86_64" ]: rpmFullDir = os.path.join ( self.rpmbuildDir, rpmDir ) if not os.path.isdir(rpmFullDir): os.makedirs ( rpmFullDir ) #rpmSpecFile = os.path.join ( self.rpmbuildDir, "SPECS" , "coriolis2.spec" ) rpmSourceFile = os.path.join ( self.rpmbuildDir, "SOURCES", self.sourceTarBz2 ) sourceFile = os.path.join ( self.tarballDir, self.sourceTarBz2 ) #if os.path.isfile ( rpmSpecFile ): # os.unlink ( rpmSpecFile ) #os.symlink ( self.specFile, rpmSpecFile ) if not os.path.islink ( rpmSourceFile ): os.symlink ( sourceFile, rpmSourceFile ) os.chdir ( self.rpmbuildDir ) command = [ "/usr/bin/rpmbuild" , "--define", "_topdir %s" % self.rpmbuildDir , "--define", "_tmppath %s" % self.tmppathDir , "--define", "_enable_debug_packages 0" , "-ta", "--with", "binarytar", rpmSourceFile ] self._execute ( command, "Rebuild rpm packages" ) return def doDeb ( self ): self.svnTarball ( [], self.packageProjects ) if not os.path.isdir(self.debbuildDir): os.makedirs ( self.debbuildDir ) os.chdir ( self.debbuildDir ) sourceFile = os.path.join ( self.tarballDir , self.sourceTarBz2 ) debOrigFile = os.path.join ( self.debbuildDir, "coriolis2_1.0.%s.orig.tar.bz2" % self.svnTag ) if not os.path.islink(debOrigFile): os.link ( sourceFile, debOrigFile ) command = [ "/bin/tar", "jxf", debOrigFile ] self._execute ( command, "Unpacking pristine sources" ) #command = [ "/bin/cp", "-r", self.debianDir, "." ] #self._execute ( command, "Copying Debian/Ubuntu package control files" ) packageDir = os.path.join ( self.debbuildDir, "coriolis2-1.0.%s" % self.svnTag ) os.chdir ( packageDir ) self._environment["CFLAGS" ] = "-O2" self._environment["CXXFLAGS"] = "-O2" command = [ "/usr/bin/debuild", "-us", "-uc" ] self._execute ( command, "Rebuild Debian packages" ) return def getProject ( self, name ): return self._conf.getProject(name) def loadConfiguration ( self, confFile ): self._conf.load( confFile ) def showConfiguration ( self ): self._conf.show()