From 25ba4ad3d46460c939e7f6a8e09125ed81a90ef3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Chaput Date: Thu, 18 Jun 2015 14:54:45 +0200 Subject: [PATCH] Partial rewrite of socInstaller.py to be more object-oriented. --- bootstrap/socInstaller.py | 383 ++++++++++++++++++++++---------------- 1 file changed, 221 insertions(+), 162 deletions(-) diff --git a/bootstrap/socInstaller.py b/bootstrap/socInstaller.py index 3284ff37..2396a39b 100755 --- a/bootstrap/socInstaller.py +++ b/bootstrap/socInstaller.py @@ -194,93 +194,189 @@ class GitRepository ( object ): return -def openLog ( logDir, stem ): - if not os.path.isdir(logDir): - os.makedirs( logDir ) +class Configuration ( object ): - index = 0 - timeTag = time.strftime( "%Y.%m.%d" ) - while True: - logFile = os.path.join(logDir,"%s-%s-%02d.log" % (stem,timeTag,index)) - if not os.path.isfile(logFile): - print "Report log: <%s>" % logFile - break - index += 1 - fd = open( logFile, "w" ) - return (fd,logFile) + PrimaryNames = \ + [ 'sender' , 'receiver' + , 'coriolisRepo', 'chamsRepo' , 'benchsRepo' + , 'homeDir' , 'masterHost' + , 'debugArg' , 'nightlyMode' + , 'rmSource' , 'rmBuild', 'doGit', 'doBuild', 'doBenchs', 'doSendReport' + , 'success' , 'rcode' + ] + SecondaryNames = \ + [ 'rootDir', 'srcDir', 'logDir', 'logs', 'fds' + ] + + def __init__ ( self ): + self._sender = 'Jean-Paul.Chaput@soc.lip6.fr' + self._receiver = 'Jean-Paul.Chaput@lip6.fr' + self._coriolisRepo = 'https://www-soc.lip6.fr/git/coriolis.git' + self._chamsRepo = 'file:///users/outil/chams/chams.git' + self._benchsRepo = 'https://www-soc.lip6.fr/git/alliance-check-toolkit.git' + self._homeDir = os.environ['HOME'] + self._debugArg = '' + self._rmSource = False + self._rmBuild = False + self._doGit = True + self._doBuild = True + self._doBenchs = False + self._doSendReport = True + self._nightlyMode = False + self._logs = { 'build':None, 'benchs':None } + self._fds = { 'build':None, 'benchs':None } + self._masterHost = self._detectMasterHost() + self._success = False + self._rcode = 0 + + self._updateSecondaries() + return + + def __setattr__ ( self, attribute, value ): + if attribute in Configuration.SecondaryNames: + print ErrorMessage( 1, 'Attempt to write in read-only attribute <%s> in Configuration.'%attribute ) + return + + if attribute == 'masterHost' or attribute == '_masterHost': + if value == 'lepka': + print 'Never touch the Git tree when running on .' + self._rmSource = False + self._rmBuild = False + self._doGit = False + self._doSendReport = False + self._targets = { 'SL6' :None + , 'SL6_64':None + , 'SL7_64':'lepka' + } + else: + self._targets = { 'SL6' :'rock' + , 'SL6_64':'bip' + , 'SL7_64':None + } + + if attribute[0] == '_': + self.__dict__[attribute] = value + return + + if attribute == 'homeDir': value = os.path.expanduser(value) + + self.__dict__['_'+attribute] = value + self._updateSecondaries() + return + + def __getattr__ ( self, attribute ): + if attribute[0] != '_': attribute = '_'+attribute + if not self.__dict__.has_key(attribute): + raise ErrorMessage( 1, 'Configuration has no attribute <%s>.'%attribute ) + return self.__dict__[attribute] + + def _updateSecondaries ( self ): + if self._nightlyMode: + self._targets['SL6'] = None + self._rootDir = self._homeDir + '/nightly/coriolis-2.x' + else: + if self._masterHost != 'lepka': + self._targets['SL6'] = 'rock' + self._rootDir = self._homeDir + '/coriolis-2.x' + self._srcDir = self._rootDir + '/src' + self._logDir = self._srcDir + '/logs' + return + + def _detectMasterHost ( self ): + masterHost = 'unknown' + hostname = socket.gethostname() + hostAddr = socket.gethostbyname(hostname) + + if hostname == 'lepka' and hostAddr == '127.0.0.1': + masterHost = 'lepka' + else: + masterHost = hostname.split('.')[0] + return masterHost + + def openLog ( self, stem ): + if not os.path.isdir(self._logDir): + os.makedirs( self._logDir ) + + index = 0 + timeTag = time.strftime( "%Y.%m.%d" ) + while True: + logFile = os.path.join(self._logDir,"%s-%s-%02d.log" % (stem,timeTag,index)) + if not os.path.isfile(logFile): + print "Report log: <%s>" % logFile + break + index += 1 + fd = open( logFile, "w" ) + self._logs[stem] = logFile + self._fds [stem] = fd + return + + def closeLogs ( self ): + for fd in self._fds.values(): + if fd: fd.close() + return -def sendReport ( state, buildLog, benchsLog, nightlyBuild ): - sender = 'Jean-Paul.Chaput@soc.lip6.fr' - receiver = 'Jean-Paul.Chaput@lip6.fr' - date = time.strftime( "%A %d %B %Y" ) +class Report ( object ): - stateText = 'FAILED' - buildText = 'SoC installation' - if state: stateText = 'SUCCESS' - if nightlyBuild: buildText = 'Nightly build' + def __init__ ( self, conf ): + self.conf = conf - message = MIMEMultipart() - message['Subject'] = '[%s] Coriolis & Chams %s %s' % (stateText,buildText,date) - message['From' ] = sender - message['To' ] = receiver + date = time.strftime( "%A %d %B %Y" ) + stateText = 'FAILED' + modeText = 'SoC installation' + if self.conf.success: stateText = 'SUCCESS' + if self.conf.nightlyMode: modeText = 'Nightly build' - mainText = '\n' - mainText += 'Salut le Crevard,\n' - mainText += '\n' - if nightlyBuild: - mainText += 'This is the nightly build report of Coriolis & Chams.\n' - else: - mainText += 'SoC installer report of Coriolis & Chams.\n' - mainText += '%s\n' % date - mainText += '\n' - if state: - mainText += 'Build was SUCCESSFUL\n' - else: - mainText += 'Build has FAILED, please have a look to the attached log file.\n' - mainText += '\n' - mainText += 'Complete log file can be found here:\n' - mainText += ' <%s>\n' % buildLog - if benchsLog: - mainText += ' <%s>\n' % benchsLog - mainText += '\n' - message.attach( MIMEText(mainText) ) + self.message = MIMEMultipart() + self.message['Subject'] = '[%s] Coriolis & Chams %s %s' % (stateText,modeText,date) + self.message['From' ] = self.conf.sender + self.message['To' ] = self.conf.receiver + self.attachements = [] - fd = open( buildLog, 'rb' ) - fd.seek( -1024*100, os.SEEK_END ) - tailLines = '' - for line in fd.readlines()[1:]: - tailLines += line - message.attach( MIMEApplication(tailLines) ) - fd.close() + self.mainText = '\n' + self.mainText += 'Salut le Crevard,\n' + self.mainText += '\n' + if self.conf.nightlyMode: + self.mainText += 'This is the nightly build report of Coriolis & Chams.\n' + else: + self.mainText += 'SoC installer report of Coriolis & Chams.\n' + self.mainText += '%s\n' % date + self.mainText += '\n' + if self.conf.success: + self.mainText += 'Build was SUCCESSFUL\n' + else: + self.mainText += 'Build has FAILED, please have a look to the attached log file(s).\n' + self.mainText += '\n' + self.mainText += 'Complete log file(s) can be found here:\n' + return - if benchsLog: - fd = open( benchsLog, 'rb' ) - fd.seek( -1024*100, os.SEEK_END ) + def attachLog ( self, logFile ): + if not logFile: return + + fd = open( logFile, 'rb' ) + try: + fd.seek( -1024*100, os.SEEK_END ) + except IOError, e: + pass tailLines = '' for line in fd.readlines()[1:]: tailLines += line - message.attach( MIMEApplication(tailLines) ) fd.close() + self.mainText += ' <%s>\n' % logFile - session = smtplib.SMTP( 'localhost' ) - session.sendmail( sender, receiver, message.as_string() ) - session.quit() - return + self.attachements.append( MIMEApplication(tailLines) ) + return + def send ( self ): + self.message.attach( MIMEText(self.mainText) ) + for attachement in self.attachements: + self.message.attach( attachement ) -def detectRunningHost (): - runningHost = 'unknown' - hostname = socket.gethostname() - hostAddr = socket.gethostbyname(hostname) - - if hostname == 'lepka' and hostAddr == '127.0.0.1': - print 'Running on , watchout mode enabled.' - runningHost = 'lepka' - else: - runningHost = hostname.split('.')[0] - - return runningHost + print "Sending mail report to <%s>" % self.conf.receiver + session = smtplib.SMTP( 'localhost' ) + session.sendmail( self.conf.sender, self.conf.receiver, self.message.as_string() ) + session.quit() + return # ------------------------------------------------------------------- @@ -290,6 +386,7 @@ def detectRunningHost (): parser = optparse.OptionParser () parser.add_option ( "--debug" , action="store_true" , dest="debug" , help="Build a aka (-g) version." ) parser.add_option ( "--no-git" , action="store_true" , dest="noGit" , help="Do not pull/update Git repositories before building." ) +parser.add_option ( "--no-build" , action="store_true" , dest="noBuild" , help="Do not rebuild the tools, must have already be done." ) parser.add_option ( "--no-report" , action="store_true" , dest="noReport" , help="Do not send a final report." ) parser.add_option ( "--nightly" , action="store_true" , dest="nightly" , help="Perform a nighly build." ) parser.add_option ( "--benchs" , action="store_true" , dest="benchs" , help="Run the sanity benchs." ) @@ -299,74 +396,38 @@ parser.add_option ( "--rm-all" , action="store_true" , dest= parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" , help="The root directory (default: <~/coriolis-2.x/>)." ) (options, args) = parser.parse_args () -nightlyBuild = False -fdBuildLog = None -fdBenchsLog = None -buildLogFile = None -benchsLogFile = None +conf = Configuration() try: - rmSource = False - rmBuild = False - doGit = True - doBench = False - doSendReport = True - debugArg = '' - runningHost = detectRunningHost() - targetSL6 = 'rock' - targetSL6_64 = 'bip' - targetSL7_64 = None + if options.debug: conf.debugArg = '--debug' + if options.nightly: conf.nightlyMode = True + if options.noGit: conf.doGit = False + if options.noBuild: conf.doBuild = False + if options.benchs: conf.doBenchs = True + if options.noReport: conf.doSendReport = False + if options.rmSource or options.rmAll: conf.rmSource = True + if options.rmBuild or options.rmAll: conf.rmBuild = True - if options.debug: debugArg = '--debug' - if options.nightly: nightlyBuild = True - if options.noGit: doGit = False - if options.benchs: doBenchs = True - if options.noReport: doSendReport = False - if options.rmSource or options.rmAll: rmSource = True - if options.rmBuild or options.rmAll: rmBuild = True + gitCoriolis = GitRepository( conf.coriolisRepo, conf.srcDir ) + gitChams = GitRepository( conf.chamsRepo , conf.srcDir ) + gitBenchs = GitRepository( conf.benchsRepo , conf.srcDir ) - if runningHost == 'lepka': - print 'Never touch the Git tree when running on .' - doGit = False - rmSource = False - rmBuild = False - targetSL7_64 = 'lepka' - targetSL6 = None - targetSL6_64 = None - - if nightlyBuild: - targetSL6 = None - - allianceCheckRepo = 'https://www-soc.lip6.fr/git/alliance-check-toolkit.git' - coriolisRepo = 'https://www-soc.lip6.fr/git/coriolis.git' - chamsRepo = 'file:///users/outil/chams/chams.git' - homeDir = os.environ['HOME'] - rootDir = homeDir + '/coriolis-2.x' - if nightlyBuild: - rootDir = homeDir + '/nightly/coriolis-2.x' - srcDir = rootDir + '/src' - logDir = srcDir + '/logs' - - gitCoriolis = GitRepository( coriolisRepo , srcDir ) - gitChams = GitRepository( chamsRepo , srcDir ) - gitAllianceCheck = GitRepository( allianceCheckRepo, srcDir ) - - if doGit: - if rmSource: gitCoriolis.removeLocalRepo() + if conf.doGit: + if conf.rmSource: gitCoriolis.removeLocalRepo() gitCoriolis.clone () gitCoriolis.checkout( 'devel' ) - if rmSource: gitChams.removeLocalRepo() + if conf.rmSource: gitChams.removeLocalRepo() gitChams.clone () gitChams.checkout( 'devel' ) - if rmSource: AllianceCheck.removeLocalRepo() - AllianceCheck.clone () + if conf.rmSource: gitBenchs.removeLocalRepo() + gitBenchs.clone() - if rmBuild: - for entry in os.listdir(rootDir): + if conf.rmBuild: + for entry in os.listdir(conf.rootDir): if entry.startswith('Linux.'): - buildDir = rootDir+'/'+entry + buildDir = conf.rootDir+'/'+entry print 'Removing OS build directory: <%s>' % buildDir shutil.rmtree( buildDir ) @@ -376,51 +437,49 @@ try: , ' <%s>' % ccbBin ] ) - fdBuildLog,buildLogFile = openLog( logDir, 'build' ) - if doBenchs: - fdBenchsLog,benchsLogFile = openLog( logDir, 'benchs' ) + if conf.doBuild: conf.openLog( 'build' ) + if conf.doBenchs: conf.openLog( 'benchs' ) buildCommand = '%s --root=%s --project=coriolis --project=chams --make="-j%%d install" %%s' \ - % (ccbBin,rootDir) + % (ccbBin,conf.rootDir) benchsCommand = 'cd %s/benchs && ./bin/go.sh clean && ./bin/go.sh lvx' \ - % (gitAllianceCheck.localRepoDir) + % (gitBenchs.localRepoDir) - if targetSL7_64: - commands = [ ( targetSL7_64, buildCommand % (3,debugArg) , fdBuildLog ) - , ( targetSL7_64, buildCommand % (1,debugArg+' --doc'), fdBuildLog ) - ] - if doBenchs: - commands += [ ( targetSL7_64, benchsCommand, fdBenchsLog ) ] - if targetSL6_64: - commands = [ ( targetSL6_64, buildCommand % (6,debugArg+' --devtoolset-2') , fdBuildLog ) - , ( targetSL6_64, buildCommand % (1,debugArg+' --devtoolset-2 --doc'), fdBuildLog ) - ] - if doBenchs: - commands += [ ( targetSL6_64, benchsCommand, fdBenchsLog ) ] - if targetSL6: - commands = [ ( targetSL6, buildCommand % (2,debugArg+' --devtoolset-2') , fdBuildLog ) - , ( targetSL6, buildCommand % (1,debugArg+' --devtoolset-2 --doc'), fdBuildLog ) - ] - if doBenchs: - commands += [ ( targetSL6, benchsCommand, fdBenchsLog ) ] + commands = \ + [ ( conf.targets['SL7_64'], buildCommand % (3,conf.debugArg) , conf.fds['build' ] ) + , ( conf.targets['SL7_64'], buildCommand % (1,conf.debugArg+' --doc') , conf.fds['build' ] ) + , ( conf.targets['SL7_64'], benchsCommand , conf.fds['benchs'] ) + , ( conf.targets['SL6_64'], buildCommand % (6,conf.debugArg+' --devtoolset-2') , conf.fds['build' ] ) + , ( conf.targets['SL6_64'], buildCommand % (1,conf.debugArg+' --devtoolset-2 --doc'), conf.fds['build' ] ) + , ( conf.targets['SL6_64'], benchsCommand , conf.fds['benchs'] ) + , ( conf.targets['SL6'] , buildCommand % (2,conf.debugArg+' --devtoolset-2') , conf.fds['build' ] ) + , ( conf.targets['SL6'] , buildCommand % (1,conf.debugArg+' --devtoolset-2 --doc'), conf.fds['build' ] ) + , ( conf.targets['SL6'] , benchsCommand , conf.fds['benchs'] ) + ] for host,command,fd in commands: - Command( [ 'ssh', host, command ], fd ).execute() + if host and fd: + print 'Executing command on <%s>:' % host + print ' %s' % command + Command( [ 'ssh', host, command ], fd ).execute() - fdBuildLog.close() - if doBenchs: fdBenchsLog.close() - - if doSendReport: - sendReport( True, buildLogFile, benchsLogFile, nightlyBuild ) + conf.closeLogs() + conf.success = True except ErrorMessage, e: print e - if fdBuildLog: fdBuildLog.close() - if fdBenchsLog: fdBenchsLog.close() + conf.closeLogs() + conf.success = False + if showTrace: print '\nPython stack trace:' traceback.print_tb( sys.exc_info()[2] ) - sendReport( False, buildLogFile, benchsLogFile, nightlyBuild ) - sys.exit( e.code ) + conf.rcode = e.code -sys.exit( 0 ) +if conf.doSendReport: + report = Report( conf ) + report.attachLog( conf.logs['build' ] ) + report.attachLog( conf.logs['benchs'] ) + report.send() + +sys.exit( conf.rcode )