From fa1081275d52ff8436b3d6cfa8af694e7d23cb40 Mon Sep 17 00:00:00 2001 From: Jean-Paul Chaput Date: Sun, 14 Jun 2015 16:23:11 +0200 Subject: [PATCH] Now run also the benches. Added a "local" mode run. --- bootstrap/socInstaller.py | 169 ++++++++++++++++++++++++++++---------- 1 file changed, 126 insertions(+), 43 deletions(-) diff --git a/bootstrap/socInstaller.py b/bootstrap/socInstaller.py index aedbcb85..3284ff37 100755 --- a/bootstrap/socInstaller.py +++ b/bootstrap/socInstaller.py @@ -32,6 +32,7 @@ try: import traceback import distutils.sysconfig import subprocess + import socket import re import smtplib from email.mime.text import MIMEText @@ -193,14 +194,14 @@ class GitRepository ( object ): return -def openLog ( logDir ): +def openLog ( logDir, stem ): if not os.path.isdir(logDir): os.makedirs( logDir ) index = 0 timeTag = time.strftime( "%Y.%m.%d" ) while True: - logFile = os.path.join(logDir,"install-%s-%02d.log" % (timeTag,index)) + logFile = os.path.join(logDir,"%s-%s-%02d.log" % (stem,timeTag,index)) if not os.path.isfile(logFile): print "Report log: <%s>" % logFile break @@ -209,23 +210,28 @@ def openLog ( logDir ): return (fd,logFile) -def sendReport ( state, installLog ): +def sendReport ( state, buildLog, benchsLog, nightlyBuild ): sender = 'Jean-Paul.Chaput@soc.lip6.fr' receiver = 'Jean-Paul.Chaput@lip6.fr' date = time.strftime( "%A %d %B %Y" ) stateText = 'FAILED' - if state: stateText = 'SUCCESS' + buildText = 'SoC installation' + if state: stateText = 'SUCCESS' + if nightlyBuild: buildText = 'Nightly build' message = MIMEMultipart() - message['Subject'] = '[%s] Coriolis & Chams Nightly build %s' % (stateText,date) + message['Subject'] = '[%s] Coriolis & Chams %s %s' % (stateText,buildText,date) message['From' ] = sender message['To' ] = receiver mainText = '\n' mainText += 'Salut le Crevard,\n' mainText += '\n' - mainText += 'This is the nightly build report of Coriolis & Chams.\n' + if nightlyBuild: + mainText += 'This is the nightly build report of Coriolis & Chams.\n' + else: + mainText += 'SoC installer report of Coriolis & Chams.\n' mainText += '%s\n' % date mainText += '\n' if state: @@ -234,11 +240,13 @@ def sendReport ( state, installLog ): mainText += 'Build has FAILED, please have a look to the attached log file.\n' mainText += '\n' mainText += 'Complete log file can be found here:\n' - mainText += ' <%s>\n' % installLog + mainText += ' <%s>\n' % buildLog + if benchsLog: + mainText += ' <%s>\n' % benchsLog mainText += '\n' message.attach( MIMEText(mainText) ) - fd = open( installLog, 'rb' ) + fd = open( buildLog, 'rb' ) fd.seek( -1024*100, os.SEEK_END ) tailLines = '' for line in fd.readlines()[1:]: @@ -246,58 +254,114 @@ def sendReport ( state, installLog ): message.attach( MIMEApplication(tailLines) ) fd.close() + if benchsLog: + fd = open( benchsLog, 'rb' ) + fd.seek( -1024*100, os.SEEK_END ) + tailLines = '' + for line in fd.readlines()[1:]: + tailLines += line + message.attach( MIMEApplication(tailLines) ) + fd.close() + session = smtplib.SMTP( 'localhost' ) session.sendmail( sender, receiver, message.as_string() ) session.quit() return +def detectRunningHost (): + runningHost = 'unknown' + hostname = socket.gethostname() + hostAddr = socket.gethostbyname(hostname) + + if hostname == 'lepka' and hostAddr == '127.0.0.1': + print 'Running on , watchout mode enabled.' + runningHost = 'lepka' + else: + runningHost = hostname.split('.')[0] + + return runningHost + + # ------------------------------------------------------------------- # Main Part. parser = optparse.OptionParser () parser.add_option ( "--debug" , action="store_true" , dest="debug" , help="Build a aka (-g) version." ) +parser.add_option ( "--no-git" , action="store_true" , dest="noGit" , help="Do not pull/update Git repositories before building." ) +parser.add_option ( "--no-report" , action="store_true" , dest="noReport" , help="Do not send a final report." ) parser.add_option ( "--nightly" , action="store_true" , dest="nightly" , help="Perform a nighly build." ) +parser.add_option ( "--benchs" , action="store_true" , dest="benchs" , help="Run the sanity benchs." ) parser.add_option ( "--rm-build" , action="store_true" , dest="rmBuild" , help="Remove the build/install directories." ) parser.add_option ( "--rm-source" , action="store_true" , dest="rmSource" , help="Remove the Git source repositories." ) parser.add_option ( "--rm-all" , action="store_true" , dest="rmAll" , help="Remove everything (source+build+install)." ) parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" , help="The root directory (default: <~/coriolis-2.x/>)." ) (options, args) = parser.parse_args () -fdLog = None -logFile = None +nightlyBuild = False +fdBuildLog = None +fdBenchsLog = None +buildLogFile = None +benchsLogFile = None try: - nightlyBuild = False rmSource = False rmBuild = False + doGit = True + doBench = False + doSendReport = True debugArg = '' + runningHost = detectRunningHost() + targetSL6 = 'rock' + targetSL6_64 = 'bip' + targetSL7_64 = None - if options.debug: debugArg = '--debug' - if options.nightly: nightlyBuild = True + if options.debug: debugArg = '--debug' + if options.nightly: nightlyBuild = True + if options.noGit: doGit = False + if options.benchs: doBenchs = True + if options.noReport: doSendReport = False if options.rmSource or options.rmAll: rmSource = True if options.rmBuild or options.rmAll: rmBuild = True - coriolisRepo = 'https://www-soc.lip6.fr/git/coriolis.git' - chamsRepo = 'file:///users/outil/chams/chams.git' - homeDir = os.environ['HOME'] - rootDir = homeDir + '/coriolis-2.x' + if runningHost == 'lepka': + print 'Never touch the Git tree when running on .' + doGit = False + rmSource = False + rmBuild = False + targetSL7_64 = 'lepka' + targetSL6 = None + targetSL6_64 = None + if nightlyBuild: - rootDir = homeDir + '/nightly/coriolis-2.x' - srcDir = rootDir + '/src' - logDir = rootDir + '/log' + targetSL6 = None + allianceCheckRepo = 'https://www-soc.lip6.fr/git/alliance-check-toolkit.git' + coriolisRepo = 'https://www-soc.lip6.fr/git/coriolis.git' + chamsRepo = 'file:///users/outil/chams/chams.git' + homeDir = os.environ['HOME'] + rootDir = homeDir + '/coriolis-2.x' + if nightlyBuild: + rootDir = homeDir + '/nightly/coriolis-2.x' + srcDir = rootDir + '/src' + logDir = srcDir + '/logs' - gitCoriolis = GitRepository( coriolisRepo, srcDir ) - if rmSource: gitCoriolis.removeLocalRepo() - gitCoriolis.clone () - gitCoriolis.checkout( 'devel' ) + gitCoriolis = GitRepository( coriolisRepo , srcDir ) + gitChams = GitRepository( chamsRepo , srcDir ) + gitAllianceCheck = GitRepository( allianceCheckRepo, srcDir ) - gitChams = GitRepository( chamsRepo, srcDir ) - if rmSource: gitChams.removeLocalRepo() - gitChams.clone () - gitChams.checkout( 'devel' ) + if doGit: + if rmSource: gitCoriolis.removeLocalRepo() + gitCoriolis.clone () + gitCoriolis.checkout( 'devel' ) + + if rmSource: gitChams.removeLocalRepo() + gitChams.clone () + gitChams.checkout( 'devel' ) + + if rmSource: AllianceCheck.removeLocalRepo() + AllianceCheck.clone () if rmBuild: for entry in os.listdir(rootDir): @@ -312,32 +376,51 @@ try: , ' <%s>' % ccbBin ] ) - commandFormat = '%s --root=%s --project=coriolis --project=chams --devtoolset-2 --make="-j%%d install" %%s' \ - % (ccbBin,rootDir) + fdBuildLog,buildLogFile = openLog( logDir, 'build' ) + if doBenchs: + fdBenchsLog,benchsLogFile = openLog( logDir, 'benchs' ) - commands = [ ( 'bip', commandFormat % (6,debugArg) ) - , ( 'bip', commandFormat % (1,debugArg+' --doc') ) - ] - if not nightlyBuild: - commands = [ ( 'rock', commandFormat % (2,debugArg) ) - , ( 'rock', commandFormat % (1,debugArg+' --doc') ) + buildCommand = '%s --root=%s --project=coriolis --project=chams --make="-j%%d install" %%s' \ + % (ccbBin,rootDir) + benchsCommand = 'cd %s/benchs && ./bin/go.sh clean && ./bin/go.sh lvx' \ + % (gitAllianceCheck.localRepoDir) + + if targetSL7_64: + commands = [ ( targetSL7_64, buildCommand % (3,debugArg) , fdBuildLog ) + , ( targetSL7_64, buildCommand % (1,debugArg+' --doc'), fdBuildLog ) ] + if doBenchs: + commands += [ ( targetSL7_64, benchsCommand, fdBenchsLog ) ] + if targetSL6_64: + commands = [ ( targetSL6_64, buildCommand % (6,debugArg+' --devtoolset-2') , fdBuildLog ) + , ( targetSL6_64, buildCommand % (1,debugArg+' --devtoolset-2 --doc'), fdBuildLog ) + ] + if doBenchs: + commands += [ ( targetSL6_64, benchsCommand, fdBenchsLog ) ] + if targetSL6: + commands = [ ( targetSL6, buildCommand % (2,debugArg+' --devtoolset-2') , fdBuildLog ) + , ( targetSL6, buildCommand % (1,debugArg+' --devtoolset-2 --doc'), fdBuildLog ) + ] + if doBenchs: + commands += [ ( targetSL6, benchsCommand, fdBenchsLog ) ] - fdLog,logFile = openLog( logDir ) + for host,command,fd in commands: + Command( [ 'ssh', host, command ], fd ).execute() - for host,command in commands: - Command( [ 'ssh', host, command ], fdLog ).execute() + fdBuildLog.close() + if doBenchs: fdBenchsLog.close() - fdLog.close() - sendReport( True, logFile ) + if doSendReport: + sendReport( True, buildLogFile, benchsLogFile, nightlyBuild ) except ErrorMessage, e: print e - if fdLog: fdLog.close() + if fdBuildLog: fdBuildLog.close() + if fdBenchsLog: fdBenchsLog.close() if showTrace: print '\nPython stack trace:' traceback.print_tb( sys.exc_info()[2] ) - sendReport( False, logFile ) + sendReport( False, buildLogFile, benchsLogFile, nightlyBuild ) sys.exit( e.code ) sys.exit( 0 )