497 lines
17 KiB
Python
Executable File
497 lines
17 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# -*- mode:Python -*-
|
|
#
|
|
# This file is part of the Coriolis Software.
|
|
# Copyright (c) UPMC 2015-2016, All Rights Reserved
|
|
#
|
|
# +-----------------------------------------------------------------+
|
|
# | C O R I O L I S |
|
|
# | C o r i o l i s / C h a m s I n s t a l l e r |
|
|
# | |
|
|
# | Authors : Jean-Paul Chaput |
|
|
# | E-mail : Jean-Paul.Chaput@asim.lip6.fr |
|
|
# | =============================================================== |
|
|
# | Python : "./socInstaller.py" |
|
|
# +-----------------------------------------------------------------+
|
|
#
|
|
# WARNING:
|
|
# This script has been designed only for internal use in the
|
|
# LIP6/SoC department. If you want to use it you will need to
|
|
# change the hardwired configuration.
|
|
|
|
|
|
showTrace = True
|
|
|
|
try:
|
|
import sys
|
|
import os.path
|
|
import shutil
|
|
import optparse
|
|
import time
|
|
import traceback
|
|
import distutils.sysconfig
|
|
import subprocess
|
|
import socket
|
|
import re
|
|
import smtplib
|
|
from email.mime.text import MIMEText
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.application import MIMEApplication
|
|
except ImportError, e:
|
|
module = str(e).split()[-1]
|
|
|
|
|
|
class ErrorMessage ( Exception ):
|
|
|
|
def __init__ ( self, code, *arguments ):
|
|
self._code = code
|
|
self._errors = [ 'Malformed call to ErrorMessage()', '%s' % str(arguments) ]
|
|
|
|
text = None
|
|
if len(arguments) == 1:
|
|
if isinstance(arguments[0],Exception): text = str(arguments[0]).split('\n')
|
|
else:
|
|
self._errors = arguments[0]
|
|
elif len(arguments) > 1:
|
|
text = list(arguments)
|
|
|
|
if text:
|
|
self._errors = []
|
|
while len(text[0]) == 0: del text[0]
|
|
|
|
lstrip = 0
|
|
if text[0].startswith('[ERROR]'): lstrip = 8
|
|
|
|
for line in text:
|
|
if line[0:lstrip ] == ' '*lstrip or \
|
|
line[0:lstrip-1] == '[ERROR]':
|
|
self._errors += [ line[lstrip:] ]
|
|
else:
|
|
self._errors += [ line.lstrip() ]
|
|
return
|
|
|
|
def __str__ ( self ):
|
|
if not isinstance(self._errors,list):
|
|
return "[ERROR] %s" % self._errors
|
|
|
|
formatted = "\n"
|
|
for i in range(len(self._errors)):
|
|
if i == 0: formatted += "[ERROR] %s" % self._errors[i]
|
|
else: formatted += " %s" % self._errors[i]
|
|
if i+1 < len(self._errors): formatted += "\n"
|
|
return formatted
|
|
|
|
def addMessage ( self, message ):
|
|
if not isinstance(self._errors,list):
|
|
self._errors = [ self._errors ]
|
|
if isinstance(message,list):
|
|
for line in message:
|
|
self._errors += [ line ]
|
|
else:
|
|
self._errors += [ message ]
|
|
return
|
|
|
|
def terminate ( self ):
|
|
print self
|
|
sys.exit(self._code)
|
|
|
|
@property
|
|
def code ( self ): return self._code
|
|
|
|
|
|
class BadBinary ( ErrorMessage ):
|
|
|
|
def __init__ ( self, binary ):
|
|
ErrorMessage.__init__( self, 1, "Binary not found: <%s>." % binary )
|
|
return
|
|
|
|
|
|
class BadReturnCode ( ErrorMessage ):
|
|
|
|
def __init__ ( self, status ):
|
|
ErrorMessage.__init__( self, 1, "Command returned status:%d." % status )
|
|
return
|
|
|
|
|
|
class Command ( object ):
|
|
|
|
def __init__ ( self, arguments, fdLog=None ):
|
|
self.arguments = arguments
|
|
self.fdLog = fdLog
|
|
|
|
if self.fdLog != None and not isinstance(self.fdLog,file):
|
|
print '[WARNING] Command.__init__(): <fdLog> is neither None or a file.'
|
|
return
|
|
|
|
def execute ( self ):
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
|
|
try:
|
|
child = subprocess.Popen( self.arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
|
|
|
|
while True:
|
|
line = child.stdout.readline()
|
|
if not line: break
|
|
|
|
print line[:-1]
|
|
sys.stdout.flush()
|
|
|
|
if isinstance(self.fdLog,file):
|
|
self.fdLog.write( line )
|
|
self.fdLog.flush()
|
|
except OSError, e:
|
|
raise BadBinary( self.arguments[0] )
|
|
|
|
(pid,status) = os.waitpid( child.pid, 0 )
|
|
status >>= 8
|
|
if status != 0:
|
|
raise BadReturnCode( status )
|
|
|
|
return
|
|
|
|
|
|
class GitRepository ( object ):
|
|
|
|
@staticmethod
|
|
def getLocalRepository ( url ):
|
|
localRepo = url.split( '/' )[-1]
|
|
if localRepo.endswith('.git'):
|
|
localRepo = localRepo[:-4]
|
|
return localRepo
|
|
|
|
def __init__ ( self, url, cloneDir ):
|
|
self.url = url
|
|
self.cloneDir = cloneDir
|
|
self.localRepo = GitRepository.getLocalRepository( url )
|
|
return
|
|
|
|
@property
|
|
def localRepoDir ( self ): return self.cloneDir+'/'+self.localRepo
|
|
|
|
def removeLocalRepo ( self ):
|
|
if os.path.isdir(self.localRepoDir):
|
|
print 'Removing Git local repository: <%s>' % self.localRepoDir
|
|
shutil.rmtree( self.localRepoDir )
|
|
return
|
|
|
|
def clone ( self ):
|
|
if not os.path.isdir(self.cloneDir):
|
|
os.makedirs( self.cloneDir )
|
|
|
|
if not os.path.isdir(self.localRepoDir):
|
|
os.chdir( self.cloneDir )
|
|
Command( [ 'git', 'clone', self.url ] ).execute()
|
|
else:
|
|
os.chdir( self.localRepoDir )
|
|
Command( [ 'git', 'pull' ] ).execute()
|
|
return
|
|
|
|
def checkout ( self, branch ):
|
|
os.chdir( self.localRepoDir )
|
|
Command( [ 'git', 'checkout', branch ] ).execute()
|
|
return
|
|
|
|
|
|
class Configuration ( object ):
|
|
|
|
PrimaryNames = \
|
|
[ 'sender' , 'receivers'
|
|
, 'coriolisRepo', 'chamsRepo' , 'benchsRepo', 'supportRepos'
|
|
, 'homeDir' , 'masterHost'
|
|
, 'debugArg' , 'nightlyMode'
|
|
, 'rmSource' , 'rmBuild', 'doGit', 'doBuild', 'doBenchs', 'doSendReport'
|
|
, 'success' , 'rcode'
|
|
]
|
|
SecondaryNames = \
|
|
[ 'rootDir', 'srcDir', 'logDir', 'logs', 'fds'
|
|
]
|
|
|
|
def __init__ ( self ):
|
|
self._sender = 'Jean-Paul.Chaput@soc.lip6.fr'
|
|
self._receivers = [ 'Jean-Paul.Chaput@lip6.fr', 'Eric.Lao@lip6.fr' ]
|
|
self._supportRepos = [ 'http://github.com/miloyip/rapidjson' ]
|
|
self._coriolisRepo = 'https://www-soc.lip6.fr/git/coriolis.git'
|
|
self._chamsRepo = 'file:///users/outil/chams/chams.git'
|
|
self._benchsRepo = 'https://www-soc.lip6.fr/git/alliance-check-toolkit.git'
|
|
self._homeDir = os.environ['HOME']
|
|
self._debugArg = ''
|
|
self._rmSource = False
|
|
self._rmBuild = False
|
|
self._doGit = True
|
|
self._doBuild = True
|
|
self._doBenchs = False
|
|
self._doSendReport = True
|
|
self._nightlyMode = False
|
|
self._logs = { 'build':None, 'benchs':None }
|
|
self._fds = { 'build':None, 'benchs':None }
|
|
self._masterHost = self._detectMasterHost()
|
|
self._success = False
|
|
self._rcode = 0
|
|
|
|
self._updateSecondaries()
|
|
return
|
|
|
|
def __setattr__ ( self, attribute, value ):
|
|
if attribute in Configuration.SecondaryNames:
|
|
print ErrorMessage( 1, 'Attempt to write in read-only attribute <%s> in Configuration.'%attribute )
|
|
return
|
|
|
|
if attribute == 'masterHost' or attribute == '_masterHost':
|
|
if value == 'lepka':
|
|
print 'Never touch the Git tree when running on <lepka>.'
|
|
self._rmSource = False
|
|
self._rmBuild = False
|
|
self._doGit = False
|
|
self._doSendReport = False
|
|
self._targets = { 'SL6' :None
|
|
, 'SL6_64':None
|
|
, 'SL7_64':'lepka'
|
|
}
|
|
else:
|
|
self._targets = { 'SL6' :'rock'
|
|
, 'SL6_64':'bip'
|
|
, 'SL7_64':None
|
|
}
|
|
|
|
if attribute[0] == '_':
|
|
self.__dict__[attribute] = value
|
|
return
|
|
|
|
if attribute == 'homeDir': value = os.path.expanduser(value)
|
|
|
|
self.__dict__['_'+attribute] = value
|
|
self._updateSecondaries()
|
|
return
|
|
|
|
def __getattr__ ( self, attribute ):
|
|
if attribute[0] != '_': attribute = '_'+attribute
|
|
if not self.__dict__.has_key(attribute):
|
|
raise ErrorMessage( 1, 'Configuration has no attribute <%s>.'%attribute )
|
|
return self.__dict__[attribute]
|
|
|
|
def _updateSecondaries ( self ):
|
|
if self._nightlyMode:
|
|
self._targets['SL6'] = None
|
|
self._rootDir = self._homeDir + '/nightly/coriolis-2.x'
|
|
else:
|
|
if self._masterHost != 'lepka':
|
|
self._targets['SL6'] = 'rock'
|
|
self._rootDir = self._homeDir + '/coriolis-2.x'
|
|
self._srcDir = self._rootDir + '/src'
|
|
self._logDir = self._srcDir + '/logs'
|
|
return
|
|
|
|
def _detectMasterHost ( self ):
|
|
masterHost = 'unknown'
|
|
hostname = socket.gethostname()
|
|
hostAddr = socket.gethostbyname(hostname)
|
|
|
|
if hostname == 'lepka' and hostAddr == '127.0.0.1':
|
|
masterHost = 'lepka'
|
|
else:
|
|
masterHost = hostname.split('.')[0]
|
|
return masterHost
|
|
|
|
def openLog ( self, stem ):
|
|
if not os.path.isdir(self._logDir):
|
|
os.makedirs( self._logDir )
|
|
|
|
index = 0
|
|
timeTag = time.strftime( "%Y.%m.%d" )
|
|
while True:
|
|
logFile = os.path.join(self._logDir,"%s-%s-%02d.log" % (stem,timeTag,index))
|
|
if not os.path.isfile(logFile):
|
|
print "Report log: <%s>" % logFile
|
|
break
|
|
index += 1
|
|
fd = open( logFile, "w" )
|
|
self._logs[stem] = logFile
|
|
self._fds [stem] = fd
|
|
return
|
|
|
|
def closeLogs ( self ):
|
|
for fd in self._fds.values():
|
|
if fd: fd.close()
|
|
return
|
|
|
|
|
|
class Report ( object ):
|
|
|
|
def __init__ ( self, conf ):
|
|
self.conf = conf
|
|
|
|
commaspace = ', '
|
|
date = time.strftime( "%A %d %B %Y" )
|
|
stateText = 'FAILED'
|
|
modeText = 'SoC installation'
|
|
if self.conf.success: stateText = 'SUCCESS'
|
|
if self.conf.nightlyMode: modeText = 'Nightly build'
|
|
|
|
self.message = MIMEMultipart()
|
|
self.message['Subject'] = '[%s] Coriolis & Chams %s %s' % (stateText,modeText,date)
|
|
self.message['From' ] = self.conf.sender
|
|
self.message['To' ] = commaspace.join( self.conf.receivers )
|
|
self.attachements = []
|
|
|
|
self.mainText = '\n'
|
|
self.mainText += 'Salut le Crevard,\n'
|
|
self.mainText += '\n'
|
|
if self.conf.nightlyMode:
|
|
self.mainText += 'This is the nightly build report of Coriolis & Chams.\n'
|
|
else:
|
|
self.mainText += 'SoC installer report of Coriolis & Chams.\n'
|
|
self.mainText += '%s\n' % date
|
|
self.mainText += '\n'
|
|
if self.conf.success:
|
|
self.mainText += 'Build was SUCCESSFUL\n'
|
|
else:
|
|
self.mainText += 'Build has FAILED, please have a look to the attached log file(s).\n'
|
|
self.mainText += '\n'
|
|
self.mainText += 'Complete log file(s) can be found here:\n'
|
|
return
|
|
|
|
def attachLog ( self, logFile ):
|
|
if not logFile: return
|
|
|
|
fd = open( logFile, 'rb' )
|
|
try:
|
|
fd.seek( -1024*100, os.SEEK_END )
|
|
except IOError, e:
|
|
pass
|
|
tailLines = ''
|
|
for line in fd.readlines()[1:]:
|
|
tailLines += line
|
|
fd.close()
|
|
self.mainText += ' <%s>\n' % logFile
|
|
|
|
self.attachements.append( MIMEApplication(tailLines) )
|
|
return
|
|
|
|
def send ( self ):
|
|
self.message.attach( MIMEText(self.mainText) )
|
|
for attachement in self.attachements:
|
|
self.message.attach( attachement )
|
|
|
|
print "Sending mail report to <%s>" % self.conf.receivers
|
|
session = smtplib.SMTP( 'localhost' )
|
|
session.sendmail( self.conf.sender, self.conf.receivers, self.message.as_string() )
|
|
session.quit()
|
|
return
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# <socInstaller> Main Part.
|
|
|
|
|
|
parser = optparse.OptionParser ()
|
|
parser.add_option ( "--debug" , action="store_true" , dest="debug" , help="Build a <Debug> aka (-g) version." )
|
|
parser.add_option ( "--no-git" , action="store_true" , dest="noGit" , help="Do not pull/update Git repositories before building." )
|
|
parser.add_option ( "--no-build" , action="store_true" , dest="noBuild" , help="Do not rebuild the tools, must have already be done." )
|
|
parser.add_option ( "--no-report" , action="store_true" , dest="noReport" , help="Do not send a final report." )
|
|
parser.add_option ( "--nightly" , action="store_true" , dest="nightly" , help="Perform a nighly build." )
|
|
parser.add_option ( "--benchs" , action="store_true" , dest="benchs" , help="Run the <alliance-checker-toolkit> sanity benchs." )
|
|
parser.add_option ( "--rm-build" , action="store_true" , dest="rmBuild" , help="Remove the build/install directories." )
|
|
parser.add_option ( "--rm-source" , action="store_true" , dest="rmSource" , help="Remove the Git source repositories." )
|
|
parser.add_option ( "--rm-all" , action="store_true" , dest="rmAll" , help="Remove everything (source+build+install)." )
|
|
parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" , help="The root directory (default: <~/coriolis-2.x/>)." )
|
|
(options, args) = parser.parse_args ()
|
|
|
|
conf = Configuration()
|
|
|
|
try:
|
|
if options.debug: conf.debugArg = '--debug'
|
|
if options.nightly: conf.nightlyMode = True
|
|
if options.noGit: conf.doGit = False
|
|
if options.noBuild: conf.doBuild = False
|
|
if options.benchs: conf.doBenchs = True
|
|
if options.noReport: conf.doSendReport = False
|
|
if options.rmSource or options.rmAll: conf.rmSource = True
|
|
if options.rmBuild or options.rmAll: conf.rmBuild = True
|
|
|
|
gitSupports = []
|
|
for supportRepo in conf.supportRepos:
|
|
gitSupports.append( GitRepository( supportRepo, conf.srcDir+'/support' ) )
|
|
gitCoriolis = GitRepository( conf.coriolisRepo, conf.srcDir )
|
|
gitChams = GitRepository( conf.chamsRepo , conf.srcDir )
|
|
gitBenchs = GitRepository( conf.benchsRepo , conf.srcDir )
|
|
|
|
if conf.doGit:
|
|
for gitSupport in gitSupports:
|
|
if conf.rmSource: gitSupport.removeLocalRepo()
|
|
gitSupport.clone()
|
|
#if gitSupport.url.endswith('rapidjson'):
|
|
# gitSupport.checkout( 'a1c4f32' )
|
|
|
|
if conf.rmSource: gitCoriolis.removeLocalRepo()
|
|
gitCoriolis.clone ()
|
|
gitCoriolis.checkout( 'devel_anabatic' )
|
|
|
|
if conf.rmSource: gitChams.removeLocalRepo()
|
|
gitChams.clone ()
|
|
gitChams.checkout( 'devel' )
|
|
|
|
if conf.rmSource: gitBenchs.removeLocalRepo()
|
|
gitBenchs.clone()
|
|
|
|
if conf.rmBuild:
|
|
for entry in os.listdir(conf.rootDir):
|
|
if entry.startswith('Linux.'):
|
|
buildDir = conf.rootDir+'/'+entry
|
|
print 'Removing OS build directory: <%s>' % buildDir
|
|
shutil.rmtree( buildDir )
|
|
|
|
ccbBin = gitCoriolis.localRepoDir+'/bootstrap/ccb.py'
|
|
if not os.path.isfile( ccbBin ):
|
|
raise ErrorMessage( 1, [ 'Cannot find <ccb.py>, should be here:'
|
|
, ' <%s>' % ccbBin
|
|
] )
|
|
|
|
if conf.doBuild: conf.openLog( 'build' )
|
|
if conf.doBenchs: conf.openLog( 'benchs' )
|
|
|
|
buildCommand = '%s --root=%s --project=support --project=coriolis --project=chams --make="-j%%d install" %%s' \
|
|
% (ccbBin,conf.rootDir)
|
|
benchsCommand = 'cd %s/benchs && ./bin/go.sh clean && ./bin/go.sh lvx' \
|
|
% (gitBenchs.localRepoDir)
|
|
|
|
commands = \
|
|
[ ( conf.targets['SL7_64'], buildCommand % (3,conf.debugArg) , conf.fds['build' ] )
|
|
, ( conf.targets['SL7_64'], buildCommand % (1,conf.debugArg+' --doc') , conf.fds['build' ] )
|
|
, ( conf.targets['SL7_64'], benchsCommand , conf.fds['benchs'] )
|
|
, ( conf.targets['SL6_64'], buildCommand % (6,conf.debugArg+' --devtoolset-2') , conf.fds['build' ] )
|
|
, ( conf.targets['SL6_64'], buildCommand % (1,conf.debugArg+' --devtoolset-2 --doc'), conf.fds['build' ] )
|
|
, ( conf.targets['SL6_64'], benchsCommand , conf.fds['benchs'] )
|
|
, ( conf.targets['SL6'] , buildCommand % (2,conf.debugArg+' --devtoolset-2') , conf.fds['build' ] )
|
|
, ( conf.targets['SL6'] , buildCommand % (1,conf.debugArg+' --devtoolset-2 --doc'), conf.fds['build' ] )
|
|
, ( conf.targets['SL6'] , benchsCommand , conf.fds['benchs'] )
|
|
]
|
|
|
|
for host,command,fd in commands:
|
|
if host and fd:
|
|
print 'Executing command on <%s>:' % host
|
|
print ' %s' % command
|
|
Command( [ 'ssh', host, command ], fd ).execute()
|
|
|
|
conf.closeLogs()
|
|
conf.success = True
|
|
|
|
except ErrorMessage, e:
|
|
print e
|
|
conf.closeLogs()
|
|
conf.success = False
|
|
|
|
if showTrace:
|
|
print '\nPython stack trace:'
|
|
traceback.print_tb( sys.exc_info()[2] )
|
|
conf.rcode = e.code
|
|
|
|
if conf.doSendReport:
|
|
report = Report( conf )
|
|
report.attachLog( conf.logs['build' ] )
|
|
report.attachLog( conf.logs['benchs'] )
|
|
report.send()
|
|
|
|
sys.exit( conf.rcode )
|