coriolis/bootstrap/socInstaller.py

#!/usr/bin/env python3
#
# -*- mode:Python -*-
#
# This file is part of the Coriolis Software.
# Copyright (c) Sorbonne Université 2015-2021, All Rights Reserved
#
# +-----------------------------------------------------------------+ 
# |                   C O R I O L I S                               |
# |         C o r i o l i s   I n s t a l l e r                     |
# |                                                                 |
# |  Authors     :                    Jean-Paul Chaput              |
# |  E-mail      :       Jean-Paul.Chaput@asim.lip6.fr              |
# | =============================================================== |
# |  Python      :   "./socInstaller.py"                            |
# +-----------------------------------------------------------------+
#
# WARNING:
#   This script has been designed only for internal use in the
#   LIP6/CIAN department. If you want to use it you will need to
#   change the hardwired configuration.


showTrace = True

try:
    import sys
    import os.path
    import shutil
    import optparse
    import time
    import traceback
    import distutils.sysconfig
    import subprocess
    import socket
    import re
    import bz2
    import smtplib
    from io                     import IOBase
    from email.mime.text        import MIMEText
    from email.mime.multipart   import MIMEMultipart
    from email.mime.application import MIMEApplication
except ImportError as e:
    module = str(e).split()[-1]


class ErrorMessage ( Exception ):

    def __init__ ( self, code, *arguments ):
        self._code   = code
        self._errors = [ 'Malformed call to ErrorMessage()', '{}'.format(arguments) ]
        text = None
        if len(arguments) == 1:
            if isinstance(arguments[0],Exception): text = str(arguments[0]).split('\n')
            else:
                self._errors = arguments[0]
        elif len(arguments) > 1:
            text = list(arguments)
        if text:
            self._errors = []
            while len(text[0]) == 0: del text[0]
            lstrip = 0
            if text[0].startswith('[ERROR]'): lstrip = 8
            for line in text:
                if line[0:lstrip  ] == ' '*lstrip or \
                   line[0:lstrip-1] == '[ERROR]':
                    self._errors += [ line[lstrip:] ]
                else:
                    self._errors += [ line.lstrip() ]
        return

    def __str__ ( self ):
        if not isinstance(self._errors,list):
            return "[ERROR] {}".format(self._errors)
        formatted = "\n"
        for i in range(len(self._errors)):
            if i == 0: formatted += "[ERROR] {}".format(self._errors[i])
            else:      formatted += "        {}".format(self._errors[i])
            if i+1 < len(self._errors): formatted += "\n"
        return formatted

    def addMessage ( self, message ):
        if not isinstance(self._errors,list):
            self._errors = [ self._errors ]
        if isinstance(message,list):
            for line in message:
                  self._errors += [ line ]
        else:
            self._errors += [ message ]
        return

    def terminate ( self ):
        print( self )
        sys.exit(self._code)

    @property
    def code ( self ): return self._code


class BadBinary ( ErrorMessage ):

    def __init__ ( self, binary ):
        ErrorMessage.__init__( self, 1, 'Binary not found: "{}".'.format(binary) )
        return


class BadReturnCode ( ErrorMessage ):

    def __init__ ( self, status ):
        ErrorMessage.__init__( self, 1, 'Command returned status:{}.'.format(status) )
        return


class Command ( object ):

    def __init__ ( self, arguments, fdLog=None ):
        self.arguments = arguments
        self.fdLog     = fdLog
        if self.fdLog != None and not isinstance(self.fdLog,IOBase):
            print( '[WARNING] Command.__init__(): "fdLog" is neither None or a file.' )
        return

    def _argumentsToStr ( self, arguments ):
        s = ''
        for argument in arguments:
            if argument.find(' ') >= 0: s += ' "' + argument + '"'
            else:                       s += ' '  + argument
        return s

    def log ( self, text ):
        if isinstance(self.fdLog,IOBase):
            if isinstance(text,bytes):
                print( text[:-1].decode('utf-8') )
                self.fdLog.write( text.decode('utf-8') )
            elif isinstance(text,str):
                print( text[:-1] )
                self.fdLog.write( text )
            else:
                print( '[ERROR] Command.log(): "text" is neither bytes or str.' )
                print( '        {}'.format(text) )
            self.fdLog.flush()
        sys.stdout.flush()
        sys.stderr.flush()
        return

    def execute ( self ):
        global conf
        sys.stdout.flush()
        sys.stderr.flush()
        
        homeDir = os.environ['HOME']
        workDir = os.getcwd()
        if homeDir.startswith(homeDir):
            workDir = '~' + workDir[ len(homeDir) : ]
        user = 'root'
        if 'USER' in os.environ: user = os.environ['USER']
        prompt = '{}@{}:{}$'.format(user,conf.masterHost,workDir)
        
        try:
            self.log( '{}{}\n'.format(prompt,self._argumentsToStr(self.arguments)) )
            print( self.arguments )
            child = subprocess.Popen( self.arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
            while True:
                line = child.stdout.readline()
                if not line: break
                self.log( line )
        except OSError as e:
            raise BadBinary( self.arguments[0] )
        
        (pid,status) = os.waitpid( child.pid, 0 )
        status >>= 8
        if status != 0:
            raise BadReturnCode( status )
        return


class CommandArg ( object ):

    def __init__ ( self, command, wd=None, host=None, fdLog=None ):
        self.command = command
        self.host    = host
        self.wd      = wd
        self.fdLog   = fdLog
        return

    def __str__ ( self ):
        s = ''
        if self.wd: s = 'cd {} && '.format(self.wd)
        for i in range(len(self.command)):
            if i: s += ' '
            s += self.command[i]
        return s

    def getArgs ( self ):
        if not self.host: return self.command
        return [ 'ssh', self.host, str(self) ]

    def execute ( self ):
        if not self.host and self.wd: os.chdir( self.wd )
        Command( self.getArgs(), self.fdLog ).execute()
        return


class YosysCommand ( CommandArg ):

    def __init__ ( self, yosysBin, fdLog=None ):
        CommandArg.__init__ ( self, [ yosysBin ], fdLog=fdLog )
        return


class AllianceCommand ( CommandArg ):

    def __init__ ( self, alcBin, fdLog=None ):
        CommandArg.__init__ ( self, [ alcBin ], fdLog=fdLog )
        return


class CoriolisCommand ( CommandArg ):

    def __init__ ( self, ccbBin, rootDir, threads=1, otherArgs=[], fdLog=None ):
        CommandArg.__init__ ( self, [ ccbBin
                                    , '--root='+rootDir
                                    , '--project=coriolis'
                                    , '--make=-j{} install'.format(threads)
                                    ] + otherArgs
                                  , fdLog=fdLog )
        return


class BenchsCommand ( CommandArg ):

    def __init__ ( self, benchsDir, fdLog=None ):
        CommandArg.__init__ ( self, [ '../bin/go.sh' ], wd=benchsDir, fdLog=fdLog )
        return
        


class GitRepository ( object ):

    @staticmethod
    def getLocalRepository ( url ):
        localRepo = url.split( '/' )[-1]
        if localRepo.endswith('.git'):
          localRepo = localRepo[:-4]
        return localRepo

    def __init__ ( self, url, cloneDir, fdLog=None ):
        self.url       = url
        self.cloneDir  = cloneDir
        self.localRepo = GitRepository.getLocalRepository( url )
        self.fdLog     = fdLog
        return

    @property
    def localRepoDir ( self ): return self.cloneDir+'/'+self.localRepo

    def removeLocalRepo ( self ):
        if os.path.isdir(self.localRepoDir):
            print( 'Removing Git local repository: "{}"'.format(self.localRepoDir) )
            shutil.rmtree( self.localRepoDir )
        return

    def clone ( self ):
        print( 'Clone/pull from:', self.url )
        if not os.path.isdir(self.cloneDir):
            os.makedirs( self.cloneDir )
        if not os.path.isdir(self.localRepoDir):
            os.chdir( self.cloneDir )
            Command( [ 'git', 'clone', self.url ], self.fdLog ).execute()
        else:
            os.chdir( self.localRepoDir )
            Command( [ 'git', 'pull' ], self.fdLog ).execute()
        return

    def checkout ( self, branch ):
        os.chdir( self.localRepoDir )
        Command( [ 'git', 'checkout', branch ], self.fdLog ).execute()
        return


class Configuration ( object ):

    PrimaryNames = \
        [ 'sender'      , 'receivers'
        , 'coriolisRepo', 'benchsRepo' , 'supportRepos'
        , 'homeDir'     , 'masterHost'
        , 'debugArg'    , 'nightlyMode', 'dockerMode', 'chrootMode'
        , 'rmSource'    , 'rmBuild'
        , 'doGit'       , 'doAlliance' , 'doCoriolis', 'doBenchs', 'doSendReport'
        , 'success'     , 'rcode'
        ]
    SecondaryNames = \
        [ 'rootDir', 'srcDir', 'logDir', 'logs', 'fds', 'yosysBin', 'alcBin', 'ccbBin', 'benchsDir'
        ]

    def __init__ ( self ):
        self._sender       = 'Jean-Paul.Chaput@soc.lip6.fr'
        self._receivers    = [ 'Jean-Paul.Chaput@lip6.fr', ]
        self._supportRepos = [ 'http://github.com/miloyip/rapidjson' ]
        self._allianceRepo = 'https://gitlab.lip6.fr/jpc/alliance.git'
        self._coriolisRepo = 'https://gitlab.lip6.fr/jpc/coriolis.git'
        self._benchsRepo   = 'https://gitlab.lip6.fr/jpc/alliance-check-toolkit.git'
        self._homeDir      = os.environ['HOME']
        self._debugArg     = ''
        self._rmSource     = False
        self._rmBuild      = False
        self._doGit        = True
        self._doYosys      = False
        self._doAlliance   = False
        self._doCoriolis   = False
        self._doBenchs     = False
        self._doSendReport = False
        self._nightlyMode  = False
        self._dockerMode   = False
        self._chrootMode   = None
        self._logs         = { 'alliance':None, 'coriolis':None, 'benchs':None }
        self._fds          = { 'alliance':None, 'coriolis':None, 'benchs':None }
        self._ccbBin       = None
        self._benchsDir    = None
        self._masterHost   = self._detectMasterHost()
        self._success      = False
        self._rcode        = 0
        self._updateSecondaries()
        return

    def __setattr__ ( self, attribute, value ):
        if attribute in Configuration.SecondaryNames:
            print( ErrorMessage( 1, 'Attempt to write in read-only attribute "{}" in Configuration.' \
                                    .format(attribute) ))
            return
        
        if attribute == 'masterHost' or attribute == '_masterHost':
            if value == 'lepka':
                print( 'Never touch the Git tree when running on "lepka".' )
                self._rmSource     = False
                self._rmBuild      = False
                self._doGit        = False
                self._doSendReport = False
        if attribute[0] == '_':
            self.__dict__[attribute] = value
            return
        if attribute == 'homeDir': value = os.path.expanduser(value)
        
        self.__dict__['_'+attribute] = value
        self._updateSecondaries()
        return

    def __getattr__ ( self, attribute ):
        if attribute[0] != '_': attribute = '_'+attribute
        if not attribute in self.__dict__:
            raise ErrorMessage( 1, 'Configuration has no attribute "{}".'.format(attribute) )
        return self.__dict__[attribute]

    def _updateSecondaries ( self ):
        if self._nightlyMode:
            self._rootDir = self._homeDir + '/nightly/coriolis-2.x'
        else:
            self._rootDir  = self._homeDir + '/coriolis-2.x'
        self._srcDir     = self._rootDir + '/src'
        self._logDir     = self._srcDir  + '/logs'
        self._yosysBin   = self._srcDir  + '/' + GitRepository.getLocalRepository(self._coriolisRepo) + '/bootstrap/yosysInstaller.sh'
        self._alcBin     = self._srcDir  + '/' + GitRepository.getLocalRepository(self._coriolisRepo) + '/bootstrap/allianceInstaller.sh'
        self._ccbBin     = self._srcDir  + '/' + GitRepository.getLocalRepository(self._coriolisRepo) + '/bootstrap/ccb.py'
        self._benchsDir  = self._srcDir  + '/' + GitRepository.getLocalRepository(self._benchsRepo  ) + '/benchs'
        self._masterHost = self._detectMasterHost()
        return

    def _detectMasterHost ( self ):
        if self._chrootMode is None: return 'unknown'
        if self._chrootMode: return 'chrooted-host'
        masterHost = 'unknown'
        hostname   = socket.gethostname()
        hostAddr   = socket.gethostbyname(hostname)
        if hostname == 'lepka' and hostAddr == '127.0.0.1':
            masterHost = 'lepka'
        else:
            masterHost = hostname.split('.')[0]
        return masterHost

    def openLog ( self, stem ):
        if not os.path.isdir(self._logDir):
            os.makedirs( self._logDir )
        index   = 0
        timeTag = time.strftime( "%Y.%m.%d" )
        while True:
            logFile = os.path.join(self._logDir,"{}-{}-{:02}.log".format(stem,timeTag,index))
            if not os.path.isfile(logFile):
                print( 'Report log: "{}"'.format(logFile) )
                break
            index += 1
        fd = open( logFile, "w" )
        self._logs[stem] = logFile
        self._fds [stem] = fd
        return

    def closeLogs ( self ):
        for fd in self._fds.values():
            if fd: fd.close()
        return

    def compressLogs ( self ):
        for log in self._logs.values():
            if not log: continue
            fd   = open( log, 'r' )
            bzfd = bz2.BZ2File( log+'.bz2', 'w' )
            for line in fd.readlines():
                if isinstance(line,str):
                  bzfd.write( line.encode('utf-8') )
                elif isinstance(line,bytes):
                  bzfd.write( line )
            bzfd.close()
            fd.close()
            os.unlink( log )
        return

    def getCommands ( self, target ):
        commands = []
        if self.doYosys:
            if not os.path.isfile( self.yosysBin ):
                raise ErrorMessage( 1, [ 'Cannot find <yosysInstaller.sh>, should be here:'
                                       , '   "{}"'.format(self.yosysBin)
                                       ] )
            commands.append( YosysCommand( self.yosysBin, fdLog=self.fds['yosys'] ) )
        if self.doAlliance:
            if not os.path.isfile( self.alcBin ):
                raise ErrorMessage( 1, [ 'Cannot find <allianceInstaller.sh>, should be here:'
                                       , '   "{}"'.format(self.alcBin)
                                       ] )
            commands.append( AllianceCommand( self.alcBin, fdLog=self.fds['alliance'] ) )
        if self.doCoriolis:
            if not os.path.isfile( self.ccbBin ):
                raise ErrorMessage( 1, [ 'Cannot find <ccb.py>, should be here:'
                                       , '   "{}"'.format(self.ccbBin)
                                       ] )
            otherArgs = []
            if self.debugArg: otherArgs.append( self.debugArg )
            if target == 'SL7_64':
                otherArgs.append( '--project=support' )
                commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs          , fdLog=self.fds['coriolis'] ) )
                commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) )
            elif target == 'SL6_64' or target == 'SL6':
                otherArgs.append( '--project=support' )
                otherArgs.append( '--devtoolset=8' )
                commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 6, otherArgs          , fdLog=self.fds['coriolis'] ) )
                commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) )
            elif target == 'Ubuntu18' or target == 'Debian9' or target == 'Debian10':
                if target == 'Ubuntu18': otherArgs.append( '--qt5' )
                commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs, fdLog=self.fds['coriolis'] ) )
        if self.doBenchs:
            commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) )
        return commands


class Report ( object ):

    def __init__ ( self, conf ):
        self.conf = conf
        commaspace = ', '
        date       = time.strftime( "%A %d %B %Y" )
        stateText  = 'FAILED'
        modeText   = 'SoC installation'
        if self.conf.success:     stateText = 'SUCCESS'
        if self.conf.nightlyMode: modeText  = 'Nightly build'
        self.message = MIMEMultipart()
        self.message['Subject'] = '[{}] Coriolis {} {}'.format(stateText,modeText,date)
        self.message['From'   ] = self.conf.sender
        self.message['To'     ] = commaspace.join( self.conf.receivers )
        self.attachements = []
        self.mainText  = '\n'
        self.mainText += 'Salut le Crevard,\n'
        self.mainText += '\n'
        if self.conf.nightlyMode:
            self.mainText += 'This is the nightly build report of Coriolis.\n'
        else:
            self.mainText += 'SoC installer report of Coriolis.\n'
        self.mainText += '{}\n'.format(date)
        self.mainText += '\n'
        if self.conf.success:
            self.mainText += 'Build was SUCCESSFUL\n'
        else:
            self.mainText += 'Build has FAILED, please have a look to the attached log file(s).\n'
        self.mainText += '\n'
        self.mainText += 'Complete log file(s) can be found here:\n'
        return

    def attachLog ( self, logFile ):
        if not logFile: return
        
        fd = open( logFile, 'rb' )
        try:
            fd.seek( -1024*100, os.SEEK_END )
        except IOError as e:
            pass
        tailLines = ''
        for line in fd.readlines()[1:]:
            tailLines += line.decode( 'latin_1' )
        fd.close()
        self.mainText += '    "{}"\n'.format(logFile)
        
        attachement = MIMEApplication(tailLines)
        attachement.add_header( 'Content-Disposition', 'attachment', filename=os.path.basename(logFile) )
        self.attachements.append( attachement )
        return

    def send ( self ):
        self.message.attach( MIMEText(self.mainText) )
        for attachement in self.attachements:
            self.message.attach( attachement )
        
        print( "Sending mail report to:" )
        for receiver in self.conf.receivers: print( '  <{}>'.format(receiver) )
        session = smtplib.SMTP( 'localhost' )
        session.sendmail( self.conf.sender, self.conf.receivers, self.message.as_string() )
        session.quit()
        return


# ------------------------------------------------------------------- 
# <socInstaller> Main Part.


parser = optparse.OptionParser ()  
parser.add_option ( "--debug"       , action="store_true" ,                dest="debug"        , help="Build a <Debug> aka (-g) version." )
parser.add_option ( "--no-git"      , action="store_true" ,                dest="noGit"        , help="Do not pull/update Git repositories before building." )
parser.add_option ( "--do-yosys"    , action="store_true" ,                dest="doYosys"      , help="Rebuild Yosys." )
parser.add_option ( "--do-alliance" , action="store_true" ,                dest="doAlliance"   , help="Rebuild the Alliance tools." )
parser.add_option ( "--do-coriolis" , action="store_true" ,                dest="doCoriolis"   , help="Rebuild the Coriolis tools." )
parser.add_option ( "--do-report"   , action="store_true" ,                dest="doReport"     , help="Send a final report." )
parser.add_option ( "--nightly"     , action="store_true" ,                dest="nightly"      , help="Perform a nighly build." )
parser.add_option ( "--docker"      , action="store_true" ,                dest="docker"       , help="Perform a build inside a docker container." )
parser.add_option ( "--chroot"      , action="store_true" ,                dest="chroot"       , help="Perform a build inside a chrooted environment." )
parser.add_option ( "--benchs"      , action="store_true" ,                dest="benchs"       , help="Run the <alliance-checker-toolkit> sanity benchs." )
parser.add_option ( "--rm-build"    , action="store_true" ,                dest="rmBuild"      , help="Remove the build/install directories." )
parser.add_option ( "--rm-source"   , action="store_true" ,                dest="rmSource"     , help="Remove the Git source repositories." )
parser.add_option ( "--rm-all"      , action="store_true" ,                dest="rmAll"        , help="Remove everything (source+build+install)." )
parser.add_option ( "--root"        , action="store"      , type="string", dest="rootDir"      , help="The root directory (default: <~/coriolis-2.x/>)." )
parser.add_option ( "--profile"     , action="store"      , type="string", dest="profile"      , help="The targeted OS for the build." )
(options, args) = parser.parse_args ()


conf = Configuration()

try:
    if options.debug:                     conf.debugArg     = '--debug' 
    if options.nightly:                   conf.nightlyMode  = True
    if options.docker:                    conf.dockerMode   = True
    if options.chroot:                    conf.chrootMode   = True
    if options.noGit:                     conf.doGit        = False
    if options.doYosys:                   conf.doYosys      = True
    if options.doAlliance:                conf.doAlliance   = True
    if options.doCoriolis:                conf.doCoriolis   = True
    if options.benchs:                    conf.doBenchs     = True
    if options.doReport:                  conf.doSendReport = True
    if options.rmSource or options.rmAll: conf.rmSource     = True
    if options.rmBuild  or options.rmAll: conf.rmBuild      = True


    if conf.doYosys:    conf.openLog( 'yosys'    )
    if conf.doAlliance: conf.openLog( 'alliance' )
    if conf.doCoriolis: conf.openLog( 'coriolis' )
    if conf.doBenchs:   conf.openLog( 'benchs'   )
    if conf.dockerMode: os.environ['USER'] = 'root'

    gitSupports = []
    for supportRepo in conf.supportRepos:
        gitSupports.append( GitRepository( supportRepo, conf.srcDir+'/support' ) )
    gitCoriolis = GitRepository( conf.coriolisRepo, conf.srcDir, conf.fds['coriolis'] )
    gitBenchs   = GitRepository( conf.benchsRepo  , conf.srcDir, conf.fds['coriolis'] )

    if conf.doAlliance:
        gitAlliance = GitRepository( conf.allianceRepo, conf.srcDir, conf.fds['alliance'] )

    if conf.doGit:
        for gitSupport in gitSupports:
            if conf.rmSource: gitSupport.removeLocalRepo()
            gitSupport.clone()
            if gitSupport.url.endswith('rapidjson'):
              gitSupport.checkout( 'b1a4d91' )
        
            if conf.doAlliance:
                if conf.rmSource: gitAlliance.removeLocalRepo()
                gitAlliance.clone   ()
               #gitAlliance.checkout( 'devel' )
            
            if conf.doCoriolis:
                if conf.rmSource: gitCoriolis.removeLocalRepo()
                gitCoriolis.clone   ()
                gitCoriolis.checkout( 'devel' )
      
        if conf.rmSource: gitBenchs.removeLocalRepo()
        gitBenchs.clone()

    if conf.rmBuild:
        for entry in os.listdir(conf.rootDir):
            if entry.startswith('Linux.'):
                buildDir = conf.rootDir+'/'+entry
                print( 'Removing OS build directory: "{}"'.format(buildDir) )
                shutil.rmtree( buildDir )

    commands = conf.getCommands( options.profile )
    for command in commands:
        if command.host:
            print( 'Executing command on remote host "{}":'.format(host) )
        else:
            print( 'Executing command on *local* host:' )
        print( '  {}'.format(command) )
        command.execute()

    conf.closeLogs()
    conf.success = True

except ErrorMessage as e:
    print( e )
    conf.closeLogs()
    conf.success = False
    
    if showTrace:
        print( '\nPython stack trace:' )
        traceback.print_tb( sys.exc_info()[2] )
    conf.rcode = e.code

if conf.doSendReport:
    report = Report( conf )
    report.attachLog( conf.logs['coriolis' ] )
    report.attachLog( conf.logs['benchs'   ] )
    report.send()

conf.compressLogs()

sys.exit( conf.rcode )