coriolis/bootstrap/socInstaller.py

#!/usr/bin/env python
#
# -*- mode:Python -*-
#
# This file is part of the Coriolis Software.
# Copyright (c) UPMC 2015-2015, All Rights Reserved
#
# +-----------------------------------------------------------------+ 
# |                   C O R I O L I S                               |
# |  C o r i o l i s  /  C h a m s   I n s t a l l e r              |
# |                                                                 |
# |  Authors     :                    Jean-Paul Chaput              |
# |  E-mail      :       Jean-Paul.Chaput@asim.lip6.fr              |
# | =============================================================== |
# |  Python      :   "./socInstaller.py"                            |
# +-----------------------------------------------------------------+
#
# WARNING:
#   This script has been designed only for internal use in the
#   LIP6/SoC department. If you want to use it you will need to
#   change the hardwired configuration.


showTrace = True

try:
  import sys
  import os.path
  import shutil
  import optparse
  import time
  import traceback
  import distutils.sysconfig
  import subprocess
  import socket
  import re
  import smtplib
  from email.mime.text        import MIMEText
  from email.mime.multipart   import MIMEMultipart
  from email.mime.application import MIMEApplication
except ImportError, e:
  module = str(e).split()[-1]


class ErrorMessage ( Exception ):

    def __init__ ( self, code, *arguments ):
      self._code   = code
      self._errors = [ 'Malformed call to ErrorMessage()', '%s' % str(arguments) ]

      text = None
      if len(arguments) == 1:
        if isinstance(arguments[0],Exception): text = str(arguments[0]).split('\n')
        else:
          self._errors = arguments[0]
      elif len(arguments) > 1:
        text = list(arguments)

      if text:
        self._errors = []
        while len(text[0]) == 0: del text[0]

        lstrip = 0
        if text[0].startswith('[ERROR]'): lstrip = 8

        for line in text:
          if line[0:lstrip  ] == ' '*lstrip or \
             line[0:lstrip-1] == '[ERROR]':
            self._errors += [ line[lstrip:] ]
          else:
            self._errors += [ line.lstrip() ]
      return

    def __str__ ( self ):
        if not isinstance(self._errors,list):
          return "[ERROR] %s" % self._errors

        formatted = "\n"
        for i in range(len(self._errors)):
          if i == 0: formatted += "[ERROR] %s" % self._errors[i]
          else:      formatted += "        %s" % self._errors[i]
          if i+1 < len(self._errors): formatted += "\n"
        return formatted

    def addMessage ( self, message ):
        if not isinstance(self._errors,list):
          self._errors = [ self._errors ]
        if isinstance(message,list):
          for line in message:
              self._errors += [ line ]
        else:
          self._errors += [ message ]
        return

    def terminate ( self ):
        print self
        sys.exit(self._code)

    @property
    def code ( self ): return self._code


class BadBinary ( ErrorMessage ):

    def __init__ ( self, binary ):
      ErrorMessage.__init__( self, 1, "Binary not found: <%s>." % binary )
      return


class BadReturnCode ( ErrorMessage ):

    def __init__ ( self, status ):
      ErrorMessage.__init__( self, 1, "Command returned status:%d." % status )
      return


class Command ( object ):

    def __init__ ( self, arguments, fdLog=None ):
      self.arguments = arguments
      self.fdLog     = fdLog

      if self.fdLog != None and not isinstance(self.fdLog,file):
        print '[WARNING] Command.__init__(): <fdLog> is neither None or a file.'
      return

    def execute ( self ):
      sys.stdout.flush()
      sys.stderr.flush()

      try:
        child = subprocess.Popen( self.arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )

        while True:
          line = child.stdout.readline()
          if not line: break

          print line[:-1]
          sys.stdout.flush()

          if isinstance(self.fdLog,file):
            self.fdLog.write( line )
            self.fdLog.flush()
      except OSError, e:
        raise BadBinary( self.arguments[0] )

      (pid,status) = os.waitpid( child.pid, 0 )
      status >>= 8
      if status != 0:
        raise BadReturnCode( status )

      return


class GitRepository ( object ):

    @staticmethod
    def getLocalRepository ( gitRepository ):
      localRepo = gitRepository.split( '/' )[-1]
      if localRepo.endswith('.git'):
        localRepo = localRepo[:-4]
      return localRepo

    def __init__ ( self, url, cloneDir ):
      self.url       = url
      self.cloneDir  = cloneDir
      self.localRepo = GitRepository.getLocalRepository( url )
      return

    @property
    def localRepoDir ( self ): return self.cloneDir+'/'+self.localRepo

    def removeLocalRepo ( self ):
      if os.path.isdir(self.localRepoDir):
        print 'Removing Git local repository: <%s>' % self.localRepoDir
        shutil.rmtree( self.localRepoDir )
      return

    def clone ( self ):
      if not os.path.isdir(self.cloneDir):
        os.makedirs( self.cloneDir )

      if not os.path.isdir(self.localRepoDir):
        os.chdir( self.cloneDir )
        Command( [ 'git', 'clone', self.url ] ).execute()
      else:
        os.chdir( self.localRepoDir )
        Command( [ 'git', 'pull' ] ).execute()
      return

    def checkout ( self, branch ):
      os.chdir( self.localRepoDir )
      Command( [ 'git', 'checkout', branch ] ).execute()
      return


def openLog ( logDir, stem ):
    if not os.path.isdir(logDir):
      os.makedirs( logDir )

    index   = 0
    timeTag = time.strftime( "%Y.%m.%d" )
    while True:
        logFile = os.path.join(logDir,"%s-%s-%02d.log" % (stem,timeTag,index))
        if not os.path.isfile(logFile):
            print "Report log: <%s>" % logFile
            break
        index += 1
    fd = open( logFile, "w" )
    return (fd,logFile)


def sendReport ( state, buildLog, benchsLog, nightlyBuild ):
    sender   = 'Jean-Paul.Chaput@soc.lip6.fr'
    receiver = 'Jean-Paul.Chaput@lip6.fr'
    date     = time.strftime( "%A %d %B %Y" )

    stateText = 'FAILED'
    buildText = 'SoC installation'
    if state:        stateText = 'SUCCESS'
    if nightlyBuild: buildText = 'Nightly build'

    message = MIMEMultipart()
    message['Subject'] = '[%s] Coriolis & Chams %s %s' % (stateText,buildText,date)
    message['From'   ] = sender
    message['To'     ] = receiver

    mainText  = '\n'
    mainText += 'Salut le Crevard,\n'
    mainText += '\n'
    if nightlyBuild:
      mainText += 'This is the nightly build report of Coriolis & Chams.\n'
    else:
      mainText += 'SoC installer report of Coriolis & Chams.\n'
    mainText += '%s\n' % date
    mainText += '\n'
    if state:
      mainText += 'Build was SUCCESSFUL\n'
    else:
      mainText += 'Build has FAILED, please have a look to the attached log file.\n'
    mainText += '\n'
    mainText += 'Complete log file can be found here:\n'
    mainText += '    <%s>\n' % buildLog
    if benchsLog:
      mainText += '    <%s>\n' % benchsLog
    mainText += '\n'
    message.attach( MIMEText(mainText) )

    fd = open( buildLog, 'rb' )
    fd.seek( -1024*100, os.SEEK_END )
    tailLines = ''
    for line in fd.readlines()[1:]:
      tailLines += line
    message.attach( MIMEApplication(tailLines) )
    fd.close()

    if benchsLog:
      fd = open( benchsLog, 'rb' )
      fd.seek( -1024*100, os.SEEK_END )
      tailLines = ''
      for line in fd.readlines()[1:]:
        tailLines += line
      message.attach( MIMEApplication(tailLines) )
      fd.close()

    session = smtplib.SMTP( 'localhost' )
    session.sendmail( sender, receiver, message.as_string() )
    session.quit()
    return


def detectRunningHost ():
    runningHost = 'unknown'
    hostname    = socket.gethostname()
    hostAddr    = socket.gethostbyname(hostname)

    if hostname == 'lepka' and hostAddr == '127.0.0.1':
        print 'Running on <lepka>, watchout mode enabled.'
        runningHost = 'lepka'
    else:
        runningHost = hostname.split('.')[0]

    return runningHost


# ------------------------------------------------------------------- 
# <socInstaller> Main Part.


parser = optparse.OptionParser ()  
parser.add_option ( "--debug"       , action="store_true" ,                dest="debug"        , help="Build a <Debug> aka (-g) version." )
parser.add_option ( "--no-git"      , action="store_true" ,                dest="noGit"        , help="Do not pull/update Git repositories before building." )
parser.add_option ( "--no-report"   , action="store_true" ,                dest="noReport"     , help="Do not send a final report." )
parser.add_option ( "--nightly"     , action="store_true" ,                dest="nightly"      , help="Perform a nighly build." )
parser.add_option ( "--benchs"      , action="store_true" ,                dest="benchs"       , help="Run the <alliance-checker-toolkit> sanity benchs." )
parser.add_option ( "--rm-build"    , action="store_true" ,                dest="rmBuild"      , help="Remove the build/install directories." )
parser.add_option ( "--rm-source"   , action="store_true" ,                dest="rmSource"     , help="Remove the Git source repositories." )
parser.add_option ( "--rm-all"      , action="store_true" ,                dest="rmAll"        , help="Remove everything (source+build+install)." )
parser.add_option ( "--root"        , action="store"      , type="string", dest="rootDir"      , help="The root directory (default: <~/coriolis-2.x/>)." )
(options, args) = parser.parse_args ()

nightlyBuild  = False
fdBuildLog    = None
fdBenchsLog   = None
buildLogFile  = None
benchsLogFile = None

try:
    rmSource     = False
    rmBuild      = False
    doGit        = True
    doBench      = False
    doSendReport = True
    debugArg     = ''
    runningHost  = detectRunningHost()
    targetSL6    = 'rock'
    targetSL6_64 = 'bip'
    targetSL7_64 = None

    if options.debug:    debugArg     = '--debug' 
    if options.nightly:  nightlyBuild = True
    if options.noGit:    doGit        = False
    if options.benchs:   doBenchs     = True
    if options.noReport: doSendReport = False
    if options.rmSource or options.rmAll: rmSource = True
    if options.rmBuild  or options.rmAll: rmBuild  = True

    if runningHost == 'lepka':
      print 'Never touch the Git tree when running on <lepka>.'
      doGit        = False
      rmSource     = False
      rmBuild      = False
      targetSL7_64 = 'lepka'
      targetSL6    = None
      targetSL6_64 = None

    if nightlyBuild:
      targetSL6    = None

    allianceCheckRepo = 'https://www-soc.lip6.fr/git/alliance-check-toolkit.git'
    coriolisRepo      = 'https://www-soc.lip6.fr/git/coriolis.git'
    chamsRepo         = 'file:///users/outil/chams/chams.git'
    homeDir           = os.environ['HOME']
    rootDir           = homeDir + '/coriolis-2.x'
    if nightlyBuild:
      rootDir         = homeDir + '/nightly/coriolis-2.x'
    srcDir            = rootDir + '/src'
    logDir            = srcDir  + '/logs'

    gitCoriolis      = GitRepository( coriolisRepo     , srcDir )
    gitChams         = GitRepository( chamsRepo        , srcDir )
    gitAllianceCheck = GitRepository( allianceCheckRepo, srcDir )

    if doGit:
      if rmSource: gitCoriolis.removeLocalRepo()
      gitCoriolis.clone   ()
      gitCoriolis.checkout( 'devel' )
      
      if rmSource: gitChams.removeLocalRepo()
      gitChams.clone   ()
      gitChams.checkout( 'devel' )

      if rmSource: AllianceCheck.removeLocalRepo()
      AllianceCheck.clone   ()

    if rmBuild:
      for entry in os.listdir(rootDir):
        if entry.startswith('Linux.'):
          buildDir = rootDir+'/'+entry
          print 'Removing OS build directory: <%s>' % buildDir
          shutil.rmtree( buildDir )

    ccbBin = gitCoriolis.localRepoDir+'/bootstrap/ccb.py'
    if not os.path.isfile( ccbBin ):
      raise ErrorMessage( 1, [ 'Cannot find <ccb.py>, should be here:'
                             , '   <%s>' % ccbBin
                             ] )

    fdBuildLog,buildLogFile = openLog( logDir, 'build' )
    if doBenchs:
      fdBenchsLog,benchsLogFile = openLog( logDir, 'benchs' )

    buildCommand  = '%s --root=%s --project=coriolis --project=chams --make="-j%%d install" %%s' \
                     % (ccbBin,rootDir)
    benchsCommand = 'cd %s/benchs && ./bin/go.sh clean && ./bin/go.sh lvx' \
                     % (gitAllianceCheck.localRepoDir)

    if targetSL7_64:
      commands = [ ( targetSL7_64, buildCommand % (3,debugArg)         , fdBuildLog )
                 , ( targetSL7_64, buildCommand % (1,debugArg+' --doc'), fdBuildLog )
                 ]
      if doBenchs:
        commands += [ ( targetSL7_64, benchsCommand, fdBenchsLog ) ]
    if targetSL6_64:
      commands = [ ( targetSL6_64, buildCommand % (6,debugArg+' --devtoolset-2')      , fdBuildLog )
                 , ( targetSL6_64, buildCommand % (1,debugArg+' --devtoolset-2 --doc'), fdBuildLog )
                 ]
      if doBenchs:
        commands += [ ( targetSL6_64, benchsCommand, fdBenchsLog ) ]
    if targetSL6:
      commands = [ ( targetSL6, buildCommand % (2,debugArg+' --devtoolset-2')      , fdBuildLog )
                 , ( targetSL6, buildCommand % (1,debugArg+' --devtoolset-2 --doc'), fdBuildLog )
                 ]
      if doBenchs:
        commands += [ ( targetSL6, benchsCommand, fdBenchsLog ) ]

    for host,command,fd in commands:
      Command( [ 'ssh', host, command ], fd ).execute()

    fdBuildLog.close()
    if doBenchs: fdBenchsLog.close()

    if doSendReport:
      sendReport( True, buildLogFile, benchsLogFile, nightlyBuild )

except ErrorMessage, e:
  print e
  if fdBuildLog:  fdBuildLog.close()
  if fdBenchsLog: fdBenchsLog.close()
  if showTrace:
      print '\nPython stack trace:'
      traceback.print_tb( sys.exc_info()[2] )
  sendReport( False, buildLogFile, benchsLogFile, nightlyBuild )
  sys.exit( e.code )

sys.exit( 0 )