diff --git a/bootstrap/allianceInstaller.sh b/bootstrap/allianceInstaller.sh new file mode 100755 index 00000000..71f02906 --- /dev/null +++ b/bootstrap/allianceInstaller.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + srcDir=${HOME}/coriolis-2.x/src/alliance/alliance/src + commonRoot=${HOME}/coriolis-2.x/Linux.el7_64/Release.Shared + buildDir=${commonRoot}/build + installDir=${commonRoot}/install + + export ALLIANCE_TOP=${installDir} + export LD_LIBRARY_PATH=${installDir}/lib:${LD_LIBRARY_PATH} + + cd ${srcDir} + # Skip doc generation to avoid pulling TeXLive in docker images. + sed -i 's,dirs="\$newdirs documentation",dirs="$newdirs",' ./autostuff + ./autostuff clean + ./autostuff + mkdir -p ${buildDir} + cd ${buildDir} + ${srcDir}/configure --prefix=${ALLIANCE_TOP} --enable-alc-shared + make -j1 install diff --git a/bootstrap/docker/debian-9/Dockerfile.coriolis b/bootstrap/docker/debian-9/Dockerfile.coriolis new file mode 100644 index 00000000..85d96956 --- /dev/null +++ b/bootstrap/docker/debian-9/Dockerfile.coriolis @@ -0,0 +1,7 @@ + +FROM debian9.system + +COPY root/socInstaller.py /root/socInstaller.py +COPY root/allianceInstaller.sh /root/allianceInstaller.sh +CMD /root/socInstaller.py --docker --profile=Debian9 --do-alliance --do-coriolis --benchs + diff --git a/bootstrap/docker/debian-9/Dockerfile.system b/bootstrap/docker/debian-9/Dockerfile.system new file mode 100644 index 00000000..f1cdce53 --- /dev/null +++ b/bootstrap/docker/debian-9/Dockerfile.system @@ -0,0 +1,20 @@ + +FROM debian:stretch-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get -y install build-essential binutils-dev \ + git cmake bison flex gcc python-dev \ + libboost-all-dev libboost-python-dev \ + zlib1g-dev libxml2-dev rapidjson-dev libbz2-dev \ + qt4-dev-tools libqwt-dev python-qt4 \ + \ + autotools-dev automake \ + libxt-dev libxpm-dev libmotif-dev \ + \ + yosys \ + \ + vim \ + && apt-get clean + diff --git a/bootstrap/docker/debian-9/docker-conf.sh b/bootstrap/docker/debian-9/docker-conf.sh new file mode 100644 index 00000000..aafb4757 --- /dev/null +++ b/bootstrap/docker/debian-9/docker-conf.sh @@ -0,0 +1,3 @@ + + systemImage="debian9.system" + coriolisImage="debian9.coriolis" diff --git a/bootstrap/docker/debian-9/root/allianceInstaller.sh b/bootstrap/docker/debian-9/root/allianceInstaller.sh new file mode 100755 index 00000000..71f02906 --- /dev/null +++ b/bootstrap/docker/debian-9/root/allianceInstaller.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + srcDir=${HOME}/coriolis-2.x/src/alliance/alliance/src + commonRoot=${HOME}/coriolis-2.x/Linux.el7_64/Release.Shared + buildDir=${commonRoot}/build + installDir=${commonRoot}/install + + export ALLIANCE_TOP=${installDir} + export LD_LIBRARY_PATH=${installDir}/lib:${LD_LIBRARY_PATH} + + cd ${srcDir} + # Skip doc generation to avoid pulling TeXLive in docker images. + sed -i 's,dirs="\$newdirs documentation",dirs="$newdirs",' ./autostuff + ./autostuff clean + ./autostuff + mkdir -p ${buildDir} + cd ${buildDir} + ${srcDir}/configure --prefix=${ALLIANCE_TOP} --enable-alc-shared + make -j1 install diff --git a/bootstrap/docker/debian-9/root/socInstaller.py b/bootstrap/docker/debian-9/root/socInstaller.py new file mode 100755 index 00000000..79b0e406 --- /dev/null +++ b/bootstrap/docker/debian-9/root/socInstaller.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python +# +# -*- mode:Python -*- +# +# This file is part of the Coriolis Software. +# Copyright (c) UPMC 2015-2018, All Rights Reserved +# +# +-----------------------------------------------------------------+ +# | C O R I O L I S | +# | C o r i o l i s I n s t a l l e r | +# | | +# | Authors : Jean-Paul Chaput | +# | E-mail : Jean-Paul.Chaput@asim.lip6.fr | +# | =============================================================== | +# | Python : "./socInstaller.py" | +# +-----------------------------------------------------------------+ +# +# WARNING: +# This script has been designed only for internal use in the +# LIP6/CIAN department. If you want to use it you will need to +# change the hardwired configuration. + + +showTrace = True + +try: + import sys + import os.path + import shutil + import optparse + import time + import traceback + import distutils.sysconfig + import subprocess + import socket + import re + import bz2 + import smtplib + from email.mime.text import MIMEText + from email.mime.multipart import MIMEMultipart + from email.mime.application import MIMEApplication +except ImportError, e: + module = str(e).split()[-1] + + +class ErrorMessage ( Exception ): + + def __init__ ( self, code, *arguments ): + self._code = code + self._errors = [ 'Malformed call to ErrorMessage()', '%s' % str(arguments) ] + + text = None + if len(arguments) == 1: + if isinstance(arguments[0],Exception): text = str(arguments[0]).split('\n') + else: + self._errors = arguments[0] + elif len(arguments) > 1: + text = list(arguments) + + if text: + self._errors = [] + while len(text[0]) == 0: del text[0] + + lstrip = 0 + if text[0].startswith('[ERROR]'): lstrip = 8 + + for line in text: + if line[0:lstrip ] == ' '*lstrip or \ + line[0:lstrip-1] == '[ERROR]': + self._errors += [ line[lstrip:] ] + else: + self._errors += [ line.lstrip() ] + return + + def __str__ ( self ): + if not isinstance(self._errors,list): + return "[ERROR] %s" % self._errors + + formatted = "\n" + for i in range(len(self._errors)): + if i == 0: formatted += "[ERROR] %s" % self._errors[i] + else: formatted += " %s" % self._errors[i] + if i+1 < len(self._errors): formatted += "\n" + return formatted + + def addMessage ( self, message ): + if not isinstance(self._errors,list): + self._errors = [ self._errors ] + if isinstance(message,list): + for line in message: + self._errors += [ line ] + else: + self._errors += [ message ] + return + + def terminate ( self ): + print self + sys.exit(self._code) + + @property + def code ( self ): return self._code + + +class BadBinary ( ErrorMessage ): + + def __init__ ( self, binary ): + ErrorMessage.__init__( self, 1, "Binary not found: <%s>." % binary ) + return + + +class BadReturnCode ( ErrorMessage ): + + def __init__ ( self, status ): + ErrorMessage.__init__( self, 1, "Command returned status:%d." % status ) + return + + +class Command ( object ): + + def __init__ ( self, arguments, fdLog=None ): + self.arguments = arguments + self.fdLog = fdLog + + if self.fdLog != None and not isinstance(self.fdLog,file): + print '[WARNING] Command.__init__(): is neither None or a file.' + return + + def _argumentsToStr ( self, arguments ): + s = '' + for argument in arguments: + if argument.find(' ') >= 0: s += ' "' + argument + '"' + else: s += ' ' + argument + return s + + def log ( self, text ): + print text[:-1] + sys.stdout.flush() + sys.stderr.flush() + if isinstance(self.fdLog,file): + self.fdLog.write( text ) + self.fdLog.flush() + return + + def execute ( self ): + global conf + sys.stdout.flush() + sys.stderr.flush() + + homeDir = os.environ['HOME'] + workDir = os.getcwd() + if homeDir.startswith(homeDir): + workDir = '~' + workDir[ len(homeDir) : ] + user = 'root' + if os.environ.has_key('USER'): user = os.environ['USER'] + prompt = '%s@%s:%s$' % (user,conf.masterHost,workDir) + + try: + self.log( '%s%s\n' % (prompt,self._argumentsToStr(self.arguments)) ) + print self.arguments + child = subprocess.Popen( self.arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + + while True: + line = child.stdout.readline() + if not line: break + + self.log( line ) + except OSError, e: + raise BadBinary( self.arguments[0] ) + + (pid,status) = os.waitpid( child.pid, 0 ) + status >>= 8 + if status != 0: + raise BadReturnCode( status ) + + return + + +class CommandArg ( object ): + + def __init__ ( self, command, wd=None, host=None, fdLog=None ): + self.command = command + self.host = host + self.wd = wd + self.fdLog = fdLog + return + + def __str__ ( self ): + s = '' + if self.wd: s = 'cd %s && ' % self.wd + + for i in range(len(self.command)): + if i: s += ' ' + s += self.command[i] + return s + + def getArgs ( self ): + if not self.host: return self.command + return [ 'ssh', self.host, str(self) ] + + def execute ( self ): + if not self.host and self.wd: os.chdir( self.wd ) + Command( self.getArgs(), self.fdLog ).execute() + return + + +class AllianceCommand ( CommandArg ): + + def __init__ ( self, fdLog=None ): + CommandArg.__init__ ( self, [ '/root/allianceInstaller.sh' ] + , fdLog=fdLog ) + return + + +class CoriolisCommand ( CommandArg ): + + def __init__ ( self, ccbBin, rootDir, threads=1, otherArgs=[], fdLog=None ): + CommandArg.__init__ ( self, [ ccbBin + , '--root='+rootDir + , '--project=coriolis' + , '--make=-j%d install' % threads + ] + otherArgs + , fdLog=fdLog ) + return + + +class BenchsCommand ( CommandArg ): + + def __init__ ( self, benchsDir, fdLog=None ): + CommandArg.__init__ ( self, [ '../bin/go.sh' ], wd=benchsDir, fdLog=fdLog ) + return + + + +class GitRepository ( object ): + + @staticmethod + def getLocalRepository ( url ): + localRepo = url.split( '/' )[-1] + if localRepo.endswith('.git'): + localRepo = localRepo[:-4] + return localRepo + + def __init__ ( self, url, cloneDir, fdLog=None ): + self.url = url + self.cloneDir = cloneDir + self.localRepo = GitRepository.getLocalRepository( url ) + self.fdLog = fdLog + return + + @property + def localRepoDir ( self ): return self.cloneDir+'/'+self.localRepo + + def removeLocalRepo ( self ): + if os.path.isdir(self.localRepoDir): + print 'Removing Git local repository: <%s>' % self.localRepoDir + shutil.rmtree( self.localRepoDir ) + return + + def clone ( self ): + print 'Clone/pull from:', self.url + if not os.path.isdir(self.cloneDir): + os.makedirs( self.cloneDir ) + + if not os.path.isdir(self.localRepoDir): + os.chdir( self.cloneDir ) + Command( [ 'git', 'clone', self.url ], self.fdLog ).execute() + else: + os.chdir( self.localRepoDir ) + Command( [ 'git', 'pull' ], self.fdLog ).execute() + return + + def checkout ( self, branch ): + os.chdir( self.localRepoDir ) + Command( [ 'git', 'checkout', branch ], self.fdLog ).execute() + return + + +class Configuration ( object ): + + PrimaryNames = \ + [ 'sender' , 'receivers' + , 'coriolisRepo', 'benchsRepo' , 'supportRepos' + , 'homeDir' , 'masterHost' + , 'debugArg' , 'nightlyMode', 'dockerMode' + , 'rmSource' , 'rmBuild' + , 'doGit' , 'doAlliance' , 'doCoriolis', 'doBenchs', 'doSendReport' + , 'success' , 'rcode' + ] + SecondaryNames = \ + [ 'rootDir', 'srcDir', 'logDir', 'logs', 'fds', 'ccbBin', 'benchsDir' + ] + + def __init__ ( self ): + self._sender = 'Jean-Paul.Chaput@soc.lip6.fr' + self._receivers = [ 'Jean-Paul.Chaput@lip6.fr', ] + self._supportRepos = [ 'http://github.com/miloyip/rapidjson' ] + self._allianceRepo = 'https://gitlab.lip6.fr/jpc/alliance.git' + self._coriolisRepo = 'https://gitlab.lip6.fr/jpc/coriolis.git' + self._benchsRepo = 'https://gitlab.lip6.fr/jpc/alliance-check-toolkit.git' + self._homeDir = os.environ['HOME'] + self._debugArg = '' + self._rmSource = False + self._rmBuild = False + self._doGit = True + self._doCoriolis = False + self._doAlliance = False + self._doBenchs = False + self._doSendReport = False + self._nightlyMode = False + self._dockerMode = False + self._logs = { 'alliance':None, 'coriolis':None, 'benchs':None } + self._fds = { 'alliance':None, 'coriolis':None, 'benchs':None } + self._ccbBin = None + self._benchsDir = None + self._masterHost = self._detectMasterHost() + self._success = False + self._rcode = 0 + + self._updateSecondaries() + return + + def __setattr__ ( self, attribute, value ): + if attribute in Configuration.SecondaryNames: + print ErrorMessage( 1, 'Attempt to write in read-only attribute <%s> in Configuration.'%attribute ) + return + + if attribute == 'masterHost' or attribute == '_masterHost': + if value == 'lepka': + print 'Never touch the Git tree when running on .' + self._rmSource = False + self._rmBuild = False + self._doGit = False + self._doSendReport = False + + if attribute[0] == '_': + self.__dict__[attribute] = value + return + + if attribute == 'homeDir': value = os.path.expanduser(value) + + self.__dict__['_'+attribute] = value + self._updateSecondaries() + return + + def __getattr__ ( self, attribute ): + if attribute[0] != '_': attribute = '_'+attribute + if not self.__dict__.has_key(attribute): + raise ErrorMessage( 1, 'Configuration has no attribute <%s>.'%attribute ) + return self.__dict__[attribute] + + def _updateSecondaries ( self ): + if self._nightlyMode: + self._rootDir = self._homeDir + '/nightly/coriolis-2.x' + else: + self._rootDir = self._homeDir + '/coriolis-2.x' + self._srcDir = self._rootDir + '/src' + self._logDir = self._srcDir + '/logs' + self._ccbBin = self._srcDir + '/' + GitRepository.getLocalRepository(self._coriolisRepo) + '/bootstrap/ccb.py' + self._benchsDir = self._srcDir + '/' + GitRepository.getLocalRepository(self._benchsRepo ) + '/benchs' + return + + def _detectMasterHost ( self ): + masterHost = 'unknown' + hostname = socket.gethostname() + hostAddr = socket.gethostbyname(hostname) + + if hostname == 'lepka' and hostAddr == '127.0.0.1': + masterHost = 'lepka' + else: + masterHost = hostname.split('.')[0] + return masterHost + + def openLog ( self, stem ): + if not os.path.isdir(self._logDir): + os.makedirs( self._logDir ) + + index = 0 + timeTag = time.strftime( "%Y.%m.%d" ) + while True: + logFile = os.path.join(self._logDir,"%s-%s-%02d.log" % (stem,timeTag,index)) + if not os.path.isfile(logFile): + print "Report log: <%s>" % logFile + break + index += 1 + fd = open( logFile, "w" ) + self._logs[stem] = logFile + self._fds [stem] = fd + return + + def closeLogs ( self ): + for fd in self._fds.values(): + if fd: fd.close() + return + + def compressLogs ( self ): + for log in self._logs.values(): + if not log: continue + + fd = open( log, 'r' ) + bzfd = bz2.BZ2File( log+'.bz2', 'w' ) + + for line in fd.readlines(): bzfd.write( line ) + + bzfd.close() + fd.close() + + os.unlink( log ) + return + + def getCommands ( self, target ): + commands = [] + + if self.doAlliance: + commands.append( AllianceCommand( fdLog=self.fds['alliance'] ) ) + + if self.doCoriolis: + if not os.path.isfile( self.ccbBin ): + raise ErrorMessage( 1, [ 'Cannot find , should be here:' + , ' <%s>' % self.ccbBin + ] ) + + otherArgs = [] + if self.debugArg: otherArgs.append( self.debugArg ) + + if target == 'SL7_64': + otherArgs.append( '--project=support' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs , fdLog=self.fds['coriolis'] ) ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand ( self.benchsDir, fdLog=self.fds['benchs'] ) ) + elif target == 'SL6_64' or target == 'SL6': + otherArgs.append( '--project=support' ) + otherArgs.append( '--devtoolset=8' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 6, otherArgs , fdLog=self.fds['coriolis'] ) ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) + elif target == 'Ubuntu18' or target == 'Debian9': + if target == 'Ubuntu18': otherArgs.append( '--qt5' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs, fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) + return commands + + +class Report ( object ): + + def __init__ ( self, conf ): + self.conf = conf + + commaspace = ', ' + date = time.strftime( "%A %d %B %Y" ) + stateText = 'FAILED' + modeText = 'SoC installation' + if self.conf.success: stateText = 'SUCCESS' + if self.conf.nightlyMode: modeText = 'Nightly build' + + self.message = MIMEMultipart() + self.message['Subject'] = '[%s] Coriolis %s %s' % (stateText,modeText,date) + self.message['From' ] = self.conf.sender + self.message['To' ] = commaspace.join( self.conf.receivers ) + self.attachements = [] + + self.mainText = '\n' + self.mainText += 'Salut le Crevard,\n' + self.mainText += '\n' + if self.conf.nightlyMode: + self.mainText += 'This is the nightly build report of Coriolis.\n' + else: + self.mainText += 'SoC installer report of Coriolis.\n' + self.mainText += '%s\n' % date + self.mainText += '\n' + if self.conf.success: + self.mainText += 'Build was SUCCESSFUL\n' + else: + self.mainText += 'Build has FAILED, please have a look to the attached log file(s).\n' + self.mainText += '\n' + self.mainText += 'Complete log file(s) can be found here:\n' + return + + def attachLog ( self, logFile ): + if not logFile: return + + fd = open( logFile, 'rb' ) + try: + fd.seek( -1024*100, os.SEEK_END ) + except IOError, e: + pass + tailLines = '' + for line in fd.readlines()[1:]: + tailLines += line + fd.close() + self.mainText += ' <%s>\n' % logFile + + attachement = MIMEApplication(tailLines) + attachement.add_header( 'Content-Disposition', 'attachment', filename=os.path.basename(logFile) ) + + self.attachements.append( attachement ) + return + + def send ( self ): + self.message.attach( MIMEText(self.mainText) ) + for attachement in self.attachements: + self.message.attach( attachement ) + + print "Sending mail report to:" + for receiver in self.conf.receivers: print ' <%s>' % receiver + session = smtplib.SMTP( 'localhost' ) + session.sendmail( self.conf.sender, self.conf.receivers, self.message.as_string() ) + session.quit() + return + + +# ------------------------------------------------------------------- +# Main Part. + + +parser = optparse.OptionParser () +parser.add_option ( "--debug" , action="store_true" , dest="debug" , help="Build a aka (-g) version." ) +parser.add_option ( "--no-git" , action="store_true" , dest="noGit" , help="Do not pull/update Git repositories before building." ) +parser.add_option ( "--do-alliance" , action="store_true" , dest="doAlliance" , help="Rebuild the Alliance tools." ) +parser.add_option ( "--do-coriolis" , action="store_true" , dest="doCoriolis" , help="Rebuild the Coriolis tools." ) +parser.add_option ( "--do-report" , action="store_true" , dest="doReport" , help="Send a final report." ) +parser.add_option ( "--nightly" , action="store_true" , dest="nightly" , help="Perform a nighly build." ) +parser.add_option ( "--docker" , action="store_true" , dest="docker" , help="Perform a build inside a docker container." ) +parser.add_option ( "--benchs" , action="store_true" , dest="benchs" , help="Run the sanity benchs." ) +parser.add_option ( "--rm-build" , action="store_true" , dest="rmBuild" , help="Remove the build/install directories." ) +parser.add_option ( "--rm-source" , action="store_true" , dest="rmSource" , help="Remove the Git source repositories." ) +parser.add_option ( "--rm-all" , action="store_true" , dest="rmAll" , help="Remove everything (source+build+install)." ) +parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" , help="The root directory (default: <~/coriolis-2.x/>)." ) +parser.add_option ( "--profile" , action="store" , type="string", dest="profile" , help="The targeted OS for the build." ) +(options, args) = parser.parse_args () + +conf = Configuration() + +try: + if options.debug: conf.debugArg = '--debug' + if options.nightly: conf.nightlyMode = True + if options.docker: conf.dockerMode = True + if options.noGit: conf.doGit = False + if options.doCoriolis: conf.doCoriolis = True + if options.doAlliance: conf.doAlliance = True + if options.benchs: conf.doBenchs = True + if options.doReport: conf.doSendReport = True + if options.rmSource or options.rmAll: conf.rmSource = True + if options.rmBuild or options.rmAll: conf.rmBuild = True + + if conf.doAlliance: conf.openLog( 'alliance' ) + if conf.doCoriolis: conf.openLog( 'coriolis' ) + if conf.doBenchs: conf.openLog( 'benchs' ) + + if conf.dockerMode: os.environ['USER'] = 'root' + + gitSupports = [] + for supportRepo in conf.supportRepos: + gitSupports.append( GitRepository( supportRepo, conf.srcDir+'/support' ) ) + gitCoriolis = GitRepository( conf.coriolisRepo, conf.srcDir, conf.fds['coriolis'] ) + gitBenchs = GitRepository( conf.benchsRepo , conf.srcDir, conf.fds['coriolis'] ) + + if conf.doAlliance: + gitAlliance = GitRepository( conf.allianceRepo, conf.srcDir, conf.fds['alliance'] ) + + if conf.doGit: + for gitSupport in gitSupports: + if conf.rmSource: gitSupport.removeLocalRepo() + gitSupport.clone() + #if gitSupport.url.endswith('rapidjson'): + # gitSupport.checkout( 'a1c4f32' ) + + if conf.doCoriolis: + if conf.rmSource: gitCoriolis.removeLocalRepo() + gitCoriolis.clone () + gitCoriolis.checkout( 'devel' ) + + if conf.doAlliance: + if conf.rmSource: gitAlliance.removeLocalRepo() + gitAlliance.clone () + #gitAlliance.checkout( 'devel' ) + + if conf.rmSource: gitBenchs.removeLocalRepo() + gitBenchs.clone() + + if conf.rmBuild: + for entry in os.listdir(conf.rootDir): + if entry.startswith('Linux.'): + buildDir = conf.rootDir+'/'+entry + print 'Removing OS build directory: <%s>' % buildDir + shutil.rmtree( buildDir ) + + commands = conf.getCommands( options.profile ) + for command in commands: + if command.host: + print 'Executing command on remote host <%s>:' % host + else: + print 'Executing command on *local* host:' + print ' %s' % str(command) + command.execute() + + conf.closeLogs() + + conf.success = True + +except ErrorMessage, e: + print e + conf.closeLogs() + conf.success = False + + if showTrace: + print '\nPython stack trace:' + traceback.print_tb( sys.exc_info()[2] ) + conf.rcode = e.code + +if conf.doSendReport: + report = Report( conf ) + report.attachLog( conf.logs['coriolis' ] ) + report.attachLog( conf.logs['benchs' ] ) + report.send() + +conf.compressLogs() + +sys.exit( conf.rcode ) diff --git a/bootstrap/docker/scientificlinux-7/Dockerfile.coriolis b/bootstrap/docker/scientificlinux-7/Dockerfile.coriolis new file mode 100644 index 00000000..ea68dc3b --- /dev/null +++ b/bootstrap/docker/scientificlinux-7/Dockerfile.coriolis @@ -0,0 +1,7 @@ + +FROM sl7.system + +COPY root/socInstaller.py /root/socInstaller.py +COPY root/allianceInstaller.sh /root/allianceInstaller.sh +CMD /root/socInstaller.py --docker --profile=SL7_64 --do-alliance --do-coriolis --benchs + diff --git a/bootstrap/docker/scientificlinux-7/Dockerfile.system b/bootstrap/docker/scientificlinux-7/Dockerfile.system new file mode 100644 index 00000000..2baa4562 --- /dev/null +++ b/bootstrap/docker/scientificlinux-7/Dockerfile.system @@ -0,0 +1,21 @@ + +FROM scientificlinux/sl:latest + +RUN yum -y update \ + && yum -y install git cmake bison flex gcc-c++ libstdc++-devel \ + make binutils-devel \ + boost-devel boost-python boost-filesystem \ + boost-regex boost-wave \ + python-devel libxml2-devel bzip2-devel \ + qt-devel PyQt4 \ + \ + autoconf automake libtool \ + libX11-devel libXt-devel libXpm-devel \ + motif motif-devel \ + \ + vim-x11 \ + && yum -y install http://ftp.lip6.fr/pub/linux/distributions/slsoc/soc/7/addons/x86_64/RPMS/qwt-6.1.2-4.fc23.x86_64.rpm \ + http://ftp.lip6.fr/pub/linux/distributions/slsoc/soc/7/addons/x86_64/RPMS/qwt-devel-6.1.2-4.fc23.x86_64.rpm \ + http://ftp.lip6.fr/pub/linux/distributions/slsoc/soc/7/addons/x86_64/RPMS/yosys-0.7-1.el7.soc.x86_64.rpm \ + && yum clean all + diff --git a/bootstrap/docker/scientificlinux-7/docker-conf.sh b/bootstrap/docker/scientificlinux-7/docker-conf.sh new file mode 100644 index 00000000..f32eaab0 --- /dev/null +++ b/bootstrap/docker/scientificlinux-7/docker-conf.sh @@ -0,0 +1,3 @@ + + systemImage="sl7.system" + coriolisImage="sl7.coriolis" diff --git a/bootstrap/docker/scientificlinux-7/root/allianceInstaller.sh b/bootstrap/docker/scientificlinux-7/root/allianceInstaller.sh new file mode 100755 index 00000000..4083c207 --- /dev/null +++ b/bootstrap/docker/scientificlinux-7/root/allianceInstaller.sh @@ -0,0 +1,18 @@ +#!/bin/bash + + srcDir=${HOME}/coriolis-2.x/src/alliance/alliance/src + commonRoot=${HOME}/coriolis-2.x/Linux.el7_64/Release.Shared + buildDir=${commonRoot}/build + installDir=${commonRoot}/install + + export ALLIANCE_TOP=${installDir} + export LD_LIBRARY_PATH=${installDir}/lib:${LD_LIBRARY_PATH} + + cd ${srcDir} + sed -i 's,dirs="\$newdirs documentation",dirs="$newdirs",' ./autostuff + ./autostuff clean + ./autostuff + mkdir -p ${buildDir} + cd ${buildDir} + ${srcDir}/configure --prefix=${ALLIANCE_TOP} --enable-alc-shared + make -j1 install diff --git a/bootstrap/docker/scientificlinux-7/root/socInstaller.py b/bootstrap/docker/scientificlinux-7/root/socInstaller.py new file mode 100755 index 00000000..6a7ab963 --- /dev/null +++ b/bootstrap/docker/scientificlinux-7/root/socInstaller.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python +# +# -*- mode:Python -*- +# +# This file is part of the Coriolis Software. +# Copyright (c) UPMC 2015-2018, All Rights Reserved +# +# +-----------------------------------------------------------------+ +# | C O R I O L I S | +# | C o r i o l i s I n s t a l l e r | +# | | +# | Authors : Jean-Paul Chaput | +# | E-mail : Jean-Paul.Chaput@asim.lip6.fr | +# | =============================================================== | +# | Python : "./socInstaller.py" | +# +-----------------------------------------------------------------+ +# +# WARNING: +# This script has been designed only for internal use in the +# LIP6/CIAN department. If you want to use it you will need to +# change the hardwired configuration. + + +showTrace = True + +try: + import sys + import os.path + import shutil + import optparse + import time + import traceback + import distutils.sysconfig + import subprocess + import socket + import re + import bz2 + import smtplib + from email.mime.text import MIMEText + from email.mime.multipart import MIMEMultipart + from email.mime.application import MIMEApplication +except ImportError, e: + module = str(e).split()[-1] + + +class ErrorMessage ( Exception ): + + def __init__ ( self, code, *arguments ): + self._code = code + self._errors = [ 'Malformed call to ErrorMessage()', '%s' % str(arguments) ] + + text = None + if len(arguments) == 1: + if isinstance(arguments[0],Exception): text = str(arguments[0]).split('\n') + else: + self._errors = arguments[0] + elif len(arguments) > 1: + text = list(arguments) + + if text: + self._errors = [] + while len(text[0]) == 0: del text[0] + + lstrip = 0 + if text[0].startswith('[ERROR]'): lstrip = 8 + + for line in text: + if line[0:lstrip ] == ' '*lstrip or \ + line[0:lstrip-1] == '[ERROR]': + self._errors += [ line[lstrip:] ] + else: + self._errors += [ line.lstrip() ] + return + + def __str__ ( self ): + if not isinstance(self._errors,list): + return "[ERROR] %s" % self._errors + + formatted = "\n" + for i in range(len(self._errors)): + if i == 0: formatted += "[ERROR] %s" % self._errors[i] + else: formatted += " %s" % self._errors[i] + if i+1 < len(self._errors): formatted += "\n" + return formatted + + def addMessage ( self, message ): + if not isinstance(self._errors,list): + self._errors = [ self._errors ] + if isinstance(message,list): + for line in message: + self._errors += [ line ] + else: + self._errors += [ message ] + return + + def terminate ( self ): + print self + sys.exit(self._code) + + @property + def code ( self ): return self._code + + +class BadBinary ( ErrorMessage ): + + def __init__ ( self, binary ): + ErrorMessage.__init__( self, 1, "Binary not found: <%s>." % binary ) + return + + +class BadReturnCode ( ErrorMessage ): + + def __init__ ( self, status ): + ErrorMessage.__init__( self, 1, "Command returned status:%d." % status ) + return + + +class Command ( object ): + + def __init__ ( self, arguments, fdLog=None ): + self.arguments = arguments + self.fdLog = fdLog + + if self.fdLog != None and not isinstance(self.fdLog,file): + print '[WARNING] Command.__init__(): is neither None or a file.' + return + + def _argumentsToStr ( self, arguments ): + s = '' + for argument in arguments: + if argument.find(' ') >= 0: s += ' "' + argument + '"' + else: s += ' ' + argument + return s + + def log ( self, text ): + print text[:-1] + sys.stdout.flush() + sys.stderr.flush() + if isinstance(self.fdLog,file): + self.fdLog.write( text ) + self.fdLog.flush() + return + + def execute ( self ): + global conf + sys.stdout.flush() + sys.stderr.flush() + + homeDir = os.environ['HOME'] + workDir = os.getcwd() + if homeDir.startswith(homeDir): + workDir = '~' + workDir[ len(homeDir) : ] + user = 'root' + if os.environ.has_key('USER'): user = os.environ['USER'] + prompt = '%s@%s:%s$' % (user,conf.masterHost,workDir) + + try: + self.log( '%s%s\n' % (prompt,self._argumentsToStr(self.arguments)) ) + print self.arguments + child = subprocess.Popen( self.arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + + while True: + line = child.stdout.readline() + if not line: break + + self.log( line ) + except OSError, e: + raise BadBinary( self.arguments[0] ) + + (pid,status) = os.waitpid( child.pid, 0 ) + status >>= 8 + if status != 0: + raise BadReturnCode( status ) + + return + + +class CommandArg ( object ): + + def __init__ ( self, command, wd=None, host=None, fdLog=None ): + self.command = command + self.host = host + self.wd = wd + self.fdLog = fdLog + return + + def __str__ ( self ): + s = '' + if self.wd: s = 'cd %s && ' % self.wd + + for i in range(len(self.command)): + if i: s += ' ' + s += self.command[i] + return s + + def getArgs ( self ): + if not self.host: return self.command + return [ 'ssh', self.host, str(self) ] + + def execute ( self ): + if not self.host and self.wd: os.chdir( self.wd ) + Command( self.getArgs(), self.fdLog ).execute() + return + + +class AllianceCommand ( CommandArg ): + + def __init__ ( self, fdLog=None ): + CommandArg.__init__ ( self, [ '/root/allianceInstaller.sh' ] + , fdLog=fdLog ) + return + + +class CoriolisCommand ( CommandArg ): + + def __init__ ( self, ccbBin, rootDir, threads=1, otherArgs=[], fdLog=None ): + CommandArg.__init__ ( self, [ ccbBin + , '--root='+rootDir + , '--project=coriolis' + , '--make=-j%d install' % threads + ] + otherArgs + , fdLog=fdLog ) + return + + +class BenchsCommand ( CommandArg ): + + def __init__ ( self, benchsDir, fdLog=None ): + CommandArg.__init__ ( self, [ '../bin/go.sh' ], wd=benchsDir, fdLog=fdLog ) + return + + + +class GitRepository ( object ): + + @staticmethod + def getLocalRepository ( url ): + localRepo = url.split( '/' )[-1] + if localRepo.endswith('.git'): + localRepo = localRepo[:-4] + return localRepo + + def __init__ ( self, url, cloneDir, fdLog=None ): + self.url = url + self.cloneDir = cloneDir + self.localRepo = GitRepository.getLocalRepository( url ) + self.fdLog = fdLog + return + + @property + def localRepoDir ( self ): return self.cloneDir+'/'+self.localRepo + + def removeLocalRepo ( self ): + if os.path.isdir(self.localRepoDir): + print 'Removing Git local repository: <%s>' % self.localRepoDir + shutil.rmtree( self.localRepoDir ) + return + + def clone ( self ): + print 'Clone/pull from:', self.url + if not os.path.isdir(self.cloneDir): + os.makedirs( self.cloneDir ) + + if not os.path.isdir(self.localRepoDir): + os.chdir( self.cloneDir ) + Command( [ 'git', 'clone', self.url ], self.fdLog ).execute() + else: + os.chdir( self.localRepoDir ) + Command( [ 'git', 'pull' ], self.fdLog ).execute() + return + + def checkout ( self, branch ): + os.chdir( self.localRepoDir ) + Command( [ 'git', 'checkout', branch ], self.fdLog ).execute() + return + + +class Configuration ( object ): + + PrimaryNames = \ + [ 'sender' , 'receivers' + , 'coriolisRepo', 'benchsRepo' , 'supportRepos' + , 'homeDir' , 'masterHost' + , 'debugArg' , 'nightlyMode', 'dockerMode' + , 'rmSource' , 'rmBuild' + , 'doGit' , 'doAlliance' , 'doCoriolis', 'doBenchs', 'doSendReport' + , 'success' , 'rcode' + ] + SecondaryNames = \ + [ 'rootDir', 'srcDir', 'logDir', 'logs', 'fds', 'ccbBin', 'benchsDir' + ] + + def __init__ ( self ): + self._sender = 'Jean-Paul.Chaput@soc.lip6.fr' + self._receivers = [ 'Jean-Paul.Chaput@lip6.fr', ] + self._supportRepos = [ 'http://github.com/miloyip/rapidjson' ] + self._allianceRepo = 'https://gitlab.lip6.fr/jpc/alliance.git' + self._coriolisRepo = 'https://gitlab.lip6.fr/jpc/coriolis.git' + self._benchsRepo = 'https://gitlab.lip6.fr/jpc/alliance-check-toolkit.git' + self._homeDir = os.environ['HOME'] + self._debugArg = '' + self._rmSource = False + self._rmBuild = False + self._doGit = True + self._doCoriolis = False + self._doAlliance = False + self._doBenchs = False + self._doSendReport = False + self._nightlyMode = False + self._dockerMode = False + self._logs = { 'alliance':None, 'coriolis':None, 'benchs':None } + self._fds = { 'alliance':None, 'coriolis':None, 'benchs':None } + self._ccbBin = None + self._benchsDir = None + self._masterHost = self._detectMasterHost() + self._success = False + self._rcode = 0 + + self._updateSecondaries() + return + + def __setattr__ ( self, attribute, value ): + if attribute in Configuration.SecondaryNames: + print ErrorMessage( 1, 'Attempt to write in read-only attribute <%s> in Configuration.'%attribute ) + return + + if attribute == 'masterHost' or attribute == '_masterHost': + if value == 'lepka': + print 'Never touch the Git tree when running on .' + self._rmSource = False + self._rmBuild = False + self._doGit = False + self._doSendReport = False + + if attribute[0] == '_': + self.__dict__[attribute] = value + return + + if attribute == 'homeDir': value = os.path.expanduser(value) + + self.__dict__['_'+attribute] = value + self._updateSecondaries() + return + + def __getattr__ ( self, attribute ): + if attribute[0] != '_': attribute = '_'+attribute + if not self.__dict__.has_key(attribute): + raise ErrorMessage( 1, 'Configuration has no attribute <%s>.'%attribute ) + return self.__dict__[attribute] + + def _updateSecondaries ( self ): + if self._nightlyMode: + self._rootDir = self._homeDir + '/nightly/coriolis-2.x' + else: + self._rootDir = self._homeDir + '/coriolis-2.x' + self._srcDir = self._rootDir + '/src' + self._logDir = self._srcDir + '/logs' + self._ccbBin = self._srcDir + '/' + GitRepository.getLocalRepository(self._coriolisRepo) + '/bootstrap/ccb.py' + self._benchsDir = self._srcDir + '/' + GitRepository.getLocalRepository(self._benchsRepo ) + '/benchs' + return + + def _detectMasterHost ( self ): + masterHost = 'unknown' + hostname = socket.gethostname() + hostAddr = socket.gethostbyname(hostname) + + if hostname == 'lepka' and hostAddr == '127.0.0.1': + masterHost = 'lepka' + else: + masterHost = hostname.split('.')[0] + return masterHost + + def openLog ( self, stem ): + if not os.path.isdir(self._logDir): + os.makedirs( self._logDir ) + + index = 0 + timeTag = time.strftime( "%Y.%m.%d" ) + while True: + logFile = os.path.join(self._logDir,"%s-%s-%02d.log" % (stem,timeTag,index)) + if not os.path.isfile(logFile): + print "Report log: <%s>" % logFile + break + index += 1 + fd = open( logFile, "w" ) + self._logs[stem] = logFile + self._fds [stem] = fd + return + + def closeLogs ( self ): + for fd in self._fds.values(): + if fd: fd.close() + return + + def compressLogs ( self ): + for log in self._logs.values(): + if not log: continue + + fd = open( log, 'r' ) + bzfd = bz2.BZ2File( log+'.bz2', 'w' ) + + for line in fd.readlines(): bzfd.write( line ) + + bzfd.close() + fd.close() + + os.unlink( log ) + return + + def getCommands ( self, target ): + commands = [] + + if self.doAlliance: + commands.append( AllianceCommand( fdLog=self.fds['alliance'] ) ) + + if self.doCoriolis: + if not os.path.isfile( self.ccbBin ): + raise ErrorMessage( 1, [ 'Cannot find , should be here:' + , ' <%s>' % self.ccbBin + ] ) + + otherArgs = [] + if self.debugArg: otherArgs.append( self.debugArg ) + + if target == 'SL7_64': + otherArgs.append( '--project=support' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs , fdLog=self.fds['coriolis'] ) ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand ( self.benchsDir, fdLog=self.fds['benchs'] ) ) + elif target == 'SL6_64' or target == 'SL6': + otherArgs.append( '--project=support' ) + otherArgs.append( '--devtoolset=8' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 6, otherArgs , fdLog=self.fds['coriolis'] ) ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) + elif target == 'Ubuntu18' or target == 'Debian9': + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs , fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) + return commands + + +class Report ( object ): + + def __init__ ( self, conf ): + self.conf = conf + + commaspace = ', ' + date = time.strftime( "%A %d %B %Y" ) + stateText = 'FAILED' + modeText = 'SoC installation' + if self.conf.success: stateText = 'SUCCESS' + if self.conf.nightlyMode: modeText = 'Nightly build' + + self.message = MIMEMultipart() + self.message['Subject'] = '[%s] Coriolis %s %s' % (stateText,modeText,date) + self.message['From' ] = self.conf.sender + self.message['To' ] = commaspace.join( self.conf.receivers ) + self.attachements = [] + + self.mainText = '\n' + self.mainText += 'Salut le Crevard,\n' + self.mainText += '\n' + if self.conf.nightlyMode: + self.mainText += 'This is the nightly build report of Coriolis.\n' + else: + self.mainText += 'SoC installer report of Coriolis.\n' + self.mainText += '%s\n' % date + self.mainText += '\n' + if self.conf.success: + self.mainText += 'Build was SUCCESSFUL\n' + else: + self.mainText += 'Build has FAILED, please have a look to the attached log file(s).\n' + self.mainText += '\n' + self.mainText += 'Complete log file(s) can be found here:\n' + return + + def attachLog ( self, logFile ): + if not logFile: return + + fd = open( logFile, 'rb' ) + try: + fd.seek( -1024*100, os.SEEK_END ) + except IOError, e: + pass + tailLines = '' + for line in fd.readlines()[1:]: + tailLines += line + fd.close() + self.mainText += ' <%s>\n' % logFile + + attachement = MIMEApplication(tailLines) + attachement.add_header( 'Content-Disposition', 'attachment', filename=os.path.basename(logFile) ) + + self.attachements.append( attachement ) + return + + def send ( self ): + self.message.attach( MIMEText(self.mainText) ) + for attachement in self.attachements: + self.message.attach( attachement ) + + print "Sending mail report to:" + for receiver in self.conf.receivers: print ' <%s>' % receiver + session = smtplib.SMTP( 'localhost' ) + session.sendmail( self.conf.sender, self.conf.receivers, self.message.as_string() ) + session.quit() + return + + +# ------------------------------------------------------------------- +# Main Part. + + +parser = optparse.OptionParser () +parser.add_option ( "--debug" , action="store_true" , dest="debug" , help="Build a aka (-g) version." ) +parser.add_option ( "--no-git" , action="store_true" , dest="noGit" , help="Do not pull/update Git repositories before building." ) +parser.add_option ( "--do-alliance" , action="store_true" , dest="doAlliance" , help="Rebuild the Alliance tools." ) +parser.add_option ( "--do-coriolis" , action="store_true" , dest="doCoriolis" , help="Rebuild the Coriolis tools." ) +parser.add_option ( "--do-report" , action="store_true" , dest="doReport" , help="Send a final report." ) +parser.add_option ( "--nightly" , action="store_true" , dest="nightly" , help="Perform a nighly build." ) +parser.add_option ( "--docker" , action="store_true" , dest="docker" , help="Perform a build inside a docker container." ) +parser.add_option ( "--benchs" , action="store_true" , dest="benchs" , help="Run the sanity benchs." ) +parser.add_option ( "--rm-build" , action="store_true" , dest="rmBuild" , help="Remove the build/install directories." ) +parser.add_option ( "--rm-source" , action="store_true" , dest="rmSource" , help="Remove the Git source repositories." ) +parser.add_option ( "--rm-all" , action="store_true" , dest="rmAll" , help="Remove everything (source+build+install)." ) +parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" , help="The root directory (default: <~/coriolis-2.x/>)." ) +parser.add_option ( "--profile" , action="store" , type="string", dest="profile" , help="The targeted OS for the build." ) +(options, args) = parser.parse_args () + +conf = Configuration() + +try: + if options.debug: conf.debugArg = '--debug' + if options.nightly: conf.nightlyMode = True + if options.docker: conf.dockerMode = True + if options.noGit: conf.doGit = False + if options.doCoriolis: conf.doCoriolis = True + if options.doAlliance: conf.doAlliance = True + if options.benchs: conf.doBenchs = True + if options.doReport: conf.doSendReport = True + if options.rmSource or options.rmAll: conf.rmSource = True + if options.rmBuild or options.rmAll: conf.rmBuild = True + + if conf.doAlliance: conf.openLog( 'alliance' ) + if conf.doCoriolis: conf.openLog( 'coriolis' ) + if conf.doBenchs: conf.openLog( 'benchs' ) + + if conf.dockerMode: os.environ['USER'] = 'root' + + gitSupports = [] + for supportRepo in conf.supportRepos: + gitSupports.append( GitRepository( supportRepo, conf.srcDir+'/support' ) ) + gitCoriolis = GitRepository( conf.coriolisRepo, conf.srcDir, conf.fds['coriolis'] ) + gitBenchs = GitRepository( conf.benchsRepo , conf.srcDir, conf.fds['coriolis'] ) + + if conf.doAlliance: + gitAlliance = GitRepository( conf.allianceRepo, conf.srcDir, conf.fds['alliance'] ) + + if conf.doGit: + for gitSupport in gitSupports: + if conf.rmSource: gitSupport.removeLocalRepo() + gitSupport.clone() + #if gitSupport.url.endswith('rapidjson'): + # gitSupport.checkout( 'a1c4f32' ) + + if conf.doCoriolis: + if conf.rmSource: gitCoriolis.removeLocalRepo() + gitCoriolis.clone () + gitCoriolis.checkout( 'devel' ) + + if conf.doAlliance: + if conf.rmSource: gitAlliance.removeLocalRepo() + gitAlliance.clone () + #gitAlliance.checkout( 'devel' ) + + if conf.rmSource: gitBenchs.removeLocalRepo() + gitBenchs.clone() + + if conf.rmBuild: + for entry in os.listdir(conf.rootDir): + if entry.startswith('Linux.'): + buildDir = conf.rootDir+'/'+entry + print 'Removing OS build directory: <%s>' % buildDir + shutil.rmtree( buildDir ) + + commands = conf.getCommands( options.profile ) + for command in commands: + if command.host: + print 'Executing command on remote host <%s>:' % host + else: + print 'Executing command on *local* host:' + print ' %s' % str(command) + command.execute() + + conf.closeLogs() + + conf.success = True + +except ErrorMessage, e: + print e + conf.closeLogs() + conf.success = False + + if showTrace: + print '\nPython stack trace:' + traceback.print_tb( sys.exc_info()[2] ) + conf.rcode = e.code + +if conf.doSendReport: + report = Report( conf ) + report.attachLog( conf.logs['coriolis' ] ) + report.attachLog( conf.logs['benchs' ] ) + report.send() + +conf.compressLogs() + +sys.exit( conf.rcode ) diff --git a/bootstrap/docker/ubuntu-18/Dockerfile.coriolis b/bootstrap/docker/ubuntu-18/Dockerfile.coriolis new file mode 100644 index 00000000..34e377bc --- /dev/null +++ b/bootstrap/docker/ubuntu-18/Dockerfile.coriolis @@ -0,0 +1,7 @@ + +FROM ubuntu18.system + +COPY root/socInstaller.py /root/socInstaller.py +COPY root/allianceInstaller.sh /root/allianceInstaller.sh +CMD /root/socInstaller.py --docker --profile=Ubuntu18 --do-alliance --do-coriolis --benchs + diff --git a/bootstrap/docker/ubuntu-18/Dockerfile.system b/bootstrap/docker/ubuntu-18/Dockerfile.system new file mode 100644 index 00000000..0932b9fe --- /dev/null +++ b/bootstrap/docker/ubuntu-18/Dockerfile.system @@ -0,0 +1,22 @@ + +FROM ubuntu:bionic + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get -y install build-essential binutils-dev \ + git cmake bison flex gcc python-dev \ + libboost-all-dev libboost-python-dev \ + zlib1g-dev libxml2-dev rapidjson-dev libbz2-dev \ + qtbase5-dev libqt5svg5-dev libqwt-qt5-dev \ + python-pyqt5 \ + \ + autotools-dev automake \ + libxt-dev libxpm-dev libmotif-dev \ + \ + yosys \ + \ + vim \ + && apt-get clean + +# qt4-dev-tools libqwt-dev python-qt4 \ diff --git a/bootstrap/docker/ubuntu-18/docker-conf.sh b/bootstrap/docker/ubuntu-18/docker-conf.sh new file mode 100644 index 00000000..ec37d6e6 --- /dev/null +++ b/bootstrap/docker/ubuntu-18/docker-conf.sh @@ -0,0 +1,3 @@ + + systemImage="ubuntu18.system" + coriolisImage="ubuntu18.coriolis" diff --git a/bootstrap/docker/ubuntu-18/root/allianceInstaller.sh b/bootstrap/docker/ubuntu-18/root/allianceInstaller.sh new file mode 100755 index 00000000..71f02906 --- /dev/null +++ b/bootstrap/docker/ubuntu-18/root/allianceInstaller.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + srcDir=${HOME}/coriolis-2.x/src/alliance/alliance/src + commonRoot=${HOME}/coriolis-2.x/Linux.el7_64/Release.Shared + buildDir=${commonRoot}/build + installDir=${commonRoot}/install + + export ALLIANCE_TOP=${installDir} + export LD_LIBRARY_PATH=${installDir}/lib:${LD_LIBRARY_PATH} + + cd ${srcDir} + # Skip doc generation to avoid pulling TeXLive in docker images. + sed -i 's,dirs="\$newdirs documentation",dirs="$newdirs",' ./autostuff + ./autostuff clean + ./autostuff + mkdir -p ${buildDir} + cd ${buildDir} + ${srcDir}/configure --prefix=${ALLIANCE_TOP} --enable-alc-shared + make -j1 install diff --git a/bootstrap/docker/ubuntu-18/root/socInstaller.py b/bootstrap/docker/ubuntu-18/root/socInstaller.py new file mode 100755 index 00000000..79b0e406 --- /dev/null +++ b/bootstrap/docker/ubuntu-18/root/socInstaller.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python +# +# -*- mode:Python -*- +# +# This file is part of the Coriolis Software. +# Copyright (c) UPMC 2015-2018, All Rights Reserved +# +# +-----------------------------------------------------------------+ +# | C O R I O L I S | +# | C o r i o l i s I n s t a l l e r | +# | | +# | Authors : Jean-Paul Chaput | +# | E-mail : Jean-Paul.Chaput@asim.lip6.fr | +# | =============================================================== | +# | Python : "./socInstaller.py" | +# +-----------------------------------------------------------------+ +# +# WARNING: +# This script has been designed only for internal use in the +# LIP6/CIAN department. If you want to use it you will need to +# change the hardwired configuration. + + +showTrace = True + +try: + import sys + import os.path + import shutil + import optparse + import time + import traceback + import distutils.sysconfig + import subprocess + import socket + import re + import bz2 + import smtplib + from email.mime.text import MIMEText + from email.mime.multipart import MIMEMultipart + from email.mime.application import MIMEApplication +except ImportError, e: + module = str(e).split()[-1] + + +class ErrorMessage ( Exception ): + + def __init__ ( self, code, *arguments ): + self._code = code + self._errors = [ 'Malformed call to ErrorMessage()', '%s' % str(arguments) ] + + text = None + if len(arguments) == 1: + if isinstance(arguments[0],Exception): text = str(arguments[0]).split('\n') + else: + self._errors = arguments[0] + elif len(arguments) > 1: + text = list(arguments) + + if text: + self._errors = [] + while len(text[0]) == 0: del text[0] + + lstrip = 0 + if text[0].startswith('[ERROR]'): lstrip = 8 + + for line in text: + if line[0:lstrip ] == ' '*lstrip or \ + line[0:lstrip-1] == '[ERROR]': + self._errors += [ line[lstrip:] ] + else: + self._errors += [ line.lstrip() ] + return + + def __str__ ( self ): + if not isinstance(self._errors,list): + return "[ERROR] %s" % self._errors + + formatted = "\n" + for i in range(len(self._errors)): + if i == 0: formatted += "[ERROR] %s" % self._errors[i] + else: formatted += " %s" % self._errors[i] + if i+1 < len(self._errors): formatted += "\n" + return formatted + + def addMessage ( self, message ): + if not isinstance(self._errors,list): + self._errors = [ self._errors ] + if isinstance(message,list): + for line in message: + self._errors += [ line ] + else: + self._errors += [ message ] + return + + def terminate ( self ): + print self + sys.exit(self._code) + + @property + def code ( self ): return self._code + + +class BadBinary ( ErrorMessage ): + + def __init__ ( self, binary ): + ErrorMessage.__init__( self, 1, "Binary not found: <%s>." % binary ) + return + + +class BadReturnCode ( ErrorMessage ): + + def __init__ ( self, status ): + ErrorMessage.__init__( self, 1, "Command returned status:%d." % status ) + return + + +class Command ( object ): + + def __init__ ( self, arguments, fdLog=None ): + self.arguments = arguments + self.fdLog = fdLog + + if self.fdLog != None and not isinstance(self.fdLog,file): + print '[WARNING] Command.__init__(): is neither None or a file.' + return + + def _argumentsToStr ( self, arguments ): + s = '' + for argument in arguments: + if argument.find(' ') >= 0: s += ' "' + argument + '"' + else: s += ' ' + argument + return s + + def log ( self, text ): + print text[:-1] + sys.stdout.flush() + sys.stderr.flush() + if isinstance(self.fdLog,file): + self.fdLog.write( text ) + self.fdLog.flush() + return + + def execute ( self ): + global conf + sys.stdout.flush() + sys.stderr.flush() + + homeDir = os.environ['HOME'] + workDir = os.getcwd() + if homeDir.startswith(homeDir): + workDir = '~' + workDir[ len(homeDir) : ] + user = 'root' + if os.environ.has_key('USER'): user = os.environ['USER'] + prompt = '%s@%s:%s$' % (user,conf.masterHost,workDir) + + try: + self.log( '%s%s\n' % (prompt,self._argumentsToStr(self.arguments)) ) + print self.arguments + child = subprocess.Popen( self.arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + + while True: + line = child.stdout.readline() + if not line: break + + self.log( line ) + except OSError, e: + raise BadBinary( self.arguments[0] ) + + (pid,status) = os.waitpid( child.pid, 0 ) + status >>= 8 + if status != 0: + raise BadReturnCode( status ) + + return + + +class CommandArg ( object ): + + def __init__ ( self, command, wd=None, host=None, fdLog=None ): + self.command = command + self.host = host + self.wd = wd + self.fdLog = fdLog + return + + def __str__ ( self ): + s = '' + if self.wd: s = 'cd %s && ' % self.wd + + for i in range(len(self.command)): + if i: s += ' ' + s += self.command[i] + return s + + def getArgs ( self ): + if not self.host: return self.command + return [ 'ssh', self.host, str(self) ] + + def execute ( self ): + if not self.host and self.wd: os.chdir( self.wd ) + Command( self.getArgs(), self.fdLog ).execute() + return + + +class AllianceCommand ( CommandArg ): + + def __init__ ( self, fdLog=None ): + CommandArg.__init__ ( self, [ '/root/allianceInstaller.sh' ] + , fdLog=fdLog ) + return + + +class CoriolisCommand ( CommandArg ): + + def __init__ ( self, ccbBin, rootDir, threads=1, otherArgs=[], fdLog=None ): + CommandArg.__init__ ( self, [ ccbBin + , '--root='+rootDir + , '--project=coriolis' + , '--make=-j%d install' % threads + ] + otherArgs + , fdLog=fdLog ) + return + + +class BenchsCommand ( CommandArg ): + + def __init__ ( self, benchsDir, fdLog=None ): + CommandArg.__init__ ( self, [ '../bin/go.sh' ], wd=benchsDir, fdLog=fdLog ) + return + + + +class GitRepository ( object ): + + @staticmethod + def getLocalRepository ( url ): + localRepo = url.split( '/' )[-1] + if localRepo.endswith('.git'): + localRepo = localRepo[:-4] + return localRepo + + def __init__ ( self, url, cloneDir, fdLog=None ): + self.url = url + self.cloneDir = cloneDir + self.localRepo = GitRepository.getLocalRepository( url ) + self.fdLog = fdLog + return + + @property + def localRepoDir ( self ): return self.cloneDir+'/'+self.localRepo + + def removeLocalRepo ( self ): + if os.path.isdir(self.localRepoDir): + print 'Removing Git local repository: <%s>' % self.localRepoDir + shutil.rmtree( self.localRepoDir ) + return + + def clone ( self ): + print 'Clone/pull from:', self.url + if not os.path.isdir(self.cloneDir): + os.makedirs( self.cloneDir ) + + if not os.path.isdir(self.localRepoDir): + os.chdir( self.cloneDir ) + Command( [ 'git', 'clone', self.url ], self.fdLog ).execute() + else: + os.chdir( self.localRepoDir ) + Command( [ 'git', 'pull' ], self.fdLog ).execute() + return + + def checkout ( self, branch ): + os.chdir( self.localRepoDir ) + Command( [ 'git', 'checkout', branch ], self.fdLog ).execute() + return + + +class Configuration ( object ): + + PrimaryNames = \ + [ 'sender' , 'receivers' + , 'coriolisRepo', 'benchsRepo' , 'supportRepos' + , 'homeDir' , 'masterHost' + , 'debugArg' , 'nightlyMode', 'dockerMode' + , 'rmSource' , 'rmBuild' + , 'doGit' , 'doAlliance' , 'doCoriolis', 'doBenchs', 'doSendReport' + , 'success' , 'rcode' + ] + SecondaryNames = \ + [ 'rootDir', 'srcDir', 'logDir', 'logs', 'fds', 'ccbBin', 'benchsDir' + ] + + def __init__ ( self ): + self._sender = 'Jean-Paul.Chaput@soc.lip6.fr' + self._receivers = [ 'Jean-Paul.Chaput@lip6.fr', ] + self._supportRepos = [ 'http://github.com/miloyip/rapidjson' ] + self._allianceRepo = 'https://gitlab.lip6.fr/jpc/alliance.git' + self._coriolisRepo = 'https://gitlab.lip6.fr/jpc/coriolis.git' + self._benchsRepo = 'https://gitlab.lip6.fr/jpc/alliance-check-toolkit.git' + self._homeDir = os.environ['HOME'] + self._debugArg = '' + self._rmSource = False + self._rmBuild = False + self._doGit = True + self._doCoriolis = False + self._doAlliance = False + self._doBenchs = False + self._doSendReport = False + self._nightlyMode = False + self._dockerMode = False + self._logs = { 'alliance':None, 'coriolis':None, 'benchs':None } + self._fds = { 'alliance':None, 'coriolis':None, 'benchs':None } + self._ccbBin = None + self._benchsDir = None + self._masterHost = self._detectMasterHost() + self._success = False + self._rcode = 0 + + self._updateSecondaries() + return + + def __setattr__ ( self, attribute, value ): + if attribute in Configuration.SecondaryNames: + print ErrorMessage( 1, 'Attempt to write in read-only attribute <%s> in Configuration.'%attribute ) + return + + if attribute == 'masterHost' or attribute == '_masterHost': + if value == 'lepka': + print 'Never touch the Git tree when running on .' + self._rmSource = False + self._rmBuild = False + self._doGit = False + self._doSendReport = False + + if attribute[0] == '_': + self.__dict__[attribute] = value + return + + if attribute == 'homeDir': value = os.path.expanduser(value) + + self.__dict__['_'+attribute] = value + self._updateSecondaries() + return + + def __getattr__ ( self, attribute ): + if attribute[0] != '_': attribute = '_'+attribute + if not self.__dict__.has_key(attribute): + raise ErrorMessage( 1, 'Configuration has no attribute <%s>.'%attribute ) + return self.__dict__[attribute] + + def _updateSecondaries ( self ): + if self._nightlyMode: + self._rootDir = self._homeDir + '/nightly/coriolis-2.x' + else: + self._rootDir = self._homeDir + '/coriolis-2.x' + self._srcDir = self._rootDir + '/src' + self._logDir = self._srcDir + '/logs' + self._ccbBin = self._srcDir + '/' + GitRepository.getLocalRepository(self._coriolisRepo) + '/bootstrap/ccb.py' + self._benchsDir = self._srcDir + '/' + GitRepository.getLocalRepository(self._benchsRepo ) + '/benchs' + return + + def _detectMasterHost ( self ): + masterHost = 'unknown' + hostname = socket.gethostname() + hostAddr = socket.gethostbyname(hostname) + + if hostname == 'lepka' and hostAddr == '127.0.0.1': + masterHost = 'lepka' + else: + masterHost = hostname.split('.')[0] + return masterHost + + def openLog ( self, stem ): + if not os.path.isdir(self._logDir): + os.makedirs( self._logDir ) + + index = 0 + timeTag = time.strftime( "%Y.%m.%d" ) + while True: + logFile = os.path.join(self._logDir,"%s-%s-%02d.log" % (stem,timeTag,index)) + if not os.path.isfile(logFile): + print "Report log: <%s>" % logFile + break + index += 1 + fd = open( logFile, "w" ) + self._logs[stem] = logFile + self._fds [stem] = fd + return + + def closeLogs ( self ): + for fd in self._fds.values(): + if fd: fd.close() + return + + def compressLogs ( self ): + for log in self._logs.values(): + if not log: continue + + fd = open( log, 'r' ) + bzfd = bz2.BZ2File( log+'.bz2', 'w' ) + + for line in fd.readlines(): bzfd.write( line ) + + bzfd.close() + fd.close() + + os.unlink( log ) + return + + def getCommands ( self, target ): + commands = [] + + if self.doAlliance: + commands.append( AllianceCommand( fdLog=self.fds['alliance'] ) ) + + if self.doCoriolis: + if not os.path.isfile( self.ccbBin ): + raise ErrorMessage( 1, [ 'Cannot find , should be here:' + , ' <%s>' % self.ccbBin + ] ) + + otherArgs = [] + if self.debugArg: otherArgs.append( self.debugArg ) + + if target == 'SL7_64': + otherArgs.append( '--project=support' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs , fdLog=self.fds['coriolis'] ) ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand ( self.benchsDir, fdLog=self.fds['benchs'] ) ) + elif target == 'SL6_64' or target == 'SL6': + otherArgs.append( '--project=support' ) + otherArgs.append( '--devtoolset=8' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 6, otherArgs , fdLog=self.fds['coriolis'] ) ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 1, otherArgs+['--doc'], fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) + elif target == 'Ubuntu18' or target == 'Debian9': + if target == 'Ubuntu18': otherArgs.append( '--qt5' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs, fdLog=self.fds['coriolis'] ) ) + if self.doBenchs: + commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) + return commands + + +class Report ( object ): + + def __init__ ( self, conf ): + self.conf = conf + + commaspace = ', ' + date = time.strftime( "%A %d %B %Y" ) + stateText = 'FAILED' + modeText = 'SoC installation' + if self.conf.success: stateText = 'SUCCESS' + if self.conf.nightlyMode: modeText = 'Nightly build' + + self.message = MIMEMultipart() + self.message['Subject'] = '[%s] Coriolis %s %s' % (stateText,modeText,date) + self.message['From' ] = self.conf.sender + self.message['To' ] = commaspace.join( self.conf.receivers ) + self.attachements = [] + + self.mainText = '\n' + self.mainText += 'Salut le Crevard,\n' + self.mainText += '\n' + if self.conf.nightlyMode: + self.mainText += 'This is the nightly build report of Coriolis.\n' + else: + self.mainText += 'SoC installer report of Coriolis.\n' + self.mainText += '%s\n' % date + self.mainText += '\n' + if self.conf.success: + self.mainText += 'Build was SUCCESSFUL\n' + else: + self.mainText += 'Build has FAILED, please have a look to the attached log file(s).\n' + self.mainText += '\n' + self.mainText += 'Complete log file(s) can be found here:\n' + return + + def attachLog ( self, logFile ): + if not logFile: return + + fd = open( logFile, 'rb' ) + try: + fd.seek( -1024*100, os.SEEK_END ) + except IOError, e: + pass + tailLines = '' + for line in fd.readlines()[1:]: + tailLines += line + fd.close() + self.mainText += ' <%s>\n' % logFile + + attachement = MIMEApplication(tailLines) + attachement.add_header( 'Content-Disposition', 'attachment', filename=os.path.basename(logFile) ) + + self.attachements.append( attachement ) + return + + def send ( self ): + self.message.attach( MIMEText(self.mainText) ) + for attachement in self.attachements: + self.message.attach( attachement ) + + print "Sending mail report to:" + for receiver in self.conf.receivers: print ' <%s>' % receiver + session = smtplib.SMTP( 'localhost' ) + session.sendmail( self.conf.sender, self.conf.receivers, self.message.as_string() ) + session.quit() + return + + +# ------------------------------------------------------------------- +# Main Part. + + +parser = optparse.OptionParser () +parser.add_option ( "--debug" , action="store_true" , dest="debug" , help="Build a aka (-g) version." ) +parser.add_option ( "--no-git" , action="store_true" , dest="noGit" , help="Do not pull/update Git repositories before building." ) +parser.add_option ( "--do-alliance" , action="store_true" , dest="doAlliance" , help="Rebuild the Alliance tools." ) +parser.add_option ( "--do-coriolis" , action="store_true" , dest="doCoriolis" , help="Rebuild the Coriolis tools." ) +parser.add_option ( "--do-report" , action="store_true" , dest="doReport" , help="Send a final report." ) +parser.add_option ( "--nightly" , action="store_true" , dest="nightly" , help="Perform a nighly build." ) +parser.add_option ( "--docker" , action="store_true" , dest="docker" , help="Perform a build inside a docker container." ) +parser.add_option ( "--benchs" , action="store_true" , dest="benchs" , help="Run the sanity benchs." ) +parser.add_option ( "--rm-build" , action="store_true" , dest="rmBuild" , help="Remove the build/install directories." ) +parser.add_option ( "--rm-source" , action="store_true" , dest="rmSource" , help="Remove the Git source repositories." ) +parser.add_option ( "--rm-all" , action="store_true" , dest="rmAll" , help="Remove everything (source+build+install)." ) +parser.add_option ( "--root" , action="store" , type="string", dest="rootDir" , help="The root directory (default: <~/coriolis-2.x/>)." ) +parser.add_option ( "--profile" , action="store" , type="string", dest="profile" , help="The targeted OS for the build." ) +(options, args) = parser.parse_args () + +conf = Configuration() + +try: + if options.debug: conf.debugArg = '--debug' + if options.nightly: conf.nightlyMode = True + if options.docker: conf.dockerMode = True + if options.noGit: conf.doGit = False + if options.doCoriolis: conf.doCoriolis = True + if options.doAlliance: conf.doAlliance = True + if options.benchs: conf.doBenchs = True + if options.doReport: conf.doSendReport = True + if options.rmSource or options.rmAll: conf.rmSource = True + if options.rmBuild or options.rmAll: conf.rmBuild = True + + if conf.doAlliance: conf.openLog( 'alliance' ) + if conf.doCoriolis: conf.openLog( 'coriolis' ) + if conf.doBenchs: conf.openLog( 'benchs' ) + + if conf.dockerMode: os.environ['USER'] = 'root' + + gitSupports = [] + for supportRepo in conf.supportRepos: + gitSupports.append( GitRepository( supportRepo, conf.srcDir+'/support' ) ) + gitCoriolis = GitRepository( conf.coriolisRepo, conf.srcDir, conf.fds['coriolis'] ) + gitBenchs = GitRepository( conf.benchsRepo , conf.srcDir, conf.fds['coriolis'] ) + + if conf.doAlliance: + gitAlliance = GitRepository( conf.allianceRepo, conf.srcDir, conf.fds['alliance'] ) + + if conf.doGit: + for gitSupport in gitSupports: + if conf.rmSource: gitSupport.removeLocalRepo() + gitSupport.clone() + #if gitSupport.url.endswith('rapidjson'): + # gitSupport.checkout( 'a1c4f32' ) + + if conf.doCoriolis: + if conf.rmSource: gitCoriolis.removeLocalRepo() + gitCoriolis.clone () + gitCoriolis.checkout( 'devel' ) + + if conf.doAlliance: + if conf.rmSource: gitAlliance.removeLocalRepo() + gitAlliance.clone () + #gitAlliance.checkout( 'devel' ) + + if conf.rmSource: gitBenchs.removeLocalRepo() + gitBenchs.clone() + + if conf.rmBuild: + for entry in os.listdir(conf.rootDir): + if entry.startswith('Linux.'): + buildDir = conf.rootDir+'/'+entry + print 'Removing OS build directory: <%s>' % buildDir + shutil.rmtree( buildDir ) + + commands = conf.getCommands( options.profile ) + for command in commands: + if command.host: + print 'Executing command on remote host <%s>:' % host + else: + print 'Executing command on *local* host:' + print ' %s' % str(command) + command.execute() + + conf.closeLogs() + + conf.success = True + +except ErrorMessage, e: + print e + conf.closeLogs() + conf.success = False + + if showTrace: + print '\nPython stack trace:' + traceback.print_tb( sys.exc_info()[2] ) + conf.rcode = e.code + +if conf.doSendReport: + report = Report( conf ) + report.attachLog( conf.logs['coriolis' ] ) + report.attachLog( conf.logs['benchs' ] ) + report.send() + +conf.compressLogs() + +sys.exit( conf.rcode ) diff --git a/bootstrap/dockerManager.sh b/bootstrap/dockerManager.sh new file mode 100755 index 00000000..061565ef --- /dev/null +++ b/bootstrap/dockerManager.sh @@ -0,0 +1,99 @@ +#!/bin/bash + + + showHelp=0 + showError=0 + doBuildSystem=0 + doBuild=0 + doRun=0 + doClean=0 + + + if [ ! -f "./docker-conf.sh" ]; then + echo "[ERROR] Missing \"./docker-conf.sh\"." + echo " (wd:\"`pwd`\")" + exit 1 + fi + . "./docker-conf.sh" + dockerImages="${systemImage},${coriolisImage}" + + + while [ $# -gt 0 ]; do + case $1 in + --help) showHelp=1;; + --system) doBuildSystem=1;; + --build) doBuild=1;; + --run) doRun=1;; + --clean) doClean=1;; + -*) NB=2; CH=`echo $1 | cut -c$NB` + while [ "$CH" != "" ]; do + case $CH in + h) showHelp=1;; + S) doBuildSystem=1;; + b) doBuild=1;; + r) doRun=1;; + C) doClean=1;; + *) showError=1; badOption="$1";; + esac + NB=`expr $NB + 1` + CH=`echo $1 | cut -c$NB` + done;; + *) showError=1; badOption="$1";; + esac + shift + done + + if [ ${showError} -ne 0 ]; then + echo "[ERROR] Unknown argument \"${badOption}\"." + exit 1 + fi + + if [ ${showHelp} -ne 0 ]; then + echo "Usage: ./manager.sh [options]" + echo "Options:" + echo " * [-h|--help]: Print this help." + echo " * [-S|--system]: Rebuild the whole OS image." + echo " * [-b|--build]: Rebuild the Coriolis image. It will remove the previous" + echo " images (${dockerImages})." + echo " * [-r|--run]: Recompile Alliance, Coriolis & perform benchs." + echo " * [-C|--clean]: Remove container(s) & image(s)." + exit 0 + fi + + + if [ ${doBuild} -ne 0 ] || [ ${doBuildSytem} -ne 0 ]; then + doClean=1 + fi + + + if [ ${doClean} -ne 0 ]; then + echo "Removing \"${coriolisImage}\" docker container." + docker rm ${coriolisImage} + docker rmi ${coriolisImage} + + if [ ${doBuildSystem} -ne 0 ]; then + echo "Removing \"${systemImage}\" docker image." + docker rm ${systemImage} + docker rmi ${systemImage} + fi + fi + + + if [ ${doBuild} -ne 0 ]; then + echo "Synching Alliance & Coriolis builder scripts." + cp ../../socInstaller.py ./root + cp ../../allianceInstaller.sh ./root + + if [ ${doBuildSystem} -ne 0 ]; then + echo "Build \"${systemImage}\" docker image." + docker build -f Dockerfile.system -t ${systemImage} . + fi + + echo "Build \"${coriolisImage}\" docker image." + docker build -f Dockerfile.coriolis -t ${coriolisImage} . + fi + + + if [ ${doRun} -ne 0 ]; then + docker run --name ${coriolisImage} ${coriolisImage} + fi diff --git a/bootstrap/socInstaller.py b/bootstrap/socInstaller.py index 6a7ab963..79b0e406 100755 --- a/bootstrap/socInstaller.py +++ b/bootstrap/socInstaller.py @@ -436,7 +436,8 @@ class Configuration ( object ): if self.doBenchs: commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) elif target == 'Ubuntu18' or target == 'Debian9': - commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs , fdLog=self.fds['coriolis'] ) ) + if target == 'Ubuntu18': otherArgs.append( '--qt5' ) + commands.append( CoriolisCommand( self.ccbBin, self.rootDir, 3, otherArgs, fdLog=self.fds['coriolis'] ) ) if self.doBenchs: commands.append( BenchsCommand( self.benchsDir, fdLog=self.fds['benchs'] ) ) return commands diff --git a/documentation/content/pages/alliance/Alliance_HTML.rst b/documentation/content/pages/alliance/Alliance_HTML.rst new file mode 100644 index 00000000..8a81d28d --- /dev/null +++ b/documentation/content/pages/alliance/Alliance_HTML.rst @@ -0,0 +1,31 @@ +.. -*- Mode: rst -*- + + +======================== +Alliance VLSI/CAD System +======================== + + +:slug: alliance +:date: 2020-02-09 13:00 +:Authors: Jean-Paul Chaput +:Contact: +:Version: February 9, 2020 (jpc) +:status: hidden + + +.. role:: raw-html(raw) + :format: html + +.. URLs that changes between the various backends. + + +.. For HTML backend + + +.. contents:: + :depth: 2 + + +.. include:: ../../../etc/definitions.rst +.. include:: Installation.rst diff --git a/documentation/content/pages/alliance/Installation.rst b/documentation/content/pages/alliance/Installation.rst new file mode 100644 index 00000000..82010e2d --- /dev/null +++ b/documentation/content/pages/alliance/Installation.rst @@ -0,0 +1,247 @@ +.. -*- Mode: rst -*- + + +About Alliance +============== + + +Circuit Designed with Alliance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Smartlabs_/Smarthome designed a complete circuit in the |XFAB| |XH035| technology + (2014). +* `Tokai University`_ (`Shimizu Lab`_) designed the |SNX|, a 16 bits processor in + the |ROHM| 0.18µm (2010). + +Those are circuits that we know of because their designers were kind enough to inform +us (it is not comprehensive). + + +Useful Links +~~~~~~~~~~~~ + +* Improved Standard Cell libraries and documentation on how to design them, + by Graham |Petley|: `VLSI Technology `_ +* A book presenting |Alliance| in depth: + `Introduction to VLSI CMOS Circuits design `_ + by Carlos Silva :sc:`Cardenas`, Takeo :sc:`Yoshida` & Alberto Palacios :sc:`Pawlovsky`. +* For spanish locutors, a set of tutorials made by Miguel Eduardo |FloresGomez| + from `Don Bosco University`_: + `Tutorial de Alliance `_ + + +Installing |Alliance| from a Distribution +========================================= + +Binary packages are avalaibles for the following distributions: + +* Fedora +* Ubuntu LTS +* MacOS X, through `MacPorts `_ + + +Fedora +~~~~~~ + +1. Pull & install the packages from the repository: + + .. code-block:: sh + + root@home:~# yum install alliance alliance-libs alliance-doc + + That's all folks. |Alliance| is ready to use. + +.. note:: With the packaged version of |Alliance|, files and directories are not at + the same places as in the default install. They have been made compliant + with the |FHS|. + + ======================== =============================== + **binaries** /usr/lib/alliance/bin + **cell libraries** /usr/share/alliance/cells + **man pages** /usr/share/alliance/man + **tutorials & examples** /usr/share/doc/alliance-5.0/ + ======================== =============================== + + Environment variables should be automatically set. + + +Ubuntu LTS 18.04 +~~~~~~~~~~~~~~~~ + +|Alliance| is available in the `Ubuntu Universe repository `_, +you may install it with the package manager. + + .. code-block:: sh + + ego@home:~> sudo apt-get install alliance + +Before using it, you must source the environment (in each terminal +you want to use |Alliance|): + +.. code-block:: sh + + ego@home:~> . /etc/alliance/profile.d/alc_env.sh + + +Rebuild From Source (Git) +========================= + +If binary packages are not avalaibles for your distribution, you may want +to rebuild |Alliance| from source. To perform this, be aware that you must +have at least a basic knowlegde of how to build a C/C++ program from source. +Sources are available through the following |git| repository: + + https://gitlab.lip6.fr/jpc/alliance.git + +The stable version is under the branch :cb:`master`, while the development +is kept under :cb:`devel` (and will be merged into :cb:`master` from time +to time...) + +In order to recompile, you will need to install the development tools in +your system. The main ones are listed below. + +=================== ======================================= =============================== +Build Dependencies +--------------------------------------------------------------------------------------------- +Software RHEL, CentOS, Scientific Linux & Fedora Debian/Ubuntu +=================== ======================================= =============================== +Basic devel tools "Development tools" (group) `build-essential` +Version system `git` `git` +GCC/G++ compiler `gcc`, `g++` `gcc`, `g++` +Autotools `autoconf`, `automake` `autotool-dev`, `automake` + `libtool` `libtool` +Parser (yacc&lex) `bison`, `flex` `bison`, `flex` +X11 libraries `libX11-devel`, `libXt-devel`, `libx11-dev`, `libxt-dev` + `libXpm-devel`, `libxpm-dev` + `motif`, `motif-devel` `libmotif-dev` +Graphics `xfig`, `ImageMagick` `xfig`, `ImageMagick` +Text Processing `texlive` `texlive`, `texlive-pictures`, + `texlive-latex-extra` +=================== ======================================= =============================== + + +.. note:: Before running the :cb:`autotools`, you have to run the :cb:`autostuff` + script in ``./alliance/src`` which generate the top-level automake files. + +.. note:: If you happen to have forgotten one of the dependency and have to + install it after running `configure`, please remove the *whole* + build directory tree and re-run `configure`. The same rule applies + if you switch from static libraries to dynamic ones. + +.. note:: **Do not build in parallel**, always uses ``-j1``, the build process + fail in strange ways when run in parallel (this is a known problem + due to the way Alliance was developped). + +.. note:: **Bison/Flex** versions. Alliance is very sensitive to the versions + of thoses programs. The reference OSes for the build are |Scientific Linux| + 6 & 7, which supplies |bison| 2.4.1 and 2.7 (resp.), |flex| 2.5.35 and + 2.5.37 (resp.). + + +Debian 9 & Ubuntu 18 +~~~~~~~~~~~~~~~~~~~~ + +In this example we show the case where |Alliance| is installed in the user's home +directory alongside with |Coriolis|. + +1. Dependencies installation: + + .. code-block:: sh + + ego@home:~> sudo apt-get install build-essential binutils-dev \ + git cmake bison flex gcc python-dev \ + autotools-dev automake \ + libxt-dev libxpm-dev libmotif-dev + +2. Getting the sources (cloning the |git| repository): + + .. code-block:: sh + + ego@home:~> mkdir -p coriolis-2.x/src + ego@home:~> cd coriolis-2.x/src + ego@home:src> git clone https://gitlab.lip6.fr/jpc/alliance.git + +3. Compilation & installation. For that step, you can use the following shell + script. + + .. note:: **The commonRoot variable**, the ``/Linux.x86_64/`` component of + the path is dependent on the |OS| you are using. It is determined + by looking to the kernel version as returned by ``uname -srm``. + In the following script, it has been set to what is choosen by the + |Coriolis| installer ``ccb.py`` for |Ubuntu|. + + .. code-block:: sh + + #!/bin/bash + + srcDir=${HOME}/coriolis-2.x/src/alliance/alliance/src + commonRoot=${HOME}/coriolis-2.x/Linux.x86_64/Release.Shared + buildDir=${commonRoot}/build + installDir=${commonRoot}/install + + export ALLIANCE_TOP=${installDir} + export LD_LIBRARY_PATH=${installDir}/lib:${LD_LIBRARY_PATH} + + cd ${srcDir} + # Skip doc generation to avoid pulling TeXLive in docker images. + #sed -i 's,dirs="\$newdirs documentation",dirs="$newdirs",' ./autostuff + ./autostuff clean + ./autostuff + mkdir -p ${buildDir} + cd ${buildDir} + ${srcDir}/configure --prefix=${ALLIANCE_TOP} --enable-alc-shared + make -j1 install + + +RHEL, CentOS, Fedora +~~~~~~~~~~~~~~~~~~~~ + +The procedure is very similar as the one for Debian/Ubuntu, except for the +package manager step and the **commonRoot** variable value, which is set +to ``/Linux.SL7_64/`` (RHEL, CentOS & Scientific Linux) +or ``/Linux.fc_64/`` (Fedora). + +1. Dependencies installation: + + .. code-block:: sh + + root@home:~> yum install git cmake bison flex gcc-c++ libstdc++-devel \ + make binutils-devel \ + autoconf automake libtool \ + libX11-devel libXt-devel libXpm-devel \ + motif motif-devel \ + + +2. Getting the sources (cloning the |git| repository): + + .. code-block:: sh + + ego@home:~> mkdir -p coriolis-2.x/src + ego@home:~> cd coriolis-2.x/src + ego@home:src> git clone https://gitlab.lip6.fr/jpc/alliance.git + +3. Compilation & installation. For that step, you can use the following shell + script. + + .. code-block:: sh + + #!/bin/bash + + srcDir=${HOME}/coriolis-2.x/src/alliance/alliance/src + commonRoot=${HOME}/coriolis-2.x/Linux.SL7_64/Release.Shared + buildDir=${commonRoot}/build + installDir=${commonRoot}/install + + export ALLIANCE_TOP=${installDir} + export LD_LIBRARY_PATH=${installDir}/lib:${LD_LIBRARY_PATH} + + cd ${srcDir} + # Skip doc generation to avoid pulling TeXLive in docker images. + #sed -i 's,dirs="\$newdirs documentation",dirs="$newdirs",' ./autostuff + ./autostuff clean + ./autostuff + mkdir -p ${buildDir} + cd ${buildDir} + ${srcDir}/configure --prefix=${ALLIANCE_TOP} --enable-alc-shared + make -j1 install + diff --git a/documentation/content/pages/documentation.rst b/documentation/content/pages/documentation.rst index c4ed2a69..579c731d 100644 --- a/documentation/content/pages/documentation.rst +++ b/documentation/content/pages/documentation.rst @@ -17,6 +17,19 @@ Documentation .. include:: ../../etc/definitions.rst +Alliance Documentation +====================== + +.. row:: + + .. column:: + :width: 5 + :offset: 1 + + `Alliance Installation <{filename}/pages/alliance/Alliance_HTML.rst>`_ |br| + How to get, build & install Alliance alongside Coriolis + + Coriolis Documentation ====================== diff --git a/documentation/content/pages/users-guide/Installation.rst b/documentation/content/pages/users-guide/Installation.rst index 71c7f636..a921419f 100644 --- a/documentation/content/pages/users-guide/Installation.rst +++ b/documentation/content/pages/users-guide/Installation.rst @@ -30,13 +30,14 @@ Main building prerequisites: * yacc & lex * Qt 4 or Qt 5 * PyQt 4 or PyQt 5 -* Qwt +* Qwt 6 Building documentation prerequisites: * doxygen * latex * python-docutils (for reStructuredText) +* pelican The following libraries get directly bundled with |Coriolis|: @@ -136,10 +137,24 @@ Installing on |RedHat| or compatible distributions boost-devel boost-python boost-filesystem \ boost-regex boost-wave \ python-devel libxml2-devel bzip2-devel \ - qt-devel qwt-devel # Qt 4 + qt-devel + + The packages ``qwt`` and ``qwt-devel`` are not provided by any standard repository + (like |EPEL|). You may download them from the + `LIP6 Addons Repository `_ + Then run: - Note, that the ``Qwt`` packages are directly available from the standart distribution - when using |Qt| 4. + .. code-block:: sh + + dummy@lepka:~> yum localinstall -y qwt-6.1.2-4.fc23.x86_64.rpm \ + qwt-devel-6.1.2-4.fc23.x86_64.rpm # Qwt for Qt 4. + + You may also install them directly (whithout an intermediate download): + + .. code-block:: sh + + dummy@lepka:~> yum install -y http://ftp.lip6.fr/pub/linux/distributions/slsoc/soc/7/addons/x86_64/RPMS/qwt-6.1.2-4.fc23.x86_64.rpm \ + http://ftp.lip6.fr/pub/linux/distributions/slsoc/soc/7/addons/x86_64/RPMS/qwt-devel-6.1.2-4.fc23.x86_64.rpm 2. Install the unpackaged prerequisites. Currently, only RapidJSON_. @@ -206,8 +221,8 @@ If you want to use Qt 5 instead of Qt 4, modify the previous steps as follows: .. code-block:: sh - dummy@lepka:~> yum localinstall -y qwt-qt5-6.1.2-4.fc23.x86_64.rpm \ - qwt-qt5-6.1.2-4.fc23.x86_64.rpm # Qwt for Qt 5. + dummy@lepka:~> yum localinstall -y qwt-qt5-6.1.2-4.fc23.x86_64.rpm \ + qwt-qt5-devel-6.1.2-4.fc23.x86_64.rpm # Qwt for Qt 5. * At **step 4**, add a ``--qt5`` argument to the ``ccb.py`` command line. @@ -272,14 +287,27 @@ First, install or check that the required prerequisites are installed: .. code-block:: sh - dummy@lepka:~> sudo apt install -y build-essential binutils-dev \ - git cmake bison flex gcc python-dev \ - libboost-all-dev libboost-python-dev \ - libbz2-dev libxml2-dev rapidjson-dev libbz2-dev \ - qt4-dev-tools libqwt5-qt4-dev \ # Qt 4 - qtbase5-dev libqt5svg5-dev libqwt-qt5-dev \ # Qt 5 - doxygen dvipng graphviz python-sphinx \ - texlive-fonts-extra texlive-lang-french + dummy@lepka:~> sudo apt-get install -y build-essential binutils-dev \ + git cmake bison flex gcc python-dev \ + libboost-all-dev libboost-python-dev \ + zlib1g-dev libxml2-dev rapidjson-dev libbz2-dev + +To use with Qt 4: + +.. code-block:: sh + + dummy@lepka:~> sudo apt-get install -y qt4-dev-tools libqwt-dev python-qt4 + +To use with Qt 5: + +.. code-block:: sh + + dummy@lepka:~> sudo apt-get install -y qtbase5-dev libqt5svg5-dev libqwt-qt5-dev \ + python-pyqt5 + +.. note:: **Do not install both versions of Qwt** (for Qt 4 and Qt 5), + this will confuse the installer and end up with a non functional software + (it uses the headers from one Qt and libraries from the other version). Second step is to create the source directory and pull the |git| repository: @@ -318,6 +346,26 @@ the system. Then proceed with the generic install instructions. +|Coriolis| & Docker +~~~~~~~~~~~~~~~~~~~ + +Under ``bootstrap/docker/`` scripts and configuration files are provided that +allow to rebuild |Alliance| and |Coriolis| and perform the regression tests +of ``alliance-check-toolkit``. You may have a look at the ``Dockerfile.system`` +configuration file to see exactly how to setup a vanilla system to build +|Coriolis|. + +To run the docker tests, call the ``dockerManage.sh`` scripts with the relevant +arguments: + +.. code-block:: sh + + ego@home:debian-9> ../../dockerManage.sh -bS # build both system & coriolis images. + ego@home:debian-9> ../../dockerManage.sh -r # compile & check coriolis. + ego@home:debian-9> ../../dockerManage.sh -C # clear the images. + + + Packaging Coriolis ~~~~~~~~~~~~~~~~~~ diff --git a/documentation/content/pdfs/PythonCpp.pdf b/documentation/content/pdfs/PythonCpp.pdf index ce281ddd..a7349d35 100644 Binary files a/documentation/content/pdfs/PythonCpp.pdf and b/documentation/content/pdfs/PythonCpp.pdf differ diff --git a/documentation/content/pdfs/PythonTutorial.pdf b/documentation/content/pdfs/PythonTutorial.pdf index c89a7885..6d822019 100644 Binary files a/documentation/content/pdfs/PythonTutorial.pdf and b/documentation/content/pdfs/PythonTutorial.pdf differ diff --git a/documentation/content/pdfs/RDS.pdf b/documentation/content/pdfs/RDS.pdf index f15b60a4..a1c06272 100644 Binary files a/documentation/content/pdfs/RDS.pdf and b/documentation/content/pdfs/RDS.pdf differ diff --git a/documentation/content/pdfs/Stratus.pdf b/documentation/content/pdfs/Stratus.pdf index 8efee06b..039a090e 100644 Binary files a/documentation/content/pdfs/Stratus.pdf and b/documentation/content/pdfs/Stratus.pdf differ diff --git a/documentation/content/pdfs/UsersGuide.pdf b/documentation/content/pdfs/UsersGuide.pdf index 51719e77..d79abc34 100644 Binary files a/documentation/content/pdfs/UsersGuide.pdf and b/documentation/content/pdfs/UsersGuide.pdf differ diff --git a/documentation/etc/definitions.rst b/documentation/etc/definitions.rst index b948454d..8530f918 100644 --- a/documentation/etc/definitions.rst +++ b/documentation/etc/definitions.rst @@ -62,6 +62,9 @@ .. |Blif| replace:: :sc:`blif` .. |TSMC| replace:: :sc:`tsmc` .. |AMS| replace:: :sc:`ams` +.. |XFAB| replace:: :sc:`xfab` +.. |XH035| replace:: :cb:`XH035` +.. |ROHM| replace:: :sc:`rohm` .. |Alexandre| replace:: :sc:`Alexandre` .. |Belloeil| replace:: :sc:`Belloeil` @@ -77,14 +80,19 @@ .. |Yifei| replace:: :sc:`Yifei` .. |Mead| replace:: :sc:`Mead` .. |Conway| replace:: :sc:`Conway` +.. |Petley| replace:: :sc:`Petley` +.. |FloresGomez| replace:: :sc:`Flores Gomez` +.. |SNX| replace:: :sc:`snx` .. |ASIC| replace:: :sc:`asic` .. |EDA| replace:: :sc:`eda` .. |RTL| replace:: :sc:`rtl` .. |HSL| replace:: :sc:`hsl` +.. |FHS| replace:: :sc:`fhs` .. |IEEE| replace:: :sc:`ieee` .. |ANSI| replace:: :sc:`ansi` .. |MIPS| replace:: :sc:`mips` +.. |Pharosc| replace:: :sc:`Pharosc` .. |Am2901| replace:: :sc:`Am2901` .. |CAIRO| replace:: :sc:`cairo` .. |CAIRO+| replace:: :sc:`cairo+` @@ -143,6 +151,8 @@ .. |gdb| replace:: :cb:`gdb` .. |valgrind| replace:: :cb:`valgrind` .. |cmake| replace:: :cb:`cmake` +.. |bison| replace:: :cb:`bison` +.. |flex| replace:: :cb:`flex` .. |struct| replace:: :cb:`struct` .. |KeyUp| replace:: :fboxtt:`Up` diff --git a/documentation/output/archives.html b/documentation/output/archives.html index 42b53e19..cfb807f4 100644 --- a/documentation/output/archives.html +++ b/documentation/output/archives.html @@ -8,7 +8,7 @@ - - Coriolis VLSI CAD Tools + - Coriolis VLSI CAD Tools [offline] @@ -44,7 +44,7 @@