libinstaller/libinstaller.py

#!/usr/bin/env python3
import os
import sys
import subprocess
import argparse
import glob
import json
import time
import shutil

SCRIPT_VERSION="v0.1"

def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+
    count = len(it)
    def show(j):
        x = int(size*j/count)
        print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count),
                end='\r', file=out, flush=True)
    show(0)
    for i, item in enumerate(it):
        yield item
        show(i+1)
    print("", flush=True, file=out)

def getFromCommand(command):
    ret=subprocess.check_output(command).decode(sys.stdout.encoding)
    return ret

def getCompilerVersion(compiler):
    if (compiler == "gnu"):
        cc="gcc"
    elif (compiler == "intel"):
        cc="icc"
    elif (compiler == "aocc"):
        cc="clang"
    compstr=getFromCommand([cc,"-dumpversion"]).strip()
    return compstr

def getMpiVersion(mpi):
    if (mpi == "hpcx"):
        hpcxDir = os.getenv('HPCX_DIR')
        verFile = hpcxDir + "/" + "VERSION"
        with open(verFile) as f:
            lines = f.readlines()
        mpistr = lines[0].split()[1].strip()
    elif (mpi == "openmpi"):
        rawstr = getFromCommand(["ompi_info"]).splitlines()[1]
        mpistr = rawstr[rawstr.find(":")+2:]
    elif (mpi == "intelmpi"):
        rawstr = getFromCommand(["ompi_info"]).splitlines()[0]
        mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1]
    return mpistr

def bordered(text):
    lines = text.splitlines()
    width = max(len(s) for s in lines)
    res = ['┌' + '─' * width + '┐']
    for s in lines:
        res.append('│' + (s + ' ' * width)[:width] + '│')
    res.append('└' + '─' * width + '┘')
    return '\n'.join(res)

def underlined(text):
    lines = text.splitlines()
    width = max(len(s) for s in lines)
    res=[]
    for s in lines:
        res.append(s)
    res.append('─' * width)
    return '\n'.join(res)

def printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, installs):
    out = underlined("Patricks Simple Library Installer "+SCRIPT_VERSION)
    out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compilerVersion
    out = out + "\n" + "MPI: " + mpi.upper() + ", Version: " + mpiVersion
    out = out + "\n" + "Install Directory: "+instDir
    out = out + "\n"
    out = out + "\n" + "The following Libraries will be installed:"
    for lib in installs:
        out = out + "\n" + lib['name'] + " v." + lib['version']
    print(bordered(out) + "\n")

    # short sleep to enable user to abprt if selected options are wrong
    for i in progressbar(range(5), "Waiting 5s to start build: ",10):
        time.sleep(1)

    print("\n")

def topological_sort(source):
    pending = [(name, set(deps)) for name, deps in source]
    emitted = []
    while pending:
        next_pending = []
        next_emitted = []
        for entry in pending:
            name, deps = entry
            deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6
            if deps:
                next_pending.append(entry)
            else:
                yield name
                emitted.append(name) # <-- not required, but preserves original order
                next_emitted.append(name)
        if not next_emitted:
            raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
        pending = next_pending
        emitted = next_emitted
    return emitted

def sortLibsByDependencies(selectedLibs):
    depList=[]
    for lib in selectedLibs:
        name = lib['name']
        dependencies = set(lib['dependencies'].split(','))
        if dependencies == {''}:
         dependencies = {}
        depList.append([name,dependencies])
    sortedDepList = topological_sort(depList)
    sorted = []
    for entry in sortedDepList:
        for lib in selectedLibs:
            if lib['name'] == entry:
                sorted.append(lib)
    return sorted



def installLib(lib,srcDir,workDir,instDir,compCC,compCXX,compFC,buildThreads,verbose):
    print(bordered("Installing "+lib['name']+" v."+lib['version']))
    if (not os.path.exists(srcDir)):
        os.makedirs(srcDir)
    if (not os.path.exists(workDir)):
        os.makedirs(workDir)
    if (not os.path.exists(instDir)):
        os.makedirs(instDir)

    os.chdir(srcDir)

    libName = lib['name']
    libVersion = lib['version']
    if (verbose == False):
        logfilePath = workDir + "/" + libName + "-" + libVersion + ".log"
        if (os.path.exists(logfilePath)):
            os.remove(logfilePath)
        logfile = open(logfilePath,'a')
    else:
        logfile = None

    # download lib src if not present
    if (not os.path.exists(lib['name']+"-"+lib['version'])):

        print("Downloading Source Code")
        loadCommand = lib['download'].replace("$VERSION_UNDERSCORE",libVersion.replace(".","_")).replace("$VERSION",libVersion).split()
        err = subprocess.call(loadCommand,stdout=logfile,stderr=logfile)
        if (err != 0):
            print("Error: Download of "+libName+" v."+libVersion+" failed!")
            print("See "+logfilePath+" for details")
            sys.exit(0)
    # configure library
    os.chdir(workDir)
    if (os.path.exists(workDir+"/"+libName+"-"+libVersion)):
        shutil.rmtree(workDir+"/"+libName+"-"+libVersion)
    shutil.copytree(srcDir+"/"+libName+"-"+libVersion, workDir+"/"+libName+"-"+libVersion)
    os.chdir(workDir+"/"+libName+"-"+libVersion)

    #configCommand = lib['configure'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split()
    configCommand = lib['configure'].replace("$PREFIX",instDir)
    if (verbose):
        print(underlined("\nConfiguring Library"))
    else:
        print("Configuring library...")
    err = subprocess.call(configCommand,stdout=logfile,stderr=logfile,shell=True)
    if (err != 0):
        print("Error: Configuring of "+libName+" v."+libVersion+" failed!")
        print("See "+logfilePath+" for details")
        sys.exit(1)

    # build library
    #buildCommand = lib['build'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split()
    buildCommand = lib['build'].replace("$BUILDTHREADS",buildThreads)
    if(verbose):
        print(underlined("\nBuilding Library"))
        err = subprocess.call(buildCommand,stdout=logfile,stderr=logfile,shell=True)
    else:
        buildTask = subprocess.Popen(buildCommand,stdout=logfile,stderr=logfile,shell=True)
        numObjects = int(lib['object files'])
        finished = False
        while (buildTask.poll() == None):
            if (not finished):
                barWidth = numObjects
                #while (barWidth > 100):
                #    barWidth = int(barWidth/10)
                barWidth = 50
                for i in progressbar(range(numObjects), "Building library: ",barWidth):
                    while(len(glob.glob('**/*.o',recursive=True)) < i):
                        time.sleep(0.5)
                        if(buildTask.poll != None):
                            break
                finished = True
        buildTask.wait()
        err=buildTask.returncode

    if (err != 0):
            print("Error: Building "+libName+" v."+libVersion+" failed!")
            print("See "+logfilePath+" for details")
            sys.exit(1)

    #install library
    installCommand = lib['install'].replace("$BUILDTHREADS",buildThreads)
    if(verbose):
        print(underlined("\nInstalling Library"))
    else:
        print("Installing library...")
    err = subprocess.call(installCommand,stdout=logfile,stderr=logfile,shell=True)
    if (err != 0):
            print("Error: Installing "+libName+" v."+libVersion+" failed!")
            print("See "+logfilePath+" for details")
            sys.exit(1)
    print("Library "+libName+" has been installed successfully!\n")


#MAIN

# set up two parsers: first parse config directory, and add dynamic arguments to full parser
config_parser = argparse.ArgumentParser(add_help=False)
parser = argparse.ArgumentParser()

# parsers need to share known arguments
config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
config_parser.add_argument('--src' ,help='Path where to download source  code to [$pwd/src]',default=os.getcwd()+"/src")
config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')

parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
parser.add_argument('--src' ,help='Path where to download source  code to [$pwd/src]',default=os.getcwd()+"/src")
parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')


# run config parser and serach  config/*.json to add a build and version argument for it to the full parser
configDir = config_parser.parse_known_args()[0].config
for cf in glob.glob(configDir+"/*.json"):
    with open(cf, 'r') as f:
        libData = json.load(f)
        parser.add_argument('--'+libData['name'],help='Enable build of '+libData['name'],action='store_true')
        parser.add_argument('--'+libData['name']+'-version',help='Set '+libData['name']+' version ['+libData['default version']+']',default=str(libData['default version']))

# run full parser
argN = parser.parse_args()
args = vars(argN)

# extract known arguments
prefix = argN.prefix
srcDir = argN.src
workDir = argN.work
keepWork = argN.keep_work
compiler = argN.compiler
mpi = argN.mpi
buildThreads = argN.threads
verbose = argN.verbose

# extract libraries and versions selected for installation
selectedLibs=[]
ignoreNames=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"]
for libName in args:
    if(libName not in ignoreNames and "version" not in libName):
        install = getattr(argN,libName)
        if (install == True):
            version = getattr(argN,libName+"_version")
            configFile = configDir+"/"+libName+".json"
            with open(configFile,'r') as cf:
                data = json.load(cf)
                data['version'] = version
                selectedLibs.append(data)

# Set up install directory
compilerVersion=getCompilerVersion(compiler)
mpiVersion=getMpiVersion(mpi)
instStr="inst-"+compiler+"_"+compilerVersion+"_"+mpi+"_"+mpiVersion
instDir=prefix+"/"+instStr

# setting MPI wrappers:
if (compiler == "gnu"):
    if (mpi == "hpcx" or mpi == "openmpi"):
        CC="mpicc"
        CXX="mpic++"
        FC="mpifort"
    elif (mpi == "intel"):
        CC="mpicc"
        CXX="mpicxx"
        FC="mpif90"
elif (compiler == "aocc"):
    if (mpi == "hpcx" or mpi == "openmpi"):
        CC="mpicc"
        CXX="mpic++"
        FC="mpifort"
        os.environ["OMPI_MPICC"] = "clang"
        os.environ["OMPI_MPICXX"] = "clang++"
        os.environ["OMPI_MPIFC"] = "flang"
        os.environ["OMPI_MPIF90"] = "flang"
        os.environ["OMPI_MPIF77"] = "flang"
    elif (mpi == "intel"):
        CC="\"mpicc -cc=clang\""
        CXX="\"mpicxx -cxx=clang++\""
        FC="\"mpif90 -fc=flang\""
elif (compiler == "intel"):
    if (mpi  == "hpcx" or mpi == "openmpi"):
        CC="mpicc"
        CXX="mpic++"
        FC="mpifort"
        os.environ["OMPI_MPICC"] = "icc"
        os.environ["OMPI_MPICXX"] = "icpc"
        os.environ["OMPI_MPIFC"] = "ifort"
        os.environ["OMPI_MPIF90"] = "ifort"
        os.environ["OMPI_MPIF77"] = "ifort"
    elif (mpi == "intel"):
        CC="mpiicc"
        CXX="mpiicpc"
        FC="mpiifort"

os.environ["CC"]=CC
os.environ["CXX"]=CXX
os.environ["FC"]=FC
os.environ["F90"]=FC
os.environ["F77"]=FC

sortedLibs=sortLibsByDependencies(selectedLibs)
printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, sortedLibs)

for lib in sortedLibs:
    installLib(lib,srcDir,workDir,instDir,CC,CXX,FC,buildThreads,verbose)
print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+instDir+" to your environment"))


print("Cleaning up work directory...")
shutil.rmtree(workDir)
print("Done.")


print("\n")