#!/usr/bin/env python3 import os import sys import subprocess import argparse import glob import json import time import shutil SCRIPT_VERSION="v0.1" def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+ count = len(it) def show(j): x = int(size*j/count) print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count), end='\r', file=out, flush=True) show(0) for i, item in enumerate(it): yield item show(i+1) print("", flush=True, file=out) def getFromCommand(command): ret=subprocess.check_output(command).decode(sys.stdout.encoding) return ret def getCompilerVersion(compiler): if (compiler == "gnu"): cc="gcc" elif (compiler == "intel"): cc="icc" elif (compiler == "aocc"): cc="clang" compstr=getFromCommand([cc,"-dumpversion"]).strip() return compstr def getMpiVersion(mpi): if (mpi == "hpcx"): hpcxDir = os.getenv('HPCX_DIR') verFile = hpcxDir + "/" + "VERSION" with open(verFile) as f: lines = f.readlines() mpistr = lines[0].split()[1].strip() elif (mpi == "openmpi"): rawstr = getFromCommand(["ompi_info"]).splitlines()[1] mpistr = rawstr[rawstr.find(":")+2:] elif (mpi == "intelmpi"): rawstr = getFromCommand(["ompi_info"]).splitlines()[0] mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1] return mpistr def bordered(text): lines = text.splitlines() width = max(len(s) for s in lines) res = ['┌' + '─' * width + '┐'] for s in lines: res.append('│' + (s + ' ' * width)[:width] + '│') res.append('└' + '─' * width + '┘') return '\n'.join(res) def underlined(text): lines = text.splitlines() width = max(len(s) for s in lines) res=[] for s in lines: res.append(s) res.append('─' * width) return '\n'.join(res) def printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, installs): out = underlined("Patricks Simple Library Installer "+SCRIPT_VERSION) out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compilerVersion out = out + "\n" + "MPI: " + mpi.upper() + ", Version: " + mpiVersion out = out + "\n" + "Install Directory: "+instDir out = out + "\n" out = out + "\n" + "The following Libraries will be installed:" for lib in installs: out = out + "\n" + lib['name'] + " v." + lib['version'] print(bordered(out) + "\n") # short sleep to enable user to abprt if selected options are wrong for i in progressbar(range(5), "Waiting 5s to start build: ",10): time.sleep(1) print("\n") def topological_sort(source): pending = [(name, set(deps)) for name, deps in source] emitted = [] while pending: next_pending = [] next_emitted = [] for entry in pending: name, deps = entry deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6 if deps: next_pending.append(entry) else: yield name emitted.append(name) # <-- not required, but preserves original order next_emitted.append(name) if not next_emitted: raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,))) pending = next_pending emitted = next_emitted return emitted def sortLibsByDependencies(selectedLibs): depList=[] for lib in selectedLibs: name = lib['name'] dependencies = set(lib['dependencies'].split(',')) if dependencies == {''}: dependencies = {} depList.append([name,dependencies]) sortedDepList = topological_sort(depList) sorted = [] for entry in sortedDepList: for lib in selectedLibs: if lib['name'] == entry: sorted.append(lib) return sorted def installLib(lib,srcDir,workDir,instDir,compCC,compCXX,compFC,buildThreads,verbose): print(bordered("Installing "+lib['name']+" v."+lib['version'])) if (not os.path.exists(srcDir)): os.makedirs(srcDir) if (not os.path.exists(workDir)): os.makedirs(workDir) if (not os.path.exists(instDir)): os.makedirs(instDir) os.chdir(srcDir) libName = lib['name'] libVersion = lib['version'] if (verbose == False): logfilePath = workDir + "/" + libName + "-" + libVersion + ".log" if (os.path.exists(logfilePath)): os.remove(logfilePath) logfile = open(logfilePath,'a') else: logfile = None # download lib src if not present if (not os.path.exists(lib['name']+"-"+lib['version'])): print("Downloading Source Code") loadCommand = lib['download'].replace("$VERSION_UNDERSCORE",libVersion.replace(".","_")).replace("$VERSION",libVersion).split() err = subprocess.call(loadCommand,stdout=logfile,stderr=logfile) if (err != 0): print("Error: Download of "+libName+" v."+libVersion+" failed!") print("See "+logfilePath+" for details") sys.exit(0) # configure library os.chdir(workDir) if (os.path.exists(workDir+"/"+libName+"-"+libVersion)): shutil.rmtree(workDir+"/"+libName+"-"+libVersion) shutil.copytree(srcDir+"/"+libName+"-"+libVersion, workDir+"/"+libName+"-"+libVersion) os.chdir(workDir+"/"+libName+"-"+libVersion) #configCommand = lib['configure'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split() configCommand = lib['configure'].replace("$PREFIX",instDir) if (verbose): print(underlined("\nConfiguring Library")) else: print("Configuring library...") err = subprocess.call(configCommand,stdout=logfile,stderr=logfile,shell=True) if (err != 0): print("Error: Configuring of "+libName+" v."+libVersion+" failed!") print("See "+logfilePath+" for details") sys.exit(1) # build library #buildCommand = lib['build'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split() buildCommand = lib['build'].replace("$BUILDTHREADS",buildThreads) if(verbose): print(underlined("\nBuilding Library")) err = subprocess.call(buildCommand,stdout=logfile,stderr=logfile,shell=True) else: buildTask = subprocess.Popen(buildCommand,stdout=logfile,stderr=logfile,shell=True) numObjects = int(lib['object files']) finished = False while (buildTask.poll() == None): if (not finished): barWidth = numObjects #while (barWidth > 100): # barWidth = int(barWidth/10) barWidth = 50 for i in progressbar(range(numObjects), "Building library: ",barWidth): while(len(glob.glob('**/*.o',recursive=True)) < i): time.sleep(0.5) if(buildTask.poll != None): break finished = True buildTask.wait() err=buildTask.returncode if (err != 0): print("Error: Building "+libName+" v."+libVersion+" failed!") print("See "+logfilePath+" for details") sys.exit(1) #install library installCommand = lib['install'].replace("$BUILDTHREADS",buildThreads) if(verbose): print(underlined("\nInstalling Library")) else: print("Installing library...") err = subprocess.call(installCommand,stdout=logfile,stderr=logfile,shell=True) if (err != 0): print("Error: Installing "+libName+" v."+libVersion+" failed!") print("See "+logfilePath+" for details") sys.exit(1) print("Library "+libName+" has been installed successfully!\n") #MAIN # set up two parsers: first parse config directory, and add dynamic arguments to full parser config_parser = argparse.ArgumentParser(add_help=False) parser = argparse.ArgumentParser() # parsers need to share known arguments config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work") config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work") parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') # run config parser and serach config/*.json to add a build and version argument for it to the full parser configDir = config_parser.parse_known_args()[0].config for cf in glob.glob(configDir+"/*.json"): with open(cf, 'r') as f: libData = json.load(f) parser.add_argument('--'+libData['name'],help='Enable build of '+libData['name'],action='store_true') parser.add_argument('--'+libData['name']+'-version',help='Set '+libData['name']+' version ['+libData['default version']+']',default=str(libData['default version'])) # run full parser argN = parser.parse_args() args = vars(argN) # extract known arguments prefix = argN.prefix srcDir = argN.src workDir = argN.work keepWork = argN.keep_work compiler = argN.compiler mpi = argN.mpi buildThreads = argN.threads verbose = argN.verbose # extract libraries and versions selected for installation selectedLibs=[] ignoreNames=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"] for libName in args: if(libName not in ignoreNames and "version" not in libName): install = getattr(argN,libName) if (install == True): version = getattr(argN,libName+"_version") configFile = configDir+"/"+libName+".json" with open(configFile,'r') as cf: data = json.load(cf) data['version'] = version selectedLibs.append(data) # Set up install directory compilerVersion=getCompilerVersion(compiler) mpiVersion=getMpiVersion(mpi) instStr="inst-"+compiler+"_"+compilerVersion+"_"+mpi+"_"+mpiVersion instDir=prefix+"/"+instStr # setting MPI wrappers: if (compiler == "gnu"): if (mpi == "hpcx" or mpi == "openmpi"): CC="mpicc" CXX="mpic++" FC="mpifort" elif (mpi == "intel"): CC="mpicc" CXX="mpicxx" FC="mpif90" elif (compiler == "aocc"): if (mpi == "hpcx" or mpi == "openmpi"): CC="mpicc" CXX="mpic++" FC="mpifort" os.environ["OMPI_MPICC"] = "clang" os.environ["OMPI_MPICXX"] = "clang++" os.environ["OMPI_MPIFC"] = "flang" os.environ["OMPI_MPIF90"] = "flang" os.environ["OMPI_MPIF77"] = "flang" elif (mpi == "intel"): CC="\"mpicc -cc=clang\"" CXX="\"mpicxx -cxx=clang++\"" FC="\"mpif90 -fc=flang\"" elif (compiler == "intel"): if (mpi == "hpcx" or mpi == "openmpi"): CC="mpicc" CXX="mpic++" FC="mpifort" os.environ["OMPI_MPICC"] = "icc" os.environ["OMPI_MPICXX"] = "icpc" os.environ["OMPI_MPIFC"] = "ifort" os.environ["OMPI_MPIF90"] = "ifort" os.environ["OMPI_MPIF77"] = "ifort" elif (mpi == "intel"): CC="mpiicc" CXX="mpiicpc" FC="mpiifort" os.environ["CC"]=CC os.environ["CXX"]=CXX os.environ["FC"]=FC os.environ["F90"]=FC os.environ["F77"]=FC sortedLibs=sortLibsByDependencies(selectedLibs) printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, sortedLibs) for lib in sortedLibs: installLib(lib,srcDir,workDir,instDir,CC,CXX,FC,buildThreads,verbose) print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+instDir+" to your environment")) print("Cleaning up work directory...") shutil.rmtree(workDir) print("Done.") print("\n")