diff --git a/config/hdf5.json b/config/hdf5.json new file mode 100644 index 0000000..ac8c731 --- /dev/null +++ b/config/hdf5.json @@ -0,0 +1,10 @@ +{ + "name" : "hdf5", + "dependencies" : "zlib,szip", + "default version" : "1.13.2", + "download" : "git clone --depth 1 --branch hdf5-$VERSION_UNDERSCORE https://github.com/HDFGroup/hdf5.git hdf5-$VERSION", + "configure" : "./configure --enable-fortran --enable-parallel --with-zlib=$PREFIX --with-szip=$PREFIX --prefix=$PREFIX", + "build" : "make -j $BUILDTHREADS", + "install" : "make install", + "object files" : "1136" +} \ No newline at end of file diff --git a/config/netcdf_c.json b/config/netcdf_c.json new file mode 100644 index 0000000..0631261 --- /dev/null +++ b/config/netcdf_c.json @@ -0,0 +1,4 @@ +{ + "name" : "netcdf-c", + "default version" : "4.2.1" +} \ No newline at end of file diff --git a/config/netcdf_f.json b/config/netcdf_f.json new file mode 100644 index 0000000..a922f7b --- /dev/null +++ b/config/netcdf_f.json @@ -0,0 +1,4 @@ +{ + "name" : "netcdf-f", + "default version" : "4.6.0" +} \ No newline at end of file diff --git a/config/szip.json b/config/szip.json new file mode 100644 index 0000000..e6af536 --- /dev/null +++ b/config/szip.json @@ -0,0 +1,10 @@ +{ + "name" : "szip", + "dependencies" : "", + "default version" : "2.1.1", + "download" : "git clone https://github.com/erdc/szip.git szip-$VERSION", + "configure" : "./configure --prefix=$PREFIX", + "build" : "make -j $BUILDTHREADS", + "install" : "make install", + "object files" : "7" +} \ No newline at end of file diff --git a/config/zlib.json b/config/zlib.json new file mode 100644 index 0000000..8168e01 --- /dev/null +++ b/config/zlib.json @@ -0,0 +1,10 @@ +{ + "name" : "zlib", + "dependencies" : "", + "default version" : "1.2.11", + "download" : "git clone --depth 1 --branch v$VERSION https://github.com/madler/zlib.git zlib-$VERSION", + "configure" : "./configure --prefix=$PREFIX", + "build" : "make -j $BUILDTHREADS", + "install" : "make install", + "object files" : "19" +} \ No newline at end of file diff --git a/libinstaller.py b/libinstaller.py new file mode 100755 index 0000000..2ea6cb0 --- /dev/null +++ b/libinstaller.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import argparse +import glob +import json +import time +import shutil + +SCRIPT_VERSION="v0.1" + +def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+ + count = len(it) + def show(j): + x = int(size*j/count) + print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count), + end='\r', file=out, flush=True) + show(0) + for i, item in enumerate(it): + yield item + show(i+1) + print("", flush=True, file=out) + +def getFromCommand(command): + ret=subprocess.check_output(command).decode(sys.stdout.encoding) + return ret + +def getCompilerVersion(compiler): + if (compiler == "gnu"): + cc="gcc" + elif (compiler == "intel"): + cc="icc" + elif (compiler == "aocc"): + cc="clang" + compstr=getFromCommand([cc,"-dumpversion"]).strip() + return compstr + +def getMpiVersion(mpi): + if (mpi == "hpcx"): + hpcxDir = os.getenv('HPCX_DIR') + verFile = hpcxDir + "/" + "VERSION" + with open(verFile) as f: + lines = f.readlines() + mpistr = lines[0].split()[1].strip() + elif (mpi == "openmpi"): + rawstr = getFromCommand(["ompi_info"]).splitlines()[1] + mpistr = rawstr[rawstr.find(":")+2:] + elif (mpi == "intelmpi"): + rawstr = getFromCommand(["ompi_info"]).splitlines()[0] + mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1] + return mpistr + +def bordered(text): + lines = text.splitlines() + width = max(len(s) for s in lines) + res = ['┌' + '─' * width + '┐'] + for s in lines: + res.append('│' + (s + ' ' * width)[:width] + '│') + res.append('└' + '─' * width + '┘') + return '\n'.join(res) + +def underlined(text): + lines = text.splitlines() + width = max(len(s) for s in lines) + res=[] + for s in lines: + res.append(s) + res.append('─' * width) + return '\n'.join(res) + +def printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, installs): + out = underlined("Patricks Simple Library Installer "+SCRIPT_VERSION) + out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compilerVersion + out = out + "\n" + "MPI: " + mpi.upper() + ", Version: " + mpiVersion + out = out + "\n" + "Install Directory: "+instDir + out = out + "\n" + out = out + "\n" + "The following Libraries will be installed:" + for lib in installs: + out = out + "\n" + lib['name'] + " v." + lib['version'] + print(bordered(out) + "\n") + + # short sleep to enable user to abprt if selected options are wrong + for i in progressbar(range(5), "Waiting 5s to start build: ",10): + time.sleep(1) + + print("\n") + +def topological_sort(source): + pending = [(name, set(deps)) for name, deps in source] + emitted = [] + while pending: + next_pending = [] + next_emitted = [] + for entry in pending: + name, deps = entry + deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6 + if deps: + next_pending.append(entry) + else: + yield name + emitted.append(name) # <-- not required, but preserves original order + next_emitted.append(name) + if not next_emitted: + raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,))) + pending = next_pending + emitted = next_emitted + return emitted + +def sortLibsByDependencies(selectedLibs): + depList=[] + for lib in selectedLibs: + name = lib['name'] + dependencies = set(lib['dependencies'].split(',')) + if dependencies == {''}: + dependencies = {} + depList.append([name,dependencies]) + sortedDepList = topological_sort(depList) + sorted = [] + for entry in sortedDepList: + for lib in selectedLibs: + if lib['name'] == entry: + sorted.append(lib) + return sorted + + + +def installLib(lib,srcDir,workDir,instDir,compCC,compCXX,compFC,buildThreads,verbose): + print(bordered("Installing "+lib['name']+" v."+lib['version'])) + if (not os.path.exists(srcDir)): + os.makedirs(srcDir) + if (not os.path.exists(workDir)): + os.makedirs(workDir) + if (not os.path.exists(instDir)): + os.makedirs(instDir) + + os.chdir(srcDir) + + libName = lib['name'] + libVersion = lib['version'] + if (verbose == False): + logfilePath = workDir + "/" + libName + "-" + libVersion + ".log" + if (os.path.exists(logfilePath)): + os.remove(logfilePath) + logfile = open(logfilePath,'a') + else: + logfile = None + + # download lib src if not present + if (not os.path.exists(lib['name']+"-"+lib['version'])): + + print("Downloading Source Code") + loadCommand = lib['download'].replace("$VERSION_UNDERSCORE",libVersion.replace(".","_")).replace("$VERSION",libVersion).split() + err = subprocess.call(loadCommand,stdout=logfile,stderr=logfile) + if (err != 0): + print("Error: Download of "+libName+" v."+libVersion+" failed!") + print("See "+logfilePath+" for details") + exit + # configure library + os.chdir(workDir) + if (os.path.exists(workDir+"/"+libName+"-"+libVersion)): + shutil.rmtree(workDir+"/"+libName+"-"+libVersion) + shutil.copytree(srcDir+"/"+libName+"-"+libVersion, workDir+"/"+libName+"-"+libVersion) + os.chdir(workDir+"/"+libName+"-"+libVersion) + + #configCommand = lib['configure'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split() + configCommand = lib['configure'].replace("$PREFIX",instDir) + if (verbose): + print(underlined("\nConfiguring Library")) + else: + print("Configuring library...") + err = subprocess.call(configCommand,stdout=logfile,stderr=logfile,shell=True) + if (err != 0): + print("Error: Configuring of "+libName+" v."+libVersion+" failed!") + print("See "+logfilePath+" for details") + exit + + # build library + #buildCommand = lib['build'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split() + buildCommand = lib['build'].replace("$BUILDTHREADS",buildThreads) + if(verbose): + print(underlined("\nBuilding Library")) + err = subprocess.call(buildCommand,stdout=logfile,stderr=logfile,shell=True) + else: + buildTask = subprocess.Popen(buildCommand,stdout=logfile,stderr=logfile,shell=True) + numObjects = int(lib['object files']) + finished = False + while (buildTask.poll() == None): + if (not finished): + barWidth = numObjects + #while (barWidth > 100): + # barWidth = int(barWidth/10) + barWidth = 50 + for i in progressbar(range(numObjects), "Building library: ",barWidth): + while(len(glob.glob('**/*.o',recursive=True)) < i): + time.sleep(0.5) + if(buildTask.poll != None): + break + finished = True + buildTask.wait() + err=buildTask.returncode + + if (err != 0): + print("Error: Building "+libName+" v."+libVersion+" failed!") + print("See "+logfilePath+" for details") + exit + + #install library + installCommand = lib['install'].replace("$BUILDTHREADS",buildThreads) + if(verbose): + print(underlined("\nInstalling Library")) + else: + print("Installing library...") + err = subprocess.call(installCommand,stdout=logfile,stderr=logfile,shell=True) + if (err != 0): + print("Error: Installing "+libName+" v."+libVersion+" failed!") + print("See "+logfilePath+" for details") + exit + print("Library "+libName+" has been installed successfully!\n") + + +#MAIN + +# set up two parsers: first parse config directory, and add dynamic arguments to full parser +config_parser = argparse.ArgumentParser(add_help=False) +parser = argparse.ArgumentParser() + +# parsers need to share known arguments +config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") +config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) +config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") +config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work") +config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') +config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') +config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') +config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') +config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') + +parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") +parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) +parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") +parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work") +parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') +parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') +parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') +parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') +parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') + + +# run config parser and serach config/*.json to add a build and version argument for it to the full parser +configDir = config_parser.parse_known_args()[0].config +for cf in glob.glob(configDir+"/*.json"): + with open(cf, 'r') as f: + libData = json.load(f) + parser.add_argument('--'+libData['name'],help='Enable build of '+libData['name'],action='store_true') + parser.add_argument('--'+libData['name']+'-version',help='Set '+libData['name']+' version ['+libData['default version']+']',default=str(libData['default version'])) + +# run full parser +argN = parser.parse_args() +args = vars(argN) + +# extract known arguments +prefix = argN.prefix +srcDir = argN.src +workDir = argN.work +keepWork = argN.keep_work +compiler = argN.compiler +mpi = argN.mpi +buildThreads = argN.threads +verbose = argN.verbose + +# extract libraries and versions selected for installation +selectedLibs=[] +ignoreNames=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"] +for libName in args: + if(libName not in ignoreNames and "version" not in libName): + install = getattr(argN,libName) + if (install == True): + version = getattr(argN,libName+"_version") + configFile = configDir+"/"+libName+".json" + with open(configFile,'r') as cf: + data = json.load(cf) + data['version'] = version + selectedLibs.append(data) + +# Set up install directory +compilerVersion=getCompilerVersion(compiler) +mpiVersion=getMpiVersion(mpi) +instStr="inst-"+compiler+"_"+compilerVersion+"_"+mpi+"_"+mpiVersion +instDir=prefix+"/"+instStr + +# setting MPI wrappers: +if (compiler == "gnu"): + if (mpi == "hpcx" or mpi == "openmpi"): + CC="mpicc" + CXX="mpic++" + FC="mpifort" + elif (mpi == "intel"): + CC="mpicc" + CXX="mpicxx" + FC="mpif90" +elif (compiler == "aocc"): + if (mpi == "hpcx" or mpi == "openmpi"): + CC="mpicc" + CXX="mpic++" + FC="mpifort" + os.environ["OMPI_MPICC"] = "clang" + os.environ["OMPI_MPICXX"] = "clang++" + os.environ["OMPI_MPIFC"] = "flang" + os.environ["OMPI_MPIF90"] = "flang" + os.environ["OMPI_MPIF77"] = "flang" + elif (mpi == "intel"): + CC="\"mpicc -cc=clang\"" + CXX="\"mpicxx -cxx=clang++\"" + FC="\"mpif90 -fc=flang\"" +elif (compiler == "intel"): + if (mpi == "hpcx" or mpi == "openmpi"): + CC="mpicc" + CXX="mpic++" + FC="mpifort" + os.environ["OMPI_MPICC"] = "icc" + os.environ["OMPI_MPICXX"] = "icpc" + os.environ["OMPI_MPIFC"] = "ifort" + os.environ["OMPI_MPIF90"] = "ifort" + os.environ["OMPI_MPIF77"] = "ifort" + elif (mpi == "intel"): + CC="mpiicc" + CXX="mpiicpc" + FC="mpiifort" + +os.environ["CC"]=CC +os.environ["CXX"]=CXX +os.environ["FC"]=FC +os.environ["F90"]=FC +os.environ["F77"]=FC + +sortedLibs=sortLibsByDependencies(selectedLibs) +printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, sortedLibs) + +for lib in sortedLibs: + installLib(lib,srcDir,workDir,instDir,CC,CXX,FC,buildThreads,verbose) +print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+instDir+" to your environment")) + + +print("Cleaning up work directory...") +shutil.rmtree(workDir) +print("Done.") + + +print("\n")