|
|
|
|
@ -0,0 +1,350 @@
|
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
import os
|
|
|
|
|
import sys
|
|
|
|
|
import subprocess
|
|
|
|
|
import argparse
|
|
|
|
|
import glob
|
|
|
|
|
import json
|
|
|
|
|
import time
|
|
|
|
|
import shutil
|
|
|
|
|
|
|
|
|
|
SCRIPT_VERSION="v0.1"
|
|
|
|
|
|
|
|
|
|
def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+
|
|
|
|
|
count = len(it)
|
|
|
|
|
def show(j):
|
|
|
|
|
x = int(size*j/count)
|
|
|
|
|
print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count),
|
|
|
|
|
end='\r', file=out, flush=True)
|
|
|
|
|
show(0)
|
|
|
|
|
for i, item in enumerate(it):
|
|
|
|
|
yield item
|
|
|
|
|
show(i+1)
|
|
|
|
|
print("", flush=True, file=out)
|
|
|
|
|
|
|
|
|
|
def getFromCommand(command):
|
|
|
|
|
ret=subprocess.check_output(command).decode(sys.stdout.encoding)
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
def getCompilerVersion(compiler):
|
|
|
|
|
if (compiler == "gnu"):
|
|
|
|
|
cc="gcc"
|
|
|
|
|
elif (compiler == "intel"):
|
|
|
|
|
cc="icc"
|
|
|
|
|
elif (compiler == "aocc"):
|
|
|
|
|
cc="clang"
|
|
|
|
|
compstr=getFromCommand([cc,"-dumpversion"]).strip()
|
|
|
|
|
return compstr
|
|
|
|
|
|
|
|
|
|
def getMpiVersion(mpi):
|
|
|
|
|
if (mpi == "hpcx"):
|
|
|
|
|
hpcxDir = os.getenv('HPCX_DIR')
|
|
|
|
|
verFile = hpcxDir + "/" + "VERSION"
|
|
|
|
|
with open(verFile) as f:
|
|
|
|
|
lines = f.readlines()
|
|
|
|
|
mpistr = lines[0].split()[1].strip()
|
|
|
|
|
elif (mpi == "openmpi"):
|
|
|
|
|
rawstr = getFromCommand(["ompi_info"]).splitlines()[1]
|
|
|
|
|
mpistr = rawstr[rawstr.find(":")+2:]
|
|
|
|
|
elif (mpi == "intelmpi"):
|
|
|
|
|
rawstr = getFromCommand(["ompi_info"]).splitlines()[0]
|
|
|
|
|
mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1]
|
|
|
|
|
return mpistr
|
|
|
|
|
|
|
|
|
|
def bordered(text):
|
|
|
|
|
lines = text.splitlines()
|
|
|
|
|
width = max(len(s) for s in lines)
|
|
|
|
|
res = ['┌' + '─' * width + '┐']
|
|
|
|
|
for s in lines:
|
|
|
|
|
res.append('│' + (s + ' ' * width)[:width] + '│')
|
|
|
|
|
res.append('└' + '─' * width + '┘')
|
|
|
|
|
return '\n'.join(res)
|
|
|
|
|
|
|
|
|
|
def underlined(text):
|
|
|
|
|
lines = text.splitlines()
|
|
|
|
|
width = max(len(s) for s in lines)
|
|
|
|
|
res=[]
|
|
|
|
|
for s in lines:
|
|
|
|
|
res.append(s)
|
|
|
|
|
res.append('─' * width)
|
|
|
|
|
return '\n'.join(res)
|
|
|
|
|
|
|
|
|
|
def printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, installs):
|
|
|
|
|
out = underlined("Patricks Simple Library Installer "+SCRIPT_VERSION)
|
|
|
|
|
out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compilerVersion
|
|
|
|
|
out = out + "\n" + "MPI: " + mpi.upper() + ", Version: " + mpiVersion
|
|
|
|
|
out = out + "\n" + "Install Directory: "+instDir
|
|
|
|
|
out = out + "\n"
|
|
|
|
|
out = out + "\n" + "The following Libraries will be installed:"
|
|
|
|
|
for lib in installs:
|
|
|
|
|
out = out + "\n" + lib['name'] + " v." + lib['version']
|
|
|
|
|
print(bordered(out) + "\n")
|
|
|
|
|
|
|
|
|
|
# short sleep to enable user to abprt if selected options are wrong
|
|
|
|
|
for i in progressbar(range(5), "Waiting 5s to start build: ",10):
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
|
|
|
|
|
print("\n")
|
|
|
|
|
|
|
|
|
|
def topological_sort(source):
|
|
|
|
|
pending = [(name, set(deps)) for name, deps in source]
|
|
|
|
|
emitted = []
|
|
|
|
|
while pending:
|
|
|
|
|
next_pending = []
|
|
|
|
|
next_emitted = []
|
|
|
|
|
for entry in pending:
|
|
|
|
|
name, deps = entry
|
|
|
|
|
deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6
|
|
|
|
|
if deps:
|
|
|
|
|
next_pending.append(entry)
|
|
|
|
|
else:
|
|
|
|
|
yield name
|
|
|
|
|
emitted.append(name) # <-- not required, but preserves original order
|
|
|
|
|
next_emitted.append(name)
|
|
|
|
|
if not next_emitted:
|
|
|
|
|
raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
|
|
|
|
|
pending = next_pending
|
|
|
|
|
emitted = next_emitted
|
|
|
|
|
return emitted
|
|
|
|
|
|
|
|
|
|
def sortLibsByDependencies(selectedLibs):
|
|
|
|
|
depList=[]
|
|
|
|
|
for lib in selectedLibs:
|
|
|
|
|
name = lib['name']
|
|
|
|
|
dependencies = set(lib['dependencies'].split(','))
|
|
|
|
|
if dependencies == {''}:
|
|
|
|
|
dependencies = {}
|
|
|
|
|
depList.append([name,dependencies])
|
|
|
|
|
sortedDepList = topological_sort(depList)
|
|
|
|
|
sorted = []
|
|
|
|
|
for entry in sortedDepList:
|
|
|
|
|
for lib in selectedLibs:
|
|
|
|
|
if lib['name'] == entry:
|
|
|
|
|
sorted.append(lib)
|
|
|
|
|
return sorted
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def installLib(lib,srcDir,workDir,instDir,compCC,compCXX,compFC,buildThreads,verbose):
|
|
|
|
|
print(bordered("Installing "+lib['name']+" v."+lib['version']))
|
|
|
|
|
if (not os.path.exists(srcDir)):
|
|
|
|
|
os.makedirs(srcDir)
|
|
|
|
|
if (not os.path.exists(workDir)):
|
|
|
|
|
os.makedirs(workDir)
|
|
|
|
|
if (not os.path.exists(instDir)):
|
|
|
|
|
os.makedirs(instDir)
|
|
|
|
|
|
|
|
|
|
os.chdir(srcDir)
|
|
|
|
|
|
|
|
|
|
libName = lib['name']
|
|
|
|
|
libVersion = lib['version']
|
|
|
|
|
if (verbose == False):
|
|
|
|
|
logfilePath = workDir + "/" + libName + "-" + libVersion + ".log"
|
|
|
|
|
if (os.path.exists(logfilePath)):
|
|
|
|
|
os.remove(logfilePath)
|
|
|
|
|
logfile = open(logfilePath,'a')
|
|
|
|
|
else:
|
|
|
|
|
logfile = None
|
|
|
|
|
|
|
|
|
|
# download lib src if not present
|
|
|
|
|
if (not os.path.exists(lib['name']+"-"+lib['version'])):
|
|
|
|
|
|
|
|
|
|
print("Downloading Source Code")
|
|
|
|
|
loadCommand = lib['download'].replace("$VERSION_UNDERSCORE",libVersion.replace(".","_")).replace("$VERSION",libVersion).split()
|
|
|
|
|
err = subprocess.call(loadCommand,stdout=logfile,stderr=logfile)
|
|
|
|
|
if (err != 0):
|
|
|
|
|
print("Error: Download of "+libName+" v."+libVersion+" failed!")
|
|
|
|
|
print("See "+logfilePath+" for details")
|
|
|
|
|
exit
|
|
|
|
|
# configure library
|
|
|
|
|
os.chdir(workDir)
|
|
|
|
|
if (os.path.exists(workDir+"/"+libName+"-"+libVersion)):
|
|
|
|
|
shutil.rmtree(workDir+"/"+libName+"-"+libVersion)
|
|
|
|
|
shutil.copytree(srcDir+"/"+libName+"-"+libVersion, workDir+"/"+libName+"-"+libVersion)
|
|
|
|
|
os.chdir(workDir+"/"+libName+"-"+libVersion)
|
|
|
|
|
|
|
|
|
|
#configCommand = lib['configure'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split()
|
|
|
|
|
configCommand = lib['configure'].replace("$PREFIX",instDir)
|
|
|
|
|
if (verbose):
|
|
|
|
|
print(underlined("\nConfiguring Library"))
|
|
|
|
|
else:
|
|
|
|
|
print("Configuring library...")
|
|
|
|
|
err = subprocess.call(configCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
|
|
|
if (err != 0):
|
|
|
|
|
print("Error: Configuring of "+libName+" v."+libVersion+" failed!")
|
|
|
|
|
print("See "+logfilePath+" for details")
|
|
|
|
|
exit
|
|
|
|
|
|
|
|
|
|
# build library
|
|
|
|
|
#buildCommand = lib['build'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split()
|
|
|
|
|
buildCommand = lib['build'].replace("$BUILDTHREADS",buildThreads)
|
|
|
|
|
if(verbose):
|
|
|
|
|
print(underlined("\nBuilding Library"))
|
|
|
|
|
err = subprocess.call(buildCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
|
|
|
else:
|
|
|
|
|
buildTask = subprocess.Popen(buildCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
|
|
|
numObjects = int(lib['object files'])
|
|
|
|
|
finished = False
|
|
|
|
|
while (buildTask.poll() == None):
|
|
|
|
|
if (not finished):
|
|
|
|
|
barWidth = numObjects
|
|
|
|
|
#while (barWidth > 100):
|
|
|
|
|
# barWidth = int(barWidth/10)
|
|
|
|
|
barWidth = 50
|
|
|
|
|
for i in progressbar(range(numObjects), "Building library: ",barWidth):
|
|
|
|
|
while(len(glob.glob('**/*.o',recursive=True)) < i):
|
|
|
|
|
time.sleep(0.5)
|
|
|
|
|
if(buildTask.poll != None):
|
|
|
|
|
break
|
|
|
|
|
finished = True
|
|
|
|
|
buildTask.wait()
|
|
|
|
|
err=buildTask.returncode
|
|
|
|
|
|
|
|
|
|
if (err != 0):
|
|
|
|
|
print("Error: Building "+libName+" v."+libVersion+" failed!")
|
|
|
|
|
print("See "+logfilePath+" for details")
|
|
|
|
|
exit
|
|
|
|
|
|
|
|
|
|
#install library
|
|
|
|
|
installCommand = lib['install'].replace("$BUILDTHREADS",buildThreads)
|
|
|
|
|
if(verbose):
|
|
|
|
|
print(underlined("\nInstalling Library"))
|
|
|
|
|
else:
|
|
|
|
|
print("Installing library...")
|
|
|
|
|
err = subprocess.call(installCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
|
|
|
if (err != 0):
|
|
|
|
|
print("Error: Installing "+libName+" v."+libVersion+" failed!")
|
|
|
|
|
print("See "+logfilePath+" for details")
|
|
|
|
|
exit
|
|
|
|
|
print("Library "+libName+" has been installed successfully!\n")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#MAIN
|
|
|
|
|
|
|
|
|
|
# set up two parsers: first parse config directory, and add dynamic arguments to full parser
|
|
|
|
|
config_parser = argparse.ArgumentParser(add_help=False)
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
|
|
|
|
|
|
# parsers need to share known arguments
|
|
|
|
|
config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
|
|
|
|
|
config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
|
|
|
|
|
config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src")
|
|
|
|
|
config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
|
|
|
|
|
config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
|
|
|
|
|
config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
|
|
|
|
|
config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
|
|
|
|
|
config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
|
|
|
|
|
config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')
|
|
|
|
|
|
|
|
|
|
parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
|
|
|
|
|
parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
|
|
|
|
|
parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src")
|
|
|
|
|
parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
|
|
|
|
|
parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
|
|
|
|
|
parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
|
|
|
|
|
parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
|
|
|
|
|
parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
|
|
|
|
|
parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# run config parser and serach config/*.json to add a build and version argument for it to the full parser
|
|
|
|
|
configDir = config_parser.parse_known_args()[0].config
|
|
|
|
|
for cf in glob.glob(configDir+"/*.json"):
|
|
|
|
|
with open(cf, 'r') as f:
|
|
|
|
|
libData = json.load(f)
|
|
|
|
|
parser.add_argument('--'+libData['name'],help='Enable build of '+libData['name'],action='store_true')
|
|
|
|
|
parser.add_argument('--'+libData['name']+'-version',help='Set '+libData['name']+' version ['+libData['default version']+']',default=str(libData['default version']))
|
|
|
|
|
|
|
|
|
|
# run full parser
|
|
|
|
|
argN = parser.parse_args()
|
|
|
|
|
args = vars(argN)
|
|
|
|
|
|
|
|
|
|
# extract known arguments
|
|
|
|
|
prefix = argN.prefix
|
|
|
|
|
srcDir = argN.src
|
|
|
|
|
workDir = argN.work
|
|
|
|
|
keepWork = argN.keep_work
|
|
|
|
|
compiler = argN.compiler
|
|
|
|
|
mpi = argN.mpi
|
|
|
|
|
buildThreads = argN.threads
|
|
|
|
|
verbose = argN.verbose
|
|
|
|
|
|
|
|
|
|
# extract libraries and versions selected for installation
|
|
|
|
|
selectedLibs=[]
|
|
|
|
|
ignoreNames=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"]
|
|
|
|
|
for libName in args:
|
|
|
|
|
if(libName not in ignoreNames and "version" not in libName):
|
|
|
|
|
install = getattr(argN,libName)
|
|
|
|
|
if (install == True):
|
|
|
|
|
version = getattr(argN,libName+"_version")
|
|
|
|
|
configFile = configDir+"/"+libName+".json"
|
|
|
|
|
with open(configFile,'r') as cf:
|
|
|
|
|
data = json.load(cf)
|
|
|
|
|
data['version'] = version
|
|
|
|
|
selectedLibs.append(data)
|
|
|
|
|
|
|
|
|
|
# Set up install directory
|
|
|
|
|
compilerVersion=getCompilerVersion(compiler)
|
|
|
|
|
mpiVersion=getMpiVersion(mpi)
|
|
|
|
|
instStr="inst-"+compiler+"_"+compilerVersion+"_"+mpi+"_"+mpiVersion
|
|
|
|
|
instDir=prefix+"/"+instStr
|
|
|
|
|
|
|
|
|
|
# setting MPI wrappers:
|
|
|
|
|
if (compiler == "gnu"):
|
|
|
|
|
if (mpi == "hpcx" or mpi == "openmpi"):
|
|
|
|
|
CC="mpicc"
|
|
|
|
|
CXX="mpic++"
|
|
|
|
|
FC="mpifort"
|
|
|
|
|
elif (mpi == "intel"):
|
|
|
|
|
CC="mpicc"
|
|
|
|
|
CXX="mpicxx"
|
|
|
|
|
FC="mpif90"
|
|
|
|
|
elif (compiler == "aocc"):
|
|
|
|
|
if (mpi == "hpcx" or mpi == "openmpi"):
|
|
|
|
|
CC="mpicc"
|
|
|
|
|
CXX="mpic++"
|
|
|
|
|
FC="mpifort"
|
|
|
|
|
os.environ["OMPI_MPICC"] = "clang"
|
|
|
|
|
os.environ["OMPI_MPICXX"] = "clang++"
|
|
|
|
|
os.environ["OMPI_MPIFC"] = "flang"
|
|
|
|
|
os.environ["OMPI_MPIF90"] = "flang"
|
|
|
|
|
os.environ["OMPI_MPIF77"] = "flang"
|
|
|
|
|
elif (mpi == "intel"):
|
|
|
|
|
CC="\"mpicc -cc=clang\""
|
|
|
|
|
CXX="\"mpicxx -cxx=clang++\""
|
|
|
|
|
FC="\"mpif90 -fc=flang\""
|
|
|
|
|
elif (compiler == "intel"):
|
|
|
|
|
if (mpi == "hpcx" or mpi == "openmpi"):
|
|
|
|
|
CC="mpicc"
|
|
|
|
|
CXX="mpic++"
|
|
|
|
|
FC="mpifort"
|
|
|
|
|
os.environ["OMPI_MPICC"] = "icc"
|
|
|
|
|
os.environ["OMPI_MPICXX"] = "icpc"
|
|
|
|
|
os.environ["OMPI_MPIFC"] = "ifort"
|
|
|
|
|
os.environ["OMPI_MPIF90"] = "ifort"
|
|
|
|
|
os.environ["OMPI_MPIF77"] = "ifort"
|
|
|
|
|
elif (mpi == "intel"):
|
|
|
|
|
CC="mpiicc"
|
|
|
|
|
CXX="mpiicpc"
|
|
|
|
|
FC="mpiifort"
|
|
|
|
|
|
|
|
|
|
os.environ["CC"]=CC
|
|
|
|
|
os.environ["CXX"]=CXX
|
|
|
|
|
os.environ["FC"]=FC
|
|
|
|
|
os.environ["F90"]=FC
|
|
|
|
|
os.environ["F77"]=FC
|
|
|
|
|
|
|
|
|
|
sortedLibs=sortLibsByDependencies(selectedLibs)
|
|
|
|
|
printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, sortedLibs)
|
|
|
|
|
|
|
|
|
|
for lib in sortedLibs:
|
|
|
|
|
installLib(lib,srcDir,workDir,instDir,CC,CXX,FC,buildThreads,verbose)
|
|
|
|
|
print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+instDir+" to your environment"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print("Cleaning up work directory...")
|
|
|
|
|
shutil.rmtree(workDir)
|
|
|
|
|
print("Done.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print("\n")
|
Patrick Lipka