v0.3 - split up script to modules, started to adapt style to PEP8
This commit is contained in:
parent
f776722842
commit
94925221c1
|
@ -21,7 +21,6 @@ dist/
|
||||||
downloads/
|
downloads/
|
||||||
eggs/
|
eggs/
|
||||||
.eggs/
|
.eggs/
|
||||||
lib/
|
|
||||||
lib64/
|
lib64/
|
||||||
parts/
|
parts/
|
||||||
sdist/
|
sdist/
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
import os
|
||||||
|
import argparse
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
|
||||||
|
def init():
|
||||||
|
# set up two parsers: first parse config directory, and add dynamic arguments to full parser
|
||||||
|
config_parser = argparse.ArgumentParser(add_help=False)
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
|
||||||
|
# parsers need to share known arguments
|
||||||
|
config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
|
||||||
|
config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
|
||||||
|
config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src")
|
||||||
|
config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
|
||||||
|
config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
|
||||||
|
config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
|
||||||
|
config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
|
||||||
|
config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
|
||||||
|
config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')
|
||||||
|
|
||||||
|
parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
|
||||||
|
parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
|
||||||
|
parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src")
|
||||||
|
parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work]',default=os.getcwd()+"/work")
|
||||||
|
parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
|
||||||
|
parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
|
||||||
|
parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
|
||||||
|
parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
|
||||||
|
parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')
|
||||||
|
|
||||||
|
# run config parser and serach config/*.json to add a build and version argument for it to the full parser
|
||||||
|
config_dir = config_parser.parse_known_args()[0].config
|
||||||
|
for cf in glob.glob(config_dir+"/*.json"):
|
||||||
|
with open(cf, 'r') as f:
|
||||||
|
lib_data = json.load(f)
|
||||||
|
parser.add_argument('--'+lib_data['name'],help='Enable build of '+lib_data['name'],action='store_true')
|
||||||
|
parser.add_argument('--'+lib_data['name']+'-version',help='Set '+lib_data['name']+' version ['+lib_data['default version']+']',default=str(lib_data['default version']))
|
||||||
|
|
||||||
|
# run full parser
|
||||||
|
arg_namespace = parser.parse_args()
|
||||||
|
args = vars(arg_namespace)
|
||||||
|
|
||||||
|
return arg_namespace,args
|
|
@ -0,0 +1,103 @@
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import glob
|
||||||
|
import shutil
|
||||||
|
from lib.ui import bordered, underlined, progressbar
|
||||||
|
|
||||||
|
def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_threads,verbose):
|
||||||
|
print(bordered("Installing "+lib['name']+" v."+lib['version']))
|
||||||
|
if not os.path.exists(src_dir):
|
||||||
|
os.makedirs(src_dir)
|
||||||
|
if not os.path.exists(work_dir):
|
||||||
|
os.makedirs(work_dir)
|
||||||
|
if not os.path.exists(inst_dir):
|
||||||
|
os.makedirs(inst_dir)
|
||||||
|
|
||||||
|
os.chdir(src_dir)
|
||||||
|
|
||||||
|
lib_name = lib['name']
|
||||||
|
lib_version = lib['version']
|
||||||
|
if verbose == False:
|
||||||
|
logfile_path = work_dir + "/" + lib_name + "-" + lib_version + ".log"
|
||||||
|
if os.path.exists(logfile_path):
|
||||||
|
os.remove(logfile_path)
|
||||||
|
logfile = open(logfile_path,'a')
|
||||||
|
else:
|
||||||
|
logfile = None
|
||||||
|
|
||||||
|
# download lib src if not present
|
||||||
|
if not os.path.exists(lib['name']+"-"+lib['version']):
|
||||||
|
|
||||||
|
print("Downloading Source Code")
|
||||||
|
load_command = lib['download'].replace("$VERSION_UNDERSCORE",lib_version.replace(".","_")).replace("$VERSION",lib_version).split()
|
||||||
|
err = subprocess.call(load_command,stdout=logfile,stderr=logfile)
|
||||||
|
if err != 0:
|
||||||
|
print("Error: Download of "+lib_name+" v."+lib_version+" failed!")
|
||||||
|
print("See "+logfile_path+" for details")
|
||||||
|
sys.exit(0)
|
||||||
|
# configure library
|
||||||
|
os.chdir(work_dir)
|
||||||
|
if os.path.exists(work_dir+"/"+lib_name+"-"+lib_version):
|
||||||
|
shutil.rmtree(work_dir+"/"+lib_name+"-"+lib_version)
|
||||||
|
shutil.copytree(src_dir+"/"+lib_name+"-"+lib_version, work_dir+"/"+lib_name+"-"+lib_version)
|
||||||
|
os.chdir(work_dir+"/"+lib_name+"-"+lib_version)
|
||||||
|
|
||||||
|
config_command = lib['configure'].replace("$PREFIX",inst_dir)
|
||||||
|
if verbose:
|
||||||
|
print(underlined("\nConfiguring Library"))
|
||||||
|
else:
|
||||||
|
print("Configuring library...")
|
||||||
|
err = subprocess.call(config_command,stdout=logfile,stderr=logfile,shell=True)
|
||||||
|
if err != 0:
|
||||||
|
print("Error: Configuring of "+lib_name+" v."+lib_version+" failed!")
|
||||||
|
print("See "+logfile_path+" for details")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# build library
|
||||||
|
build_command = lib['build'].replace("$BUILDTHREADS",build_threads)
|
||||||
|
if verbose:
|
||||||
|
print(underlined("\nBuilding Library"))
|
||||||
|
err = subprocess.call(build_command,stdout=logfile,stderr=logfile,shell=True)
|
||||||
|
else:
|
||||||
|
build_task = subprocess.Popen(build_command,stdout=logfile,stderr=logfile,shell=True)
|
||||||
|
num_objects = int(lib['object files'])
|
||||||
|
finished = False
|
||||||
|
while build_task.poll() == None:
|
||||||
|
if not finished:
|
||||||
|
# bar_width = num_objects
|
||||||
|
# while (bar_width > 100):
|
||||||
|
# bar_width = int(bar_width/10)
|
||||||
|
bar_width = 50
|
||||||
|
for i in progressbar(range(num_objects), "Building library: ",bar_width):
|
||||||
|
while len(glob.glob('**/*.o',recursive=True)) < i:
|
||||||
|
time.sleep(0.5)
|
||||||
|
if(build_task.poll != None):
|
||||||
|
break
|
||||||
|
finished = True
|
||||||
|
build_task.wait()
|
||||||
|
err = build_task.returncode
|
||||||
|
|
||||||
|
if err != 0:
|
||||||
|
print("Error: Building "+lib_name+" v."+lib_version+" failed!")
|
||||||
|
print("See "+logfile_path+" for details")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
#install library
|
||||||
|
install_command = lib['install'].replace("$BUILDTHREADS",build_threads)
|
||||||
|
if verbose:
|
||||||
|
print(underlined("\nInstalling Library"))
|
||||||
|
else:
|
||||||
|
print("Installing library...")
|
||||||
|
err = subprocess.call(install_command,stdout=logfile,stderr=logfile,shell=True)
|
||||||
|
if err != 0:
|
||||||
|
print("Error: Installing "+lib_name+" v."+lib_version+" failed!")
|
||||||
|
print("See "+logfile_path+" for details")
|
||||||
|
sys.exit(1)
|
||||||
|
print("Library "+lib_name+" has been installed successfully!\n")
|
||||||
|
build_output_path = inst_dir+"/"+"build_info"
|
||||||
|
if not os.path.exists(build_output_path):
|
||||||
|
os.makedirs(build_output_path)
|
||||||
|
shutil.copyfile(logfile_path,build_output_path+"/"+lib_name+".log")
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def get_from_command(command):
|
||||||
|
ret = subprocess.check_output(command).decode(sys.stdout.encoding)
|
||||||
|
return ret
|
|
@ -0,0 +1,36 @@
|
||||||
|
def topological_sort(source):
|
||||||
|
pending = [(name, set(deps)) for name, deps in source]
|
||||||
|
emitted = []
|
||||||
|
while pending:
|
||||||
|
next_pending = []
|
||||||
|
next_emitted = []
|
||||||
|
for entry in pending:
|
||||||
|
name, deps = entry
|
||||||
|
deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6
|
||||||
|
if deps:
|
||||||
|
next_pending.append(entry)
|
||||||
|
else:
|
||||||
|
yield name
|
||||||
|
emitted.append(name) # <-- not required, but preserves original order
|
||||||
|
next_emitted.append(name)
|
||||||
|
if not next_emitted:
|
||||||
|
raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
|
||||||
|
pending = next_pending
|
||||||
|
emitted = next_emitted
|
||||||
|
return emitted
|
||||||
|
|
||||||
|
def sort_libs_by_dependencies(selected_libs):
|
||||||
|
deplist=[]
|
||||||
|
for lib in selected_libs:
|
||||||
|
name = lib['name']
|
||||||
|
dependencies = set(lib['dependencies'].split(','))
|
||||||
|
if dependencies == {''}:
|
||||||
|
dependencies = {}
|
||||||
|
deplist.append([name,dependencies])
|
||||||
|
sorted_deplist = topological_sort(deplist)
|
||||||
|
sorted = []
|
||||||
|
for entry in sorted_deplist:
|
||||||
|
for lib in selected_libs:
|
||||||
|
if lib['name'] == entry:
|
||||||
|
sorted.append(lib)
|
||||||
|
return sorted
|
|
@ -0,0 +1,79 @@
|
||||||
|
import os
|
||||||
|
from lib.shell import get_from_command
|
||||||
|
|
||||||
|
def get_compiler_version(compiler):
|
||||||
|
if compiler == "gnu":
|
||||||
|
cc="gcc"
|
||||||
|
elif compiler == "intel":
|
||||||
|
cc="icc"
|
||||||
|
elif compiler == "aocc":
|
||||||
|
cc="clang"
|
||||||
|
compstr = get_from_command([cc,"-dumpversion"]).strip()
|
||||||
|
return compstr
|
||||||
|
|
||||||
|
def get_mpi_version(mpi):
|
||||||
|
if mpi == "hpcx":
|
||||||
|
hpcx_dir = os.getenv('HPCX_DIR')
|
||||||
|
ver_file = hpcx_dir + "/" + "VERSION"
|
||||||
|
with open(ver_file) as f:
|
||||||
|
lines = f.readlines()
|
||||||
|
mpistr = lines[0].split()[1].strip()
|
||||||
|
elif mpi == "openmpi":
|
||||||
|
rawstr = get_from_command(["ompi_info"]).splitlines()[1]
|
||||||
|
mpistr = rawstr[rawstr.find(":")+2:]
|
||||||
|
elif mpi == "intelmpi":
|
||||||
|
rawstr = get_from_command(["ompi_info"]).splitlines()[0]
|
||||||
|
mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1]
|
||||||
|
elif mpi == "mpich":
|
||||||
|
rawstr = get_from_command(["mpirun","--version"])
|
||||||
|
mpistr = rawstr.split()[4]
|
||||||
|
return mpistr
|
||||||
|
|
||||||
|
|
||||||
|
def set_toolchain(compiler,mpi):
|
||||||
|
if compiler == "gnu":
|
||||||
|
if mpi == "hpcx" or mpi == "openmpi" or mpi == "mpich":
|
||||||
|
cc = "mpicc"
|
||||||
|
cxx = "mpic++"
|
||||||
|
fc = "mpifort"
|
||||||
|
elif mpi == "intel":
|
||||||
|
cc = "mpicc"
|
||||||
|
cxx = "mpicxx"
|
||||||
|
fc = "mpif90"
|
||||||
|
elif compiler == "aocc":
|
||||||
|
if mpi == "hpcx" or mpi == "openmpi":
|
||||||
|
cc = "mpicc"
|
||||||
|
cxx = "mpic++"
|
||||||
|
fc = "mpifort"
|
||||||
|
os.environ["OMPI_MPICC"] = "clang"
|
||||||
|
os.environ["OMPI_MPICXX"] = "clang++"
|
||||||
|
os.environ["OMPI_MPIFC"] = "flang"
|
||||||
|
os.environ["OMPI_MPIF90"] = "flang"
|
||||||
|
os.environ["OMPI_MPIF77"] = "flang"
|
||||||
|
elif (mpi == "intel"):
|
||||||
|
cc = "\"mpicc -cc=clang\""
|
||||||
|
cxx = "\"mpicxx -cxx=clang++\""
|
||||||
|
fc = "\"mpif90 -fc=flang\""
|
||||||
|
elif compiler == "intel":
|
||||||
|
if mpi == "hpcx" or mpi == "openmpi":
|
||||||
|
cc = "mpicc"
|
||||||
|
cxx = "mpic++"
|
||||||
|
fc = "mpifort"
|
||||||
|
os.environ["OMPI_MPICC"] = "icc"
|
||||||
|
os.environ["OMPI_MPICXX"] = "icpc"
|
||||||
|
os.environ["OMPI_MPIFC"] = "ifort"
|
||||||
|
os.environ["OMPI_MPIF90"] = "ifort"
|
||||||
|
os.environ["OMPI_MPIF77"] = "ifort"
|
||||||
|
elif mpi == "intel":
|
||||||
|
cc = "mpiicc"
|
||||||
|
cxx = "mpiicpc"
|
||||||
|
fc = "mpiifort"
|
||||||
|
|
||||||
|
# set environment variables
|
||||||
|
os.environ["CC"] = cc
|
||||||
|
os.environ["CXX"] = cxx
|
||||||
|
os.environ["FC"] = fc
|
||||||
|
os.environ["F90"] = fc
|
||||||
|
os.environ["F77"] = fc
|
||||||
|
|
||||||
|
return cc,cxx,fc
|
|
@ -0,0 +1,49 @@
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
def progressbar(it, prefix="", size=60, out=sys.stdout):
|
||||||
|
count = len(it)
|
||||||
|
def show(j):
|
||||||
|
x = int(size*j/count)
|
||||||
|
print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count),
|
||||||
|
end='\r', file=out, flush=True)
|
||||||
|
show(0)
|
||||||
|
for i, item in enumerate(it):
|
||||||
|
yield item
|
||||||
|
show(i+1)
|
||||||
|
print("", flush=True, file=out)
|
||||||
|
|
||||||
|
def bordered(text):
|
||||||
|
lines = text.splitlines()
|
||||||
|
width = max(len(s) for s in lines)
|
||||||
|
res = ['┌' + '─' * width + '┐']
|
||||||
|
for s in lines:
|
||||||
|
res.append('│' + (s + ' ' * width)[:width] + '│')
|
||||||
|
res.append('└' + '─' * width + '┘')
|
||||||
|
return '\n'.join(res)
|
||||||
|
|
||||||
|
def underlined(text):
|
||||||
|
lines = text.splitlines()
|
||||||
|
width = max(len(s) for s in lines)
|
||||||
|
res=[]
|
||||||
|
for s in lines:
|
||||||
|
res.append(s)
|
||||||
|
res.append('─' * width)
|
||||||
|
return '\n'.join(res)
|
||||||
|
|
||||||
|
def print_welcome(script_version, compiler, compiler_version, mpi, mpi_version, instDir, installs):
|
||||||
|
out = underlined("Patricks Simple Library Installer " + script_version)
|
||||||
|
out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compiler_version
|
||||||
|
out = out + "\n" + "MPI: " + mpi.upper() + ", Version: " + mpi_version
|
||||||
|
out = out + "\n" + "Install Directory: "+instDir
|
||||||
|
out = out + "\n"
|
||||||
|
out = out + "\n" + "The following Libraries will be installed:"
|
||||||
|
for lib in installs:
|
||||||
|
out = out + "\n" + lib['name'] + " v." + lib['version']
|
||||||
|
print(bordered(out) + "\n")
|
||||||
|
|
||||||
|
# short sleep to enable user to abprt if selected options are wrong
|
||||||
|
for i in progressbar(range(5), "Waiting 5s to start build: ",10):
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
print("\n")
|
392
libinstaller.py
392
libinstaller.py
|
@ -1,357 +1,71 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import argparse
|
|
||||||
import glob
|
|
||||||
import json
|
|
||||||
import time
|
|
||||||
import shutil
|
import shutil
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from lib.ui import progressbar, bordered, underlined, print_welcome
|
||||||
|
from lib.shell import get_from_command
|
||||||
|
from lib.toolchain import get_compiler_version, get_mpi_version, set_toolchain
|
||||||
|
from lib.sort import sort_libs_by_dependencies
|
||||||
|
from lib.installer import install_lib
|
||||||
|
from lib.init import init
|
||||||
|
|
||||||
SCRIPT_VERSION="v0.2"
|
SCRIPT_VERSION="v0.3"
|
||||||
|
|
||||||
def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+
|
# initialization by argparser
|
||||||
count = len(it)
|
arg_namespace,args = init()
|
||||||
def show(j):
|
|
||||||
x = int(size*j/count)
|
|
||||||
print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count),
|
|
||||||
end='\r', file=out, flush=True)
|
|
||||||
show(0)
|
|
||||||
for i, item in enumerate(it):
|
|
||||||
yield item
|
|
||||||
show(i+1)
|
|
||||||
print("", flush=True, file=out)
|
|
||||||
|
|
||||||
def getFromCommand(command):
|
# extract known arguments from argparse namespace
|
||||||
ret=subprocess.check_output(command).decode(sys.stdout.encoding)
|
prefix = arg_namespace.prefix
|
||||||
return ret
|
config_dir = arg_namespace.config
|
||||||
|
src_dir = arg_namespace.src
|
||||||
def getCompilerVersion(compiler):
|
work_dir = arg_namespace.work
|
||||||
if (compiler == "gnu"):
|
keep_work = arg_namespace.keep_work
|
||||||
cc="gcc"
|
compiler = arg_namespace.compiler
|
||||||
elif (compiler == "intel"):
|
mpi = arg_namespace.mpi
|
||||||
cc="icc"
|
build_threads = arg_namespace.threads
|
||||||
elif (compiler == "aocc"):
|
verbose = arg_namespace.verbose
|
||||||
cc="clang"
|
|
||||||
compstr=getFromCommand([cc,"-dumpversion"]).strip()
|
|
||||||
return compstr
|
|
||||||
|
|
||||||
def getMpiVersion(mpi):
|
|
||||||
if (mpi == "hpcx"):
|
|
||||||
hpcxDir = os.getenv('HPCX_DIR')
|
|
||||||
verFile = hpcxDir + "/" + "VERSION"
|
|
||||||
with open(verFile) as f:
|
|
||||||
lines = f.readlines()
|
|
||||||
mpistr = lines[0].split()[1].strip()
|
|
||||||
elif (mpi == "openmpi"):
|
|
||||||
rawstr = getFromCommand(["ompi_info"]).splitlines()[1]
|
|
||||||
mpistr = rawstr[rawstr.find(":")+2:]
|
|
||||||
elif (mpi == "intelmpi"):
|
|
||||||
rawstr = getFromCommand(["ompi_info"]).splitlines()[0]
|
|
||||||
mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1]
|
|
||||||
elif (mpi == "mpich"):
|
|
||||||
rawstr = getFromCommand(["mpirun","--version"])
|
|
||||||
mpistr = rawstr.split()[4]
|
|
||||||
return mpistr
|
|
||||||
|
|
||||||
def bordered(text):
|
|
||||||
lines = text.splitlines()
|
|
||||||
width = max(len(s) for s in lines)
|
|
||||||
res = ['┌' + '─' * width + '┐']
|
|
||||||
for s in lines:
|
|
||||||
res.append('│' + (s + ' ' * width)[:width] + '│')
|
|
||||||
res.append('└' + '─' * width + '┘')
|
|
||||||
return '\n'.join(res)
|
|
||||||
|
|
||||||
def underlined(text):
|
|
||||||
lines = text.splitlines()
|
|
||||||
width = max(len(s) for s in lines)
|
|
||||||
res=[]
|
|
||||||
for s in lines:
|
|
||||||
res.append(s)
|
|
||||||
res.append('─' * width)
|
|
||||||
return '\n'.join(res)
|
|
||||||
|
|
||||||
def printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, installs):
|
|
||||||
out = underlined("Patricks Simple Library Installer "+SCRIPT_VERSION)
|
|
||||||
out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compilerVersion
|
|
||||||
out = out + "\n" + "MPI: " + mpi.upper() + ", Version: " + mpiVersion
|
|
||||||
out = out + "\n" + "Install Directory: "+instDir
|
|
||||||
out = out + "\n"
|
|
||||||
out = out + "\n" + "The following Libraries will be installed:"
|
|
||||||
for lib in installs:
|
|
||||||
out = out + "\n" + lib['name'] + " v." + lib['version']
|
|
||||||
print(bordered(out) + "\n")
|
|
||||||
|
|
||||||
# short sleep to enable user to abprt if selected options are wrong
|
|
||||||
for i in progressbar(range(5), "Waiting 5s to start build: ",10):
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
def topological_sort(source):
|
|
||||||
pending = [(name, set(deps)) for name, deps in source]
|
|
||||||
emitted = []
|
|
||||||
while pending:
|
|
||||||
next_pending = []
|
|
||||||
next_emitted = []
|
|
||||||
for entry in pending:
|
|
||||||
name, deps = entry
|
|
||||||
deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6
|
|
||||||
if deps:
|
|
||||||
next_pending.append(entry)
|
|
||||||
else:
|
|
||||||
yield name
|
|
||||||
emitted.append(name) # <-- not required, but preserves original order
|
|
||||||
next_emitted.append(name)
|
|
||||||
if not next_emitted:
|
|
||||||
raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
|
|
||||||
pending = next_pending
|
|
||||||
emitted = next_emitted
|
|
||||||
return emitted
|
|
||||||
|
|
||||||
def sortLibsByDependencies(selectedLibs):
|
|
||||||
depList=[]
|
|
||||||
for lib in selectedLibs:
|
|
||||||
name = lib['name']
|
|
||||||
dependencies = set(lib['dependencies'].split(','))
|
|
||||||
if dependencies == {''}:
|
|
||||||
dependencies = {}
|
|
||||||
depList.append([name,dependencies])
|
|
||||||
sortedDepList = topological_sort(depList)
|
|
||||||
sorted = []
|
|
||||||
for entry in sortedDepList:
|
|
||||||
for lib in selectedLibs:
|
|
||||||
if lib['name'] == entry:
|
|
||||||
sorted.append(lib)
|
|
||||||
return sorted
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def installLib(lib,srcDir,workDir,instDir,compCC,compCXX,compFC,buildThreads,verbose):
|
|
||||||
print(bordered("Installing "+lib['name']+" v."+lib['version']))
|
|
||||||
if (not os.path.exists(srcDir)):
|
|
||||||
os.makedirs(srcDir)
|
|
||||||
if (not os.path.exists(workDir)):
|
|
||||||
os.makedirs(workDir)
|
|
||||||
if (not os.path.exists(instDir)):
|
|
||||||
os.makedirs(instDir)
|
|
||||||
|
|
||||||
os.chdir(srcDir)
|
|
||||||
|
|
||||||
libName = lib['name']
|
|
||||||
libVersion = lib['version']
|
|
||||||
if (verbose == False):
|
|
||||||
logfilePath = workDir + "/" + libName + "-" + libVersion + ".log"
|
|
||||||
if (os.path.exists(logfilePath)):
|
|
||||||
os.remove(logfilePath)
|
|
||||||
logfile = open(logfilePath,'a')
|
|
||||||
else:
|
|
||||||
logfile = None
|
|
||||||
|
|
||||||
# download lib src if not present
|
|
||||||
if (not os.path.exists(lib['name']+"-"+lib['version'])):
|
|
||||||
|
|
||||||
print("Downloading Source Code")
|
|
||||||
loadCommand = lib['download'].replace("$VERSION_UNDERSCORE",libVersion.replace(".","_")).replace("$VERSION",libVersion).split()
|
|
||||||
err = subprocess.call(loadCommand,stdout=logfile,stderr=logfile)
|
|
||||||
if (err != 0):
|
|
||||||
print("Error: Download of "+libName+" v."+libVersion+" failed!")
|
|
||||||
print("See "+logfilePath+" for details")
|
|
||||||
sys.exit(0)
|
|
||||||
# configure library
|
|
||||||
os.chdir(workDir)
|
|
||||||
if (os.path.exists(workDir+"/"+libName+"-"+libVersion)):
|
|
||||||
shutil.rmtree(workDir+"/"+libName+"-"+libVersion)
|
|
||||||
shutil.copytree(srcDir+"/"+libName+"-"+libVersion, workDir+"/"+libName+"-"+libVersion)
|
|
||||||
os.chdir(workDir+"/"+libName+"-"+libVersion)
|
|
||||||
|
|
||||||
#configCommand = lib['configure'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split()
|
|
||||||
configCommand = lib['configure'].replace("$PREFIX",instDir)
|
|
||||||
if (verbose):
|
|
||||||
print(underlined("\nConfiguring Library"))
|
|
||||||
else:
|
|
||||||
print("Configuring library...")
|
|
||||||
err = subprocess.call(configCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
||||||
if (err != 0):
|
|
||||||
print("Error: Configuring of "+libName+" v."+libVersion+" failed!")
|
|
||||||
print("See "+logfilePath+" for details")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# build library
|
|
||||||
#buildCommand = lib['build'].replace("$CC",compCC).replace("$CXX",compCXX).replace("$FC",compFC).replace("$INSTALL",instDir).replace("$BUILDTHREADS",buildThreads).split()
|
|
||||||
buildCommand = lib['build'].replace("$BUILDTHREADS",buildThreads)
|
|
||||||
if(verbose):
|
|
||||||
print(underlined("\nBuilding Library"))
|
|
||||||
err = subprocess.call(buildCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
||||||
else:
|
|
||||||
buildTask = subprocess.Popen(buildCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
||||||
numObjects = int(lib['object files'])
|
|
||||||
finished = False
|
|
||||||
while (buildTask.poll() == None):
|
|
||||||
if (not finished):
|
|
||||||
barWidth = numObjects
|
|
||||||
#while (barWidth > 100):
|
|
||||||
# barWidth = int(barWidth/10)
|
|
||||||
barWidth = 50
|
|
||||||
for i in progressbar(range(numObjects), "Building library: ",barWidth):
|
|
||||||
while(len(glob.glob('**/*.o',recursive=True)) < i):
|
|
||||||
time.sleep(0.5)
|
|
||||||
if(buildTask.poll != None):
|
|
||||||
break
|
|
||||||
finished = True
|
|
||||||
buildTask.wait()
|
|
||||||
err=buildTask.returncode
|
|
||||||
|
|
||||||
if (err != 0):
|
|
||||||
print("Error: Building "+libName+" v."+libVersion+" failed!")
|
|
||||||
print("See "+logfilePath+" for details")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
#install library
|
|
||||||
installCommand = lib['install'].replace("$BUILDTHREADS",buildThreads)
|
|
||||||
if(verbose):
|
|
||||||
print(underlined("\nInstalling Library"))
|
|
||||||
else:
|
|
||||||
print("Installing library...")
|
|
||||||
err = subprocess.call(installCommand,stdout=logfile,stderr=logfile,shell=True)
|
|
||||||
if (err != 0):
|
|
||||||
print("Error: Installing "+libName+" v."+libVersion+" failed!")
|
|
||||||
print("See "+logfilePath+" for details")
|
|
||||||
sys.exit(1)
|
|
||||||
print("Library "+libName+" has been installed successfully!\n")
|
|
||||||
buildOutputPath = instDir+"/"+"build_info"
|
|
||||||
if (not os.path.exists(buildOutputPath)):
|
|
||||||
os.makedirs(buildOutputPath)
|
|
||||||
shutil.copyfile(logfilePath,buildOutputPath+"/"+libName+".log")
|
|
||||||
|
|
||||||
|
|
||||||
#MAIN
|
|
||||||
|
|
||||||
# set up two parsers: first parse config directory, and add dynamic arguments to full parser
|
|
||||||
config_parser = argparse.ArgumentParser(add_help=False)
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
|
|
||||||
# parsers need to share known arguments
|
|
||||||
config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
|
|
||||||
config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
|
|
||||||
config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src")
|
|
||||||
config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
|
|
||||||
config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
|
|
||||||
config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
|
|
||||||
config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
|
|
||||||
config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
|
|
||||||
config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')
|
|
||||||
|
|
||||||
parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config")
|
|
||||||
parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd())
|
|
||||||
parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src")
|
|
||||||
parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work")
|
|
||||||
parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true')
|
|
||||||
parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu')
|
|
||||||
parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx')
|
|
||||||
parser.add_argument('--threads',help='Number of threads used for make [8]',default='8')
|
|
||||||
parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true')
|
|
||||||
|
|
||||||
|
|
||||||
# run config parser and serach config/*.json to add a build and version argument for it to the full parser
|
|
||||||
configDir = config_parser.parse_known_args()[0].config
|
|
||||||
for cf in glob.glob(configDir+"/*.json"):
|
|
||||||
with open(cf, 'r') as f:
|
|
||||||
libData = json.load(f)
|
|
||||||
parser.add_argument('--'+libData['name'],help='Enable build of '+libData['name'],action='store_true')
|
|
||||||
parser.add_argument('--'+libData['name']+'-version',help='Set '+libData['name']+' version ['+libData['default version']+']',default=str(libData['default version']))
|
|
||||||
|
|
||||||
# run full parser
|
|
||||||
argN = parser.parse_args()
|
|
||||||
args = vars(argN)
|
|
||||||
|
|
||||||
# extract known arguments
|
|
||||||
prefix = argN.prefix
|
|
||||||
srcDir = argN.src
|
|
||||||
workDir = argN.work
|
|
||||||
keepWork = argN.keep_work
|
|
||||||
compiler = argN.compiler
|
|
||||||
mpi = argN.mpi
|
|
||||||
buildThreads = argN.threads
|
|
||||||
verbose = argN.verbose
|
|
||||||
|
|
||||||
# extract libraries and versions selected for installation
|
# extract libraries and versions selected for installation
|
||||||
selectedLibs=[]
|
selected_libs=[]
|
||||||
ignoreNames=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"]
|
ignore_names=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"]
|
||||||
for libName in args:
|
for lib_name in args:
|
||||||
if(libName not in ignoreNames and "version" not in libName):
|
if (lib_name not in ignore_names and "version" not in lib_name):
|
||||||
install = getattr(argN,libName)
|
install = getattr(arg_namespace,lib_name)
|
||||||
if (install == True):
|
if (install == True):
|
||||||
version = getattr(argN,libName+"_version")
|
version = getattr(arg_namespace,lib_name+"_version")
|
||||||
configFile = configDir+"/"+libName+".json"
|
config_file = config_dir+"/"+lib_name+".json"
|
||||||
with open(configFile,'r') as cf:
|
with open(config_file,'r') as cf:
|
||||||
data = json.load(cf)
|
data = json.load(cf)
|
||||||
data['version'] = version
|
data['version'] = version
|
||||||
selectedLibs.append(data)
|
selected_libs.append(data)
|
||||||
|
|
||||||
# Set up install directory
|
# set up install directory name
|
||||||
compilerVersion=getCompilerVersion(compiler)
|
compiler_version=get_compiler_version(compiler)
|
||||||
mpiVersion=getMpiVersion(mpi)
|
mpi_version=get_mpi_version(mpi)
|
||||||
instStr="inst-"+compiler+"_"+compilerVersion+"_"+mpi+"_"+mpiVersion
|
inst_str="inst-"+compiler+"_"+compiler_version+"_"+mpi+"_"+mpi_version
|
||||||
instDir=prefix+"/"+instStr
|
inst_dir=prefix+"/"+inst_str
|
||||||
|
|
||||||
# setting MPI wrappers:
|
# set up mpi wrappers and environment variables
|
||||||
if (compiler == "gnu"):
|
cc,cxx,fc = set_toolchain(compiler,mpi)
|
||||||
if (mpi == "hpcx" or mpi == "openmpi" or mpi == "mpich"):
|
|
||||||
CC="mpicc"
|
|
||||||
CXX="mpic++"
|
|
||||||
FC="mpifort"
|
|
||||||
elif (mpi == "intel"):
|
|
||||||
CC="mpicc"
|
|
||||||
CXX="mpicxx"
|
|
||||||
FC="mpif90"
|
|
||||||
elif (compiler == "aocc"):
|
|
||||||
if (mpi == "hpcx" or mpi == "openmpi"):
|
|
||||||
CC="mpicc"
|
|
||||||
CXX="mpic++"
|
|
||||||
FC="mpifort"
|
|
||||||
os.environ["OMPI_MPICC"] = "clang"
|
|
||||||
os.environ["OMPI_MPICXX"] = "clang++"
|
|
||||||
os.environ["OMPI_MPIFC"] = "flang"
|
|
||||||
os.environ["OMPI_MPIF90"] = "flang"
|
|
||||||
os.environ["OMPI_MPIF77"] = "flang"
|
|
||||||
elif (mpi == "intel"):
|
|
||||||
CC="\"mpicc -cc=clang\""
|
|
||||||
CXX="\"mpicxx -cxx=clang++\""
|
|
||||||
FC="\"mpif90 -fc=flang\""
|
|
||||||
elif (compiler == "intel"):
|
|
||||||
if (mpi == "hpcx" or mpi == "openmpi"):
|
|
||||||
CC="mpicc"
|
|
||||||
CXX="mpic++"
|
|
||||||
FC="mpifort"
|
|
||||||
os.environ["OMPI_MPICC"] = "icc"
|
|
||||||
os.environ["OMPI_MPICXX"] = "icpc"
|
|
||||||
os.environ["OMPI_MPIFC"] = "ifort"
|
|
||||||
os.environ["OMPI_MPIF90"] = "ifort"
|
|
||||||
os.environ["OMPI_MPIF77"] = "ifort"
|
|
||||||
elif (mpi == "intel"):
|
|
||||||
CC="mpiicc"
|
|
||||||
CXX="mpiicpc"
|
|
||||||
FC="mpiifort"
|
|
||||||
|
|
||||||
os.environ["CC"]=CC
|
# set up library install order by dependencies
|
||||||
os.environ["CXX"]=CXX
|
sorted_libs = sort_libs_by_dependencies(selected_libs)
|
||||||
os.environ["FC"]=FC
|
|
||||||
os.environ["F90"]=FC
|
|
||||||
os.environ["F77"]=FC
|
|
||||||
|
|
||||||
sortedLibs=sortLibsByDependencies(selectedLibs)
|
# print welcome screen
|
||||||
printWelcome(compiler, compilerVersion, mpi, mpiVersion, instDir, sortedLibs)
|
print_welcome(SCRIPT_VERSION, compiler, compiler_version, mpi, mpi_version, inst_dir, sorted_libs)
|
||||||
|
|
||||||
for lib in sortedLibs:
|
# install selected libraries
|
||||||
installLib(lib,srcDir,workDir,instDir,CC,CXX,FC,buildThreads,verbose)
|
if len(sorted_libs) > 0:
|
||||||
print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+instDir+" to your environment"))
|
for lib in sorted_libs:
|
||||||
|
install_lib(lib,src_dir,work_dir,inst_dir,cc,cxx,fc,build_threads,verbose)
|
||||||
|
print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+inst_dir+" to your environment"))
|
||||||
print("Cleaning up work directory...")
|
else:
|
||||||
shutil.rmtree(workDir)
|
print("NO LIBRARIES SELECTED FOR INSTALLATION")
|
||||||
print("Done.")
|
|
||||||
|
|
||||||
|
# cleanup
|
||||||
|
if (not keep_work and os.path.exists(work_dir)):
|
||||||
|
print("Cleaning up work directory...")
|
||||||
|
shutil.rmtree(work_dir)
|
||||||
|
print("Done.")
|
||||||
|
|
||||||
print("\n")
|
print("\n")
|
Loading…
Reference in New Issue