patrick
/
libinstaller
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Improved codestyle conformancy to PEP8

This commit is contained in:
Patrick Lipka 2022-09-22 01:51:47 +02:00
parent 94925221c1
commit a6e549a2da
7 changed files with 92 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
lib/init.py
View File

@ -3,42 +3,43 @@ import argparse
import glob import glob
import json import json
def init(): def init():
# set up two parsers: first parse config directory, and add dynamic arguments to full parser # set up two parsers: first parse config directory, and add dynamic arguments to full parser
config_parser = argparse.ArgumentParser(add_help=False) config_parser = argparse.ArgumentParser(add_help=False)
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# parsers need to share known arguments # parsers need to share known arguments
config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") config_parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config")
config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) config_parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd())
config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") config_parser.add_argument('--src', help='Path where to download source code to [$pwd/src]', default=os.getcwd()+"/src")
config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work") config_parser.add_argument('--work', help='Path to working directory for builds [$pwd/work', default=os.getcwd()+"/work")
config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') config_parser.add_argument('--keep-work', help='Disable removal of work directory after successful builds', action='store_true')
config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') config_parser.add_argument('--compiler', help='Select compiler (gnu, intel, aocc) [gnu]', default='gnu')
config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') config_parser.add_argument('--mpi', help='Select compiler (hpcx, intelmpi, openmpi) [hpcx]', default='hpcx')
config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') config_parser.add_argument('--threads', help='Number of threads used for make [8]', default='8')
config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') config_parser.add_argument('--verbose', help='Print build output to screen instead piping it to logile', action='store_true')
parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config")
parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd())
parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") parser.add_argument('--src', help='Path where to download source code to [$pwd/src]', default=os.getcwd()+"/src")
parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work]',default=os.getcwd()+"/work") parser.add_argument('--work', help='Path to working directory for builds [$pwd/work]', default=os.getcwd()+"/work")
parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') parser.add_argument('--keep-work', help='Disable removal of work directory after successful builds', action='store_true')
parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') parser.add_argument('--compiler', help='Select compiler (gnu, intel, aocc) [gnu]', default='gnu')
parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') parser.add_argument('--mpi', help='Select compiler (hpcx, intelmpi, openmpi) [hpcx]', default='hpcx')
parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') parser.add_argument('--threads', help='Number of threads used for make [8]', default='8')
parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') parser.add_argument('--verbose', help='Print build output to screen instead piping it to logile', action='store_true')
# run config parser and serach config/*.json to add a build and version argument for it to the full parser # run config parser and search config/*.json to add a build and version argument for it to the full parser
config_dir = config_parser.parse_known_args()[0].config config_dir = config_parser.parse_known_args()[0].config
for cf in glob.glob(config_dir+"/*.json"): for cf in glob.glob(config_dir+"/*.json"):
with open(cf, 'r') as f: with open(cf, 'r') as f:
lib_data = json.load(f) lib_data = json.load(f)
parser.add_argument('--'+lib_data['name'],help='Enable build of '+lib_data['name'],action='store_true') parser.add_argument('--'+lib_data['name'], help='Enable build of '+lib_data['name'], action='store_true')
parser.add_argument('--'+lib_data['name']+'-version',help='Set '+lib_data['name']+' version ['+lib_data['default version']+']',default=str(lib_data['default version'])) parser.add_argument('--'+lib_data['name']+'-version', help='Set '+lib_data['name']+' version ['+lib_data['default version']+']', default=str(lib_data['default version']))
# run full parser # run full parser
arg_namespace = parser.parse_args() arg_namespace = parser.parse_args()
args = vars(arg_namespace) args = vars(arg_namespace)
return arg_namespace,args return arg_namespace, args

45
lib/installer.py
View File

@ -4,10 +4,11 @@ import subprocess
import time import time
import glob import glob
import shutil import shutil
from lib.ui import bordered, underlined, progressbar from lib.ui import bordered, underlined, progressbar
def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_threads,verbose):
print(bordered("Installing "+lib['name']+" v."+lib['version'])) def install_lib(lib, src_dir, work_dir, inst_dir, comp_cc, comp_cxx, comp_fc, build_threads, verbose):
print(bordered("Installing " + lib['name'] + " v." + lib['version']))
if not os.path.exists(src_dir): if not os.path.exists(src_dir):
os.makedirs(src_dir) os.makedirs(src_dir)
if not os.path.exists(work_dir): if not os.path.exists(work_dir):
@ -19,11 +20,11 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr
lib_name = lib['name'] lib_name = lib['name']
lib_version = lib['version'] lib_version = lib['version']
if verbose == False: if not verbose:
logfile_path = work_dir + "/" + lib_name + "-" + lib_version + ".log" logfile_path = work_dir + "/" + lib_name + "-" + lib_version + ".log"
if os.path.exists(logfile_path): if os.path.exists(logfile_path):
os.remove(logfile_path) os.remove(logfile_path)
logfile = open(logfile_path,'a') logfile = open(logfile_path, 'a')
else: else:
logfile = None logfile = None
@ -31,49 +32,50 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr
if not os.path.exists(lib['name']+"-"+lib['version']): if not os.path.exists(lib['name']+"-"+lib['version']):
print("Downloading Source Code") print("Downloading Source Code")
load_command = lib['download'].replace("$VERSION_UNDERSCORE",lib_version.replace(".","_")).replace("$VERSION",lib_version).split() load_command = lib['download'].replace("$VERSION_UNDERSCORE", lib_version.replace(".", "_")).replace("$VERSION", lib_version).split()
err = subprocess.call(load_command,stdout=logfile,stderr=logfile) err = subprocess.call(load_command, stdout=logfile, stderr=logfile)
if err != 0: if err != 0:
print("Error: Download of "+lib_name+" v."+lib_version+" failed!") print("Error: Download of "+lib_name+" v."+lib_version+" failed!")
print("See "+logfile_path+" for details") print("See "+logfile_path+" for details")
sys.exit(0) sys.exit(0)
# configure library # configure library
os.chdir(work_dir) os.chdir(work_dir)
if os.path.exists(work_dir+"/"+lib_name+"-"+lib_version): if os.path.exists(work_dir+"/"+lib_name+"-"+lib_version):
shutil.rmtree(work_dir+"/"+lib_name+"-"+lib_version) shutil.rmtree(work_dir+"/"+lib_name+"-"+lib_version)
shutil.copytree(src_dir+"/"+lib_name+"-"+lib_version, work_dir+"/"+lib_name+"-"+lib_version) shutil.copytree(src_dir+"/"+lib_name+"-"+lib_version, work_dir+"/"+lib_name+"-"+lib_version)
os.chdir(work_dir+"/"+lib_name+"-"+lib_version) os.chdir(work_dir+"/"+lib_name+"-"+lib_version)
config_command = lib['configure'].replace("$PREFIX",inst_dir) config_command = lib['configure'].replace("$PREFIX", inst_dir)
if verbose: if verbose:
print(underlined("\nConfiguring Library")) print(underlined("\nConfiguring Library"))
else: else:
print("Configuring library...") print("Configuring library...")
err = subprocess.call(config_command,stdout=logfile,stderr=logfile,shell=True) err = subprocess.call(config_command, stdout=logfile, stderr=logfile, shell=True)
if err != 0: if err != 0:
print("Error: Configuring of "+lib_name+" v."+lib_version+" failed!") print("Error: Configuring of "+lib_name+" v."+lib_version+" failed!")
print("See "+logfile_path+" for details") print("See "+logfile_path+" for details")
sys.exit(1) sys.exit(1)
# build library # build library
build_command = lib['build'].replace("$BUILDTHREADS",build_threads) build_command = lib['build'].replace("$BUILDTHREADS", build_threads)
if verbose: if verbose:
print(underlined("\nBuilding Library")) print(underlined("\nBuilding Library"))
err = subprocess.call(build_command,stdout=logfile,stderr=logfile,shell=True) err = subprocess.call(build_command, stdout=logfile, stderr=logfile, shell=True)
else: else:
build_task = subprocess.Popen(build_command,stdout=logfile,stderr=logfile,shell=True) build_task = subprocess.Popen(build_command, stdout=logfile, stderr=logfile, shell=True)
num_objects = int(lib['object files']) num_objects = int(lib['object files'])
finished = False finished = False
while build_task.poll() == None: while build_task.poll() is None:
if not finished: if not finished:
# bar_width = num_objects # bar_width = num_objects
# while (bar_width > 100): # while (bar_width > 100):
# bar_width = int(bar_width/10) # bar_width = int(bar_width/10)
bar_width = 50 bar_width = 50
for i in progressbar(range(num_objects), "Building library: ",bar_width): for i in progressbar(range(num_objects), "Building library: ", bar_width):
while len(glob.glob('**/*.o',recursive=True)) < i: while len(glob.glob('**/*.o', recursive=True)) < i:
time.sleep(0.5) time.sleep(0.5)
if(build_task.poll != None): if build_task.poll is not None:
break break
finished = True finished = True
build_task.wait() build_task.wait()
@ -84,13 +86,13 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr
print("See "+logfile_path+" for details") print("See "+logfile_path+" for details")
sys.exit(1) sys.exit(1)
#install library # install library
install_command = lib['install'].replace("$BUILDTHREADS",build_threads) install_command = lib['install'].replace("$BUILDTHREADS", build_threads)
if verbose: if verbose:
print(underlined("\nInstalling Library")) print(underlined("\nInstalling Library"))
else: else:
print("Installing library...") print("Installing library...")
err = subprocess.call(install_command,stdout=logfile,stderr=logfile,shell=True) err = subprocess.call(install_command, stdout=logfile, stderr=logfile, shell=True)
if err != 0: if err != 0:
print("Error: Installing "+lib_name+" v."+lib_version+" failed!") print("Error: Installing "+lib_name+" v."+lib_version+" failed!")
print("See "+logfile_path+" for details") print("See "+logfile_path+" for details")
@ -99,5 +101,4 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr
build_output_path = inst_dir+"/"+"build_info" build_output_path = inst_dir+"/"+"build_info"
if not os.path.exists(build_output_path): if not os.path.exists(build_output_path):
os.makedirs(build_output_path) os.makedirs(build_output_path)
shutil.copyfile(logfile_path,build_output_path+"/"+lib_name+".log") shutil.copyfile(logfile_path, build_output_path+"/"+lib_name+".log")

1
lib/shell.py
View File

@ -1,6 +1,7 @@
import subprocess import subprocess
import sys import sys
def get_from_command(command): def get_from_command(command):
ret = subprocess.check_output(command).decode(sys.stdout.encoding) ret = subprocess.check_output(command).decode(sys.stdout.encoding)
return ret return ret

11
lib/sort.py
View File

@ -6,12 +6,12 @@ def topological_sort(source):
next_emitted = [] next_emitted = []
for entry in pending: for entry in pending:
name, deps = entry name, deps = entry
deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6 deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6
if deps: if deps:
next_pending.append(entry) next_pending.append(entry)
else: else:
yield name yield name
emitted.append(name) # <-- not required, but preserves original order emitted.append(name) # <-- not required, but preserves original order
next_emitted.append(name) next_emitted.append(name)
if not next_emitted: if not next_emitted:
raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,))) raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
@ -19,14 +19,15 @@ def topological_sort(source):
emitted = next_emitted emitted = next_emitted
return emitted return emitted
def sort_libs_by_dependencies(selected_libs): def sort_libs_by_dependencies(selected_libs):
deplist=[] deplist = []
for lib in selected_libs: for lib in selected_libs:
name = lib['name'] name = lib['name']
dependencies = set(lib['dependencies'].split(',')) dependencies = set(lib['dependencies'].split(','))
if dependencies == {''}: if dependencies == {''}:
dependencies = {} dependencies = {}
deplist.append([name,dependencies]) deplist.append([name, dependencies])
sorted_deplist = topological_sort(deplist) sorted_deplist = topological_sort(deplist)
sorted = [] sorted = []
for entry in sorted_deplist: for entry in sorted_deplist:

18
lib/toolchain.py
View File

@ -1,16 +1,18 @@
import os import os
from lib.shell import get_from_command from lib.shell import get_from_command
def get_compiler_version(compiler): def get_compiler_version(compiler):
if compiler == "gnu": if compiler == "gnu":
cc="gcc" cc = "gcc"
elif compiler == "intel": elif compiler == "intel":
cc="icc" cc = "icc"
elif compiler == "aocc": elif compiler == "aocc":
cc="clang" cc = "clang"
compstr = get_from_command([cc,"-dumpversion"]).strip() compstr = get_from_command([cc, "-dumpversion"]).strip()
return compstr return compstr
def get_mpi_version(mpi): def get_mpi_version(mpi):
if mpi == "hpcx": if mpi == "hpcx":
hpcx_dir = os.getenv('HPCX_DIR') hpcx_dir = os.getenv('HPCX_DIR')
@ -25,12 +27,12 @@ def get_mpi_version(mpi):
rawstr = get_from_command(["ompi_info"]).splitlines()[0] rawstr = get_from_command(["ompi_info"]).splitlines()[0]
mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1] mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1]
elif mpi == "mpich": elif mpi == "mpich":
rawstr = get_from_command(["mpirun","--version"]) rawstr = get_from_command(["mpirun", "--version"])
mpistr = rawstr.split()[4] mpistr = rawstr.split()[4]
return mpistr return mpistr
def set_toolchain(compiler,mpi): def set_toolchain(compiler, mpi):
if compiler == "gnu": if compiler == "gnu":
if mpi == "hpcx" or mpi == "openmpi" or mpi == "mpich": if mpi == "hpcx" or mpi == "openmpi" or mpi == "mpich":
cc = "mpicc" cc = "mpicc"
@ -68,7 +70,7 @@ def set_toolchain(compiler,mpi):
cc = "mpiicc" cc = "mpiicc"
cxx = "mpiicpc" cxx = "mpiicpc"
fc = "mpiifort" fc = "mpiifort"
# set environment variables # set environment variables
os.environ["CC"] = cc os.environ["CC"] = cc
os.environ["CXX"] = cxx os.environ["CXX"] = cxx
@ -76,4 +78,4 @@ def set_toolchain(compiler,mpi):
os.environ["F90"] = fc os.environ["F90"] = fc
os.environ["F77"] = fc os.environ["F77"] = fc
return cc,cxx,fc return cc, cxx, fc

15
lib/ui.py
View File

@ -1,18 +1,21 @@
import sys import sys
import time import time
def progressbar(it, prefix="", size=60, out=sys.stdout):
def progressbar(it, prefix="", size=60, out=sys.stdout):
count = len(it) count = len(it)
def show(j): def show(j):
x = int(size*j/count) x = int(size*j/count)
print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count), print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count), end='\r', file=out, flush=True)
end='\r', file=out, flush=True)
show(0) show(0)
for i, item in enumerate(it): for i, item in enumerate(it):
yield item yield item
show(i+1) show(i+1)
print("", flush=True, file=out) print("", flush=True, file=out)
def bordered(text): def bordered(text):
lines = text.splitlines() lines = text.splitlines()
width = max(len(s) for s in lines) width = max(len(s) for s in lines)
@ -22,15 +25,17 @@ def bordered(text):
res.append('└' + '─' * width + '┘') res.append('└' + '─' * width + '┘')
return '\n'.join(res) return '\n'.join(res)
def underlined(text): def underlined(text):
lines = text.splitlines() lines = text.splitlines()
width = max(len(s) for s in lines) width = max(len(s) for s in lines)
res=[] res = []
for s in lines: for s in lines:
res.append(s) res.append(s)
res.append('─' * width) res.append('─' * width)
return '\n'.join(res) return '\n'.join(res)
def print_welcome(script_version, compiler, compiler_version, mpi, mpi_version, instDir, installs): def print_welcome(script_version, compiler, compiler_version, mpi, mpi_version, instDir, installs):
out = underlined("Patricks Simple Library Installer " + script_version) out = underlined("Patricks Simple Library Installer " + script_version)
out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compiler_version out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compiler_version
@ -43,7 +48,7 @@ def print_welcome(script_version, compiler, compiler_version, mpi, mpi_version,
print(bordered(out) + "\n") print(bordered(out) + "\n")
# short sleep to enable user to abprt if selected options are wrong # short sleep to enable user to abprt if selected options are wrong
for i in progressbar(range(5), "Waiting 5s to start build: ",10): for i in progressbar(range(5), "Waiting 5s to start build: ", 10):
time.sleep(1) time.sleep(1)
print("\n") print("\n")

34
libinstaller.py
View File

@ -9,10 +9,10 @@ from lib.sort import sort_libs_by_dependencies
from lib.installer import install_lib from lib.installer import install_lib
from lib.init import init from lib.init import init
SCRIPT_VERSION="v0.3" SCRIPT_VERSION = "v0.3"
# initialization by argparser # initialization by argparser
arg_namespace,args = init() arg_namespace, args = init()
# extract known arguments from argparse namespace # extract known arguments from argparse namespace
prefix = arg_namespace.prefix prefix = arg_namespace.prefix
@ -26,27 +26,27 @@ build_threads = arg_namespace.threads
verbose = arg_namespace.verbose verbose = arg_namespace.verbose
# extract libraries and versions selected for installation # extract libraries and versions selected for installation
selected_libs=[] selected_libs = []
ignore_names=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"] ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"]
for lib_name in args: for lib_name in args:
if (lib_name not in ignore_names and "version" not in lib_name): if lib_name not in ignore_names and "version" not in lib_name:
install = getattr(arg_namespace,lib_name) install = getattr(arg_namespace, lib_name)
if (install == True): if install:
version = getattr(arg_namespace,lib_name+"_version") version = getattr(arg_namespace, lib_name+"_version")
config_file = config_dir+"/"+lib_name+".json" config_file = config_dir + "/" + lib_name + ".json"
with open(config_file,'r') as cf: with open(config_file, 'r') as cf:
data = json.load(cf) data = json.load(cf)
data['version'] = version data['version'] = version
selected_libs.append(data) selected_libs.append(data)
# set up install directory name # set up install directory name
compiler_version=get_compiler_version(compiler) compiler_version = get_compiler_version(compiler)
mpi_version=get_mpi_version(mpi) mpi_version = get_mpi_version(mpi)
inst_str="inst-"+compiler+"_"+compiler_version+"_"+mpi+"_"+mpi_version inst_str = "inst-"+compiler + "_" + compiler_version + "_" + mpi + "_" + mpi_version
inst_dir=prefix+"/"+inst_str inst_dir = prefix + "/" + inst_str
# set up mpi wrappers and environment variables # set up mpi wrappers and environment variables
cc,cxx,fc = set_toolchain(compiler,mpi) cc, cxx, fc = set_toolchain(compiler, mpi)
# set up library install order by dependencies # set up library install order by dependencies
sorted_libs = sort_libs_by_dependencies(selected_libs) sorted_libs = sort_libs_by_dependencies(selected_libs)
@ -57,7 +57,7 @@ print_welcome(SCRIPT_VERSION, compiler, compiler_version, mpi, mpi_version, inst
# install selected libraries # install selected libraries
if len(sorted_libs) > 0: if len(sorted_libs) > 0:
for lib in sorted_libs: for lib in sorted_libs:
install_lib(lib,src_dir,work_dir,inst_dir,cc,cxx,fc,build_threads,verbose) install_lib(lib, src_dir, work_dir, inst_dir, cc, cxx, fc, build_threads, verbose)
print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+inst_dir+" to your environment")) print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+inst_dir+" to your environment"))
else: else:
print("NO LIBRARIES SELECTED FOR INSTALLATION") print("NO LIBRARIES SELECTED FOR INSTALLATION")