From a6e549a2dae0e098ed5b2374915bd31ab0af2f30 Mon Sep 17 00:00:00 2001 From: Patrick Lipka Date: Thu, 22 Sep 2022 01:51:47 +0200 Subject: [PATCH] Improved codestyle conformancy to PEP8 --- lib/init.py | 49 ++++++++++++++++++++++++------------------------ lib/installer.py | 45 ++++++++++++++++++++++---------------------- lib/shell.py | 1 + lib/sort.py | 11 ++++++----- lib/toolchain.py | 18 ++++++++++-------- lib/ui.py | 15 ++++++++++----- libinstaller.py | 34 ++++++++++++++++----------------- 7 files changed, 92 insertions(+), 81 deletions(-) diff --git a/lib/init.py b/lib/init.py index 4f902a0..33e6edc 100644 --- a/lib/init.py +++ b/lib/init.py @@ -3,42 +3,43 @@ import argparse import glob import json + def init(): - # set up two parsers: first parse config directory, and add dynamic arguments to full parser + # set up two parsers: first parse config directory, and add dynamic arguments to full parser config_parser = argparse.ArgumentParser(add_help=False) parser = argparse.ArgumentParser() # parsers need to share known arguments - config_parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") - config_parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) - config_parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") - config_parser.add_argument('--work',help='Path to working directory for builds [$pwd/work',default=os.getcwd()+"/work") - config_parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') - config_parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') - config_parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') - config_parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') - config_parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') + config_parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config") + config_parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd()) + config_parser.add_argument('--src', help='Path where to download source code to [$pwd/src]', default=os.getcwd()+"/src") + config_parser.add_argument('--work', help='Path to working directory for builds [$pwd/work', default=os.getcwd()+"/work") + config_parser.add_argument('--keep-work', help='Disable removal of work directory after successful builds', action='store_true') + config_parser.add_argument('--compiler', help='Select compiler (gnu, intel, aocc) [gnu]', default='gnu') + config_parser.add_argument('--mpi', help='Select compiler (hpcx, intelmpi, openmpi) [hpcx]', default='hpcx') + config_parser.add_argument('--threads', help='Number of threads used for make [8]', default='8') + config_parser.add_argument('--verbose', help='Print build output to screen instead piping it to logile', action='store_true') - parser.add_argument('--config',help='Path to config directory [$pwd/config]',default=os.getcwd()+"/config") - parser.add_argument('--prefix',help='Path where install directory should be generated [$pwd]',default=os.getcwd()) - parser.add_argument('--src' ,help='Path where to download source code to [$pwd/src]',default=os.getcwd()+"/src") - parser.add_argument('--work' ,help='Path to working directory for builds [$pwd/work]',default=os.getcwd()+"/work") - parser.add_argument('--keep-work',help='Disable removal of work directory after successful builds',action='store_true') - parser.add_argument('--compiler',help='Select compiler (gnu,intel,aocc) [gnu]',default='gnu') - parser.add_argument('--mpi',help='Select compiler (hpcx,intelmpi,openmpi) [hpcx]',default='hpcx') - parser.add_argument('--threads',help='Number of threads used for make [8]',default='8') - parser.add_argument('--verbose',help='Print build output to screen instead piping it to logile',action='store_true') + parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config") + parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd()) + parser.add_argument('--src', help='Path where to download source code to [$pwd/src]', default=os.getcwd()+"/src") + parser.add_argument('--work', help='Path to working directory for builds [$pwd/work]', default=os.getcwd()+"/work") + parser.add_argument('--keep-work', help='Disable removal of work directory after successful builds', action='store_true') + parser.add_argument('--compiler', help='Select compiler (gnu, intel, aocc) [gnu]', default='gnu') + parser.add_argument('--mpi', help='Select compiler (hpcx, intelmpi, openmpi) [hpcx]', default='hpcx') + parser.add_argument('--threads', help='Number of threads used for make [8]', default='8') + parser.add_argument('--verbose', help='Print build output to screen instead piping it to logile', action='store_true') - # run config parser and serach config/*.json to add a build and version argument for it to the full parser + # run config parser and search config/*.json to add a build and version argument for it to the full parser config_dir = config_parser.parse_known_args()[0].config for cf in glob.glob(config_dir+"/*.json"): with open(cf, 'r') as f: lib_data = json.load(f) - parser.add_argument('--'+lib_data['name'],help='Enable build of '+lib_data['name'],action='store_true') - parser.add_argument('--'+lib_data['name']+'-version',help='Set '+lib_data['name']+' version ['+lib_data['default version']+']',default=str(lib_data['default version'])) + parser.add_argument('--'+lib_data['name'], help='Enable build of '+lib_data['name'], action='store_true') + parser.add_argument('--'+lib_data['name']+'-version', help='Set '+lib_data['name']+' version ['+lib_data['default version']+']', default=str(lib_data['default version'])) # run full parser arg_namespace = parser.parse_args() args = vars(arg_namespace) - - return arg_namespace,args \ No newline at end of file + + return arg_namespace, args \ No newline at end of file diff --git a/lib/installer.py b/lib/installer.py index 187deb7..98e3f6d 100644 --- a/lib/installer.py +++ b/lib/installer.py @@ -4,10 +4,11 @@ import subprocess import time import glob import shutil -from lib.ui import bordered, underlined, progressbar +from lib.ui import bordered, underlined, progressbar -def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_threads,verbose): - print(bordered("Installing "+lib['name']+" v."+lib['version'])) + +def install_lib(lib, src_dir, work_dir, inst_dir, comp_cc, comp_cxx, comp_fc, build_threads, verbose): + print(bordered("Installing " + lib['name'] + " v." + lib['version'])) if not os.path.exists(src_dir): os.makedirs(src_dir) if not os.path.exists(work_dir): @@ -19,11 +20,11 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr lib_name = lib['name'] lib_version = lib['version'] - if verbose == False: + if not verbose: logfile_path = work_dir + "/" + lib_name + "-" + lib_version + ".log" if os.path.exists(logfile_path): os.remove(logfile_path) - logfile = open(logfile_path,'a') + logfile = open(logfile_path, 'a') else: logfile = None @@ -31,49 +32,50 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr if not os.path.exists(lib['name']+"-"+lib['version']): print("Downloading Source Code") - load_command = lib['download'].replace("$VERSION_UNDERSCORE",lib_version.replace(".","_")).replace("$VERSION",lib_version).split() - err = subprocess.call(load_command,stdout=logfile,stderr=logfile) + load_command = lib['download'].replace("$VERSION_UNDERSCORE", lib_version.replace(".", "_")).replace("$VERSION", lib_version).split() + err = subprocess.call(load_command, stdout=logfile, stderr=logfile) if err != 0: print("Error: Download of "+lib_name+" v."+lib_version+" failed!") print("See "+logfile_path+" for details") sys.exit(0) + # configure library os.chdir(work_dir) if os.path.exists(work_dir+"/"+lib_name+"-"+lib_version): shutil.rmtree(work_dir+"/"+lib_name+"-"+lib_version) - shutil.copytree(src_dir+"/"+lib_name+"-"+lib_version, work_dir+"/"+lib_name+"-"+lib_version) + shutil.copytree(src_dir+"/"+lib_name+"-"+lib_version, work_dir+"/"+lib_name+"-"+lib_version) os.chdir(work_dir+"/"+lib_name+"-"+lib_version) - config_command = lib['configure'].replace("$PREFIX",inst_dir) + config_command = lib['configure'].replace("$PREFIX", inst_dir) if verbose: print(underlined("\nConfiguring Library")) else: print("Configuring library...") - err = subprocess.call(config_command,stdout=logfile,stderr=logfile,shell=True) + err = subprocess.call(config_command, stdout=logfile, stderr=logfile, shell=True) if err != 0: print("Error: Configuring of "+lib_name+" v."+lib_version+" failed!") print("See "+logfile_path+" for details") sys.exit(1) # build library - build_command = lib['build'].replace("$BUILDTHREADS",build_threads) + build_command = lib['build'].replace("$BUILDTHREADS", build_threads) if verbose: print(underlined("\nBuilding Library")) - err = subprocess.call(build_command,stdout=logfile,stderr=logfile,shell=True) + err = subprocess.call(build_command, stdout=logfile, stderr=logfile, shell=True) else: - build_task = subprocess.Popen(build_command,stdout=logfile,stderr=logfile,shell=True) + build_task = subprocess.Popen(build_command, stdout=logfile, stderr=logfile, shell=True) num_objects = int(lib['object files']) finished = False - while build_task.poll() == None: + while build_task.poll() is None: if not finished: # bar_width = num_objects # while (bar_width > 100): # bar_width = int(bar_width/10) bar_width = 50 - for i in progressbar(range(num_objects), "Building library: ",bar_width): - while len(glob.glob('**/*.o',recursive=True)) < i: + for i in progressbar(range(num_objects), "Building library: ", bar_width): + while len(glob.glob('**/*.o', recursive=True)) < i: time.sleep(0.5) - if(build_task.poll != None): + if build_task.poll is not None: break finished = True build_task.wait() @@ -84,13 +86,13 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr print("See "+logfile_path+" for details") sys.exit(1) - #install library - install_command = lib['install'].replace("$BUILDTHREADS",build_threads) + # install library + install_command = lib['install'].replace("$BUILDTHREADS", build_threads) if verbose: print(underlined("\nInstalling Library")) else: print("Installing library...") - err = subprocess.call(install_command,stdout=logfile,stderr=logfile,shell=True) + err = subprocess.call(install_command, stdout=logfile, stderr=logfile, shell=True) if err != 0: print("Error: Installing "+lib_name+" v."+lib_version+" failed!") print("See "+logfile_path+" for details") @@ -99,5 +101,4 @@ def install_lib(lib,src_dir,work_dir,inst_dir,comp_cc,comp_cxx,comp_fc,build_thr build_output_path = inst_dir+"/"+"build_info" if not os.path.exists(build_output_path): os.makedirs(build_output_path) - shutil.copyfile(logfile_path,build_output_path+"/"+lib_name+".log") - + shutil.copyfile(logfile_path, build_output_path+"/"+lib_name+".log") \ No newline at end of file diff --git a/lib/shell.py b/lib/shell.py index 290db37..165a739 100644 --- a/lib/shell.py +++ b/lib/shell.py @@ -1,6 +1,7 @@ import subprocess import sys + def get_from_command(command): ret = subprocess.check_output(command).decode(sys.stdout.encoding) return ret \ No newline at end of file diff --git a/lib/sort.py b/lib/sort.py index 2c078c8..042b2fa 100644 --- a/lib/sort.py +++ b/lib/sort.py @@ -6,12 +6,12 @@ def topological_sort(source): next_emitted = [] for entry in pending: name, deps = entry - deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6 + deps.difference_update(set((name,)), emitted) # <-- pop self from dep, req Py2.6 if deps: next_pending.append(entry) else: yield name - emitted.append(name) # <-- not required, but preserves original order + emitted.append(name) # <-- not required, but preserves original order next_emitted.append(name) if not next_emitted: raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,))) @@ -19,14 +19,15 @@ def topological_sort(source): emitted = next_emitted return emitted + def sort_libs_by_dependencies(selected_libs): - deplist=[] + deplist = [] for lib in selected_libs: name = lib['name'] dependencies = set(lib['dependencies'].split(',')) if dependencies == {''}: - dependencies = {} - deplist.append([name,dependencies]) + dependencies = {} + deplist.append([name, dependencies]) sorted_deplist = topological_sort(deplist) sorted = [] for entry in sorted_deplist: diff --git a/lib/toolchain.py b/lib/toolchain.py index b91037d..de159fd 100644 --- a/lib/toolchain.py +++ b/lib/toolchain.py @@ -1,16 +1,18 @@ import os from lib.shell import get_from_command + def get_compiler_version(compiler): if compiler == "gnu": - cc="gcc" + cc = "gcc" elif compiler == "intel": - cc="icc" + cc = "icc" elif compiler == "aocc": - cc="clang" - compstr = get_from_command([cc,"-dumpversion"]).strip() + cc = "clang" + compstr = get_from_command([cc, "-dumpversion"]).strip() return compstr + def get_mpi_version(mpi): if mpi == "hpcx": hpcx_dir = os.getenv('HPCX_DIR') @@ -25,12 +27,12 @@ def get_mpi_version(mpi): rawstr = get_from_command(["ompi_info"]).splitlines()[0] mpistr = rawstr[rawstr.find("Version")+8:rawstr.find("Build")-1] elif mpi == "mpich": - rawstr = get_from_command(["mpirun","--version"]) + rawstr = get_from_command(["mpirun", "--version"]) mpistr = rawstr.split()[4] return mpistr -def set_toolchain(compiler,mpi): +def set_toolchain(compiler, mpi): if compiler == "gnu": if mpi == "hpcx" or mpi == "openmpi" or mpi == "mpich": cc = "mpicc" @@ -68,7 +70,7 @@ def set_toolchain(compiler,mpi): cc = "mpiicc" cxx = "mpiicpc" fc = "mpiifort" - + # set environment variables os.environ["CC"] = cc os.environ["CXX"] = cxx @@ -76,4 +78,4 @@ def set_toolchain(compiler,mpi): os.environ["F90"] = fc os.environ["F77"] = fc - return cc,cxx,fc \ No newline at end of file + return cc, cxx, fc \ No newline at end of file diff --git a/lib/ui.py b/lib/ui.py index 0f2b51a..a4ecdef 100644 --- a/lib/ui.py +++ b/lib/ui.py @@ -1,18 +1,21 @@ import sys import time -def progressbar(it, prefix="", size=60, out=sys.stdout): + +def progressbar(it, prefix="", size=60, out=sys.stdout): count = len(it) + def show(j): x = int(size*j/count) - print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count), - end='\r', file=out, flush=True) + print("{}[{}{}] {}/{}".format(prefix, u"█"*x, "."*(size-x), j, count), end='\r', file=out, flush=True) + show(0) for i, item in enumerate(it): yield item show(i+1) print("", flush=True, file=out) + def bordered(text): lines = text.splitlines() width = max(len(s) for s in lines) @@ -22,15 +25,17 @@ def bordered(text): res.append('└' + '─' * width + '┘') return '\n'.join(res) + def underlined(text): lines = text.splitlines() width = max(len(s) for s in lines) - res=[] + res = [] for s in lines: res.append(s) res.append('─' * width) return '\n'.join(res) + def print_welcome(script_version, compiler, compiler_version, mpi, mpi_version, instDir, installs): out = underlined("Patricks Simple Library Installer " + script_version) out = out + "\n" + "Compiler: " + compiler.upper() + ", Version: " + compiler_version @@ -43,7 +48,7 @@ def print_welcome(script_version, compiler, compiler_version, mpi, mpi_version, print(bordered(out) + "\n") # short sleep to enable user to abprt if selected options are wrong - for i in progressbar(range(5), "Waiting 5s to start build: ",10): + for i in progressbar(range(5), "Waiting 5s to start build: ", 10): time.sleep(1) print("\n") \ No newline at end of file diff --git a/libinstaller.py b/libinstaller.py index 4d80071..91f03f7 100755 --- a/libinstaller.py +++ b/libinstaller.py @@ -9,10 +9,10 @@ from lib.sort import sort_libs_by_dependencies from lib.installer import install_lib from lib.init import init -SCRIPT_VERSION="v0.3" +SCRIPT_VERSION = "v0.3" # initialization by argparser -arg_namespace,args = init() +arg_namespace, args = init() # extract known arguments from argparse namespace prefix = arg_namespace.prefix @@ -26,27 +26,27 @@ build_threads = arg_namespace.threads verbose = arg_namespace.verbose # extract libraries and versions selected for installation -selected_libs=[] -ignore_names=["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"] +selected_libs = [] +ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "threads", "verbose", "version"] for lib_name in args: - if (lib_name not in ignore_names and "version" not in lib_name): - install = getattr(arg_namespace,lib_name) - if (install == True): - version = getattr(arg_namespace,lib_name+"_version") - config_file = config_dir+"/"+lib_name+".json" - with open(config_file,'r') as cf: + if lib_name not in ignore_names and "version" not in lib_name: + install = getattr(arg_namespace, lib_name) + if install: + version = getattr(arg_namespace, lib_name+"_version") + config_file = config_dir + "/" + lib_name + ".json" + with open(config_file, 'r') as cf: data = json.load(cf) data['version'] = version selected_libs.append(data) # set up install directory name -compiler_version=get_compiler_version(compiler) -mpi_version=get_mpi_version(mpi) -inst_str="inst-"+compiler+"_"+compiler_version+"_"+mpi+"_"+mpi_version -inst_dir=prefix+"/"+inst_str +compiler_version = get_compiler_version(compiler) +mpi_version = get_mpi_version(mpi) +inst_str = "inst-"+compiler + "_" + compiler_version + "_" + mpi + "_" + mpi_version +inst_dir = prefix + "/" + inst_str -# set up mpi wrappers and environment variables -cc,cxx,fc = set_toolchain(compiler,mpi) +# set up mpi wrappers and environment variables +cc, cxx, fc = set_toolchain(compiler, mpi) # set up library install order by dependencies sorted_libs = sort_libs_by_dependencies(selected_libs) @@ -57,7 +57,7 @@ print_welcome(SCRIPT_VERSION, compiler, compiler_version, mpi, mpi_version, inst # install selected libraries if len(sorted_libs) > 0: for lib in sorted_libs: - install_lib(lib,src_dir,work_dir,inst_dir,cc,cxx,fc,build_threads,verbose) + install_lib(lib, src_dir, work_dir, inst_dir, cc, cxx, fc, build_threads, verbose) print(bordered("ALL INSTALLS COMPLETED SUCCESSFULLY\nPlease add "+inst_dir+" to your environment")) else: print("NO LIBRARIES SELECTED FOR INSTALLATION")