diff --git a/README.md b/README.md index 4cfbac5..353585d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ This Python tool has been developed to automate the installation of libraries frequently used by HPC applications. ## Basic Usage -To install the library `` using compiler `` and MPI `` make sure to have a configuration file `.json` in the `config` directory and invoke the tool as follows: +To install the library `` and all it's dependencies (if there are any) using compiler `` and MPI ``, make sure to have a configuration file `.json` in the `config` directory and invoke the tool as follows: ``` ./libinstaller --compiler --mpi -- ``` @@ -29,8 +29,9 @@ options: --separate-lib64 Do not create symbolic links of files from lib64 in lib --disable-shared Disable building of shared libraries --all Install all libraries with config file in config/ + --ignore-deps Do not add dependencies of libraries to the install process ``` -Per library configuration available in `config`, the corresponding `--` and `---version`options are added automatically, eg: +For each library configuration file available in `config`, the corresponding `--` and `---version` options are added to the parser automatically, eg: ``` --hdf5 Enable build of hdf5 --hdf5-version HDF5_VERSION Set hdf5 version [1.13.2] @@ -58,8 +59,9 @@ The `config` directory contains various json files describing how to obtain and On startup, the installer tool will read the configuration files in `config` and add coreesponding options to the argument parser. In case `---version` is not set, the default version from the json file is used. ### dependencies -Dependencies are used to determine the best installation order of selected librarie to satisfy all dependencies. Note that it is assumed that you supply needed dependencies via `LD_LIBRARY_PATH` in case they are not part of the current install. -Also note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH` +Dependencies are used to determine the best installation order of the selected libraries to satisfy all dependencies. +When the installer encounters a library to install, all dependencies are also added to the installation process if not already selected. In case `--ignore-deps` is set, the dependencies need to be activated manually when invoking libinstaller or be present in `LD_LIBRARY_PATH`. +Note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH` ### download Download command. This is being executed in a subshell. Make sure that it works in the shell used to start libinstaller. diff --git a/config/netcdf-c.json b/config/netcdf-c.json new file mode 100644 index 0000000..5fec04e --- /dev/null +++ b/config/netcdf-c.json @@ -0,0 +1,10 @@ +{ + "name" : "netcdf-c", + "default version" : "4.7.4", + "dependencies" : "zlib,szip,curl,hdf5", + "download" : "git clone --depth 1 --branch v$VERSION https://github.com/Unidata/netcdf-c.git netcdf-c-$VERSION", + "configure" : "./configure --prefix=$PREFIX CPPFLAGS=-I$PREFIX/include LDFLAGS=-L$PREFIX/lib $SHARED", + "build" : "make -j $BUILDTHREADS", + "install" : "make install", + "object files" : "435" +} diff --git a/config/netcdf-f.json b/config/netcdf-f.json new file mode 100644 index 0000000..e33bca9 --- /dev/null +++ b/config/netcdf-f.json @@ -0,0 +1,10 @@ +{ + "name" : "netcdf-f", + "default version" : "4.5.3", + "dependencies" : "zlib,szip,hdf5,curl,netcdf-c", + "download" : "git clone --depth 1 --branch v$VERSION https://github.com/Unidata/netcdf-fortran.git netcdf-f-$VERSION", + "configure" : "./configure --prefix=$PREFIX CPPFLAGS=-I$PREFIX/include LDFLAGS=-L$PREFIX/lib $SHARED", + "build" : "make -j $BUILDTHREADS", + "install" : "make install", + "object files" : "60" +} diff --git a/lib/dependency.py b/lib/dependency.py new file mode 100644 index 0000000..86ba5ec --- /dev/null +++ b/lib/dependency.py @@ -0,0 +1,103 @@ +import os +from lib.io import load_lib_data + +def topological_sort(source): + pending = [(name, set(deps)) for name, deps in source] + emitted = [] + while pending: + next_pending = [] + next_emitted = [] + for entry in pending: + name, deps = entry + deps.difference_update(set((name,)), emitted) + if deps: + next_pending.append(entry) + else: + yield name + emitted.append(name) + next_emitted.append(name) + if not next_emitted: + raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,))) + pending = next_pending + emitted = next_emitted + return emitted + + +def sort_libs_by_dependencies(selected_libs): + # no need for sorting in case of only one lib + if len(selected_libs) < 2: + return selected_libs + + lib_names = [] + for lib in selected_libs: + lib_names.append(lib['name']) + + deplist = [] + for lib in selected_libs: + name = lib['name'] + # only sort after dependencies that are actually present to avoid cyclic dependencies + deps = [] + for dep in lib['dependencies'].split(','): + if dep in lib_names: + deps.append(dep) + dependencies = set(deps) + if dependencies == {''}: + dependencies = {} + deplist.append([name, dependencies]) + sorted_deplist = topological_sort(deplist) + + sorted = [] + for entry in sorted_deplist: + for lib in selected_libs: + if lib['name'] == entry: + sorted.append(lib) + return sorted + + +def load_dependencies(library, selected_libs, config_dir): + lib_names = [] + for lib in selected_libs: + lib_names.append(lib['name']) + + dependencies = library['dependencies'].split(',') + for dep in dependencies: + if len(dep) > 1: + if dep not in lib_names: + config_file = config_dir + "/" + dep + ".json" + if os.path.exists(config_file): + dep_data = load_lib_data(config_file) + dep_data['version'] = dep_data['default version'] + selected_libs.append(dep_data) + else: + print("Warning - library " + lib['name'] + " depends on " + dep + " for which no configration file was found. Presence in LD_LIBRARY_PATH is assumed.") + + +# load library data from names in argparse result +def load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps): + selected_libs = [] + if install_all_libs: + for config_file in glob.glob(config_dir+"/*.json"): + data = load_lib_data(config_file) + #with open(cf, 'r') as f: + #data = json.load(f) + data['version'] = data['default version'] + selected_libs.append(data) + else: + ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared", "ignore_deps"] + for lib_name in args: + if lib_name not in ignore_names and "version" not in lib_name: + install = getattr(arg_namespace, lib_name) + if install: + version = getattr(arg_namespace, lib_name + "_version") + lib_name = lib_name.replace('_','-') + config_file = config_dir + "/" + lib_name + ".json" + data = load_lib_data(config_file) + data['version'] = version + selected_libs.append(data) + + if not ignore_deps: + # also add all dependencies to the install list + for lib in selected_libs: + load_dependencies(lib, selected_libs, config_dir) + + return selected_libs diff --git a/lib/init.py b/lib/init.py index 8c4abc2..5d63404 100644 --- a/lib/init.py +++ b/lib/init.py @@ -29,6 +29,7 @@ def init(): config_parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true') config_parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true') config_parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true') + config_parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true') parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config") parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd()) @@ -42,6 +43,7 @@ def init(): parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true') parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true') parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true') + parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true') # run config parser and search config/*.json to add a build and version argument for it to the full parser config_dir = config_parser.parse_known_args()[0].config diff --git a/lib/io.py b/lib/io.py new file mode 100644 index 0000000..01b9af9 --- /dev/null +++ b/lib/io.py @@ -0,0 +1,7 @@ +import json + +# load library data from json file +def load_lib_data(lib_path): + with open (lib_path, 'r') as config_file: + data = json.load(config_file) + return data \ No newline at end of file diff --git a/libinstaller b/libinstaller index 2632deb..1da4a44 100755 --- a/libinstaller +++ b/libinstaller @@ -1,16 +1,14 @@ #!/usr/bin/env python3 import shutil -import json import os import glob -from lib.ui import progressbar, bordered, underlined, print_welcome -from lib.shell import get_from_command +from lib.ui import bordered, print_welcome from lib.toolchain import get_compiler_version, get_mpi_version, set_toolchain -from lib.sort import sort_libs_by_dependencies +from lib.dependency import sort_libs_by_dependencies, load_selected_libs from lib.installer import install_lib from lib.init import init, check_python_version -SCRIPT_VERSION = "v0.8" +SCRIPT_VERSION = "v0.9" # check if Python >=3.3.0 is used check_python_version() @@ -31,15 +29,19 @@ verbose = arg_namespace.verbose separate_lib64 = arg_namespace.separate_lib64 disable_shared = arg_namespace.disable_shared install_all_libs = arg_namespace.all +ignore_deps = arg_namespace.ignore_deps # extract libraries and versions selected for installation +selected_libs = load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps) +''' selected_libs = [] if install_all_libs: - for cf in glob.glob(config_dir+"/*.json"): - with open(cf, 'r') as f: - data = json.load(f) - data['version'] = data['default version'] - selected_libs.append(data) + for config_file in glob.glob(config_dir+"/*.json"): + data = load_lib_data(config_file) + #with open(cf, 'r') as f: + #data = json.load(f) + data['version'] = data['default version'] + selected_libs.append(data) else: ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared"] for lib_name in args: @@ -48,10 +50,14 @@ else: if install: version = getattr(arg_namespace, lib_name+"_version") config_file = config_dir + "/" + lib_name + ".json" - with open(config_file, 'r') as cf: - data = json.load(cf) - data['version'] = version - selected_libs.append(data) + data = load_lib_data(config_file) + data['version'] = data['default version'] + selected_libs.append(data) + #with open(config_file, 'r') as cf: + #data = json.load(cf) + #data['version'] = version + #selected_libs.append(data) +''' # set up install directory name compiler_version = get_compiler_version(compiler)