patrick
/
libinstaller
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

v0.9 - dependency management changed to enable automatic resolving of dependencies

This commit is contained in:
Patrick Lipka 2022-09-30 00:33:42 +02:00
parent bb9fa74826
commit 9bd62c783d
7 changed files with 158 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -3,7 +3,7 @@
This Python tool has been developed to automate the installation of libraries frequently used by HPC applications.
## Basic Usage
To install the library `<lib>` using compiler `<compiler>` and MPI `<mpi>` make sure to have a configuration file `<lib>.json` in the `config` directory and invoke the tool as follows:
To install the library `<lib>` and all it's dependencies (if there are any) using compiler `<compiler>` and MPI `<mpi>`, make sure to have a configuration file `<lib>.json` in the `config` directory and invoke the tool as follows:
```
./libinstaller --compiler <compiler> --mpi <mpi> --<lib>
```
@ -29,8 +29,9 @@ options:
--separate-lib64 Do not create symbolic links of files from lib64 in lib
--disable-shared Disable building of shared libraries
--all Install all libraries with config file in config/
--ignore-deps Do not add dependencies of libraries to the install process
```
Per library configuration available in `config`, the corresponding `--<lib>` and `--<lib>-version`options are added automatically, eg:
For each library configuration file available in `config`, the corresponding `--<lib>` and `--<lib>-version` options are added to the parser automatically, eg:
```
--hdf5 Enable build of hdf5
--hdf5-version HDF5_VERSION Set hdf5 version [1.13.2]
@ -58,8 +59,9 @@ The `config` directory contains various json files describing how to obtain and
On startup, the installer tool will read the configuration files in `config` and add coreesponding options to the argument parser. In case `--<lib>-version` is not set, the default version from the json file is used.
### dependencies
Dependencies are used to determine the best installation order of selected librarie to satisfy all dependencies. Note that it is assumed that you supply needed dependencies via `LD_LIBRARY_PATH` in case they are not part of the current install.
Also note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH`
Dependencies are used to determine the best installation order of the selected libraries to satisfy all dependencies.
When the installer encounters a library to install, all dependencies are also added to the installation process if not already selected. In case `--ignore-deps` is set, the dependencies need to be activated manually when invoking libinstaller or be present in `LD_LIBRARY_PATH`.
Note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH`
### download
Download command. This is being executed in a subshell. Make sure that it works in the shell used to start libinstaller.

10
config/netcdf-c.json Normal file
View File

@ -0,0 +1,10 @@
{
"name" : "netcdf-c",
"default version" : "4.7.4",
"dependencies" : "zlib,szip,curl,hdf5",
"download" : "git clone --depth 1 --branch v$VERSION https://github.com/Unidata/netcdf-c.git netcdf-c-$VERSION",
"configure" : "./configure --prefix=$PREFIX CPPFLAGS=-I$PREFIX/include LDFLAGS=-L$PREFIX/lib $SHARED",
"build" : "make -j $BUILDTHREADS",
"install" : "make install",
"object files" : "435"
}

10
config/netcdf-f.json Normal file
View File

@ -0,0 +1,10 @@
{
"name" : "netcdf-f",
"default version" : "4.5.3",
"dependencies" : "zlib,szip,hdf5,curl,netcdf-c",
"download" : "git clone --depth 1 --branch v$VERSION https://github.com/Unidata/netcdf-fortran.git netcdf-f-$VERSION",
"configure" : "./configure --prefix=$PREFIX CPPFLAGS=-I$PREFIX/include LDFLAGS=-L$PREFIX/lib $SHARED",
"build" : "make -j $BUILDTHREADS",
"install" : "make install",
"object files" : "60"
}

103
lib/dependency.py Normal file
View File

@ -0,0 +1,103 @@
import os
from lib.io import load_lib_data
def topological_sort(source):
pending = [(name, set(deps)) for name, deps in source]
emitted = []
while pending:
next_pending = []
next_emitted = []
for entry in pending:
name, deps = entry
deps.difference_update(set((name,)), emitted)
if deps:
next_pending.append(entry)
else:
yield name
emitted.append(name)
next_emitted.append(name)
if not next_emitted:
raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
pending = next_pending
emitted = next_emitted
return emitted
def sort_libs_by_dependencies(selected_libs):
# no need for sorting in case of only one lib
if len(selected_libs) < 2:
return selected_libs
lib_names = []
for lib in selected_libs:
lib_names.append(lib['name'])
deplist = []
for lib in selected_libs:
name = lib['name']
# only sort after dependencies that are actually present to avoid cyclic dependencies
deps = []
for dep in lib['dependencies'].split(','):
if dep in lib_names:
deps.append(dep)
dependencies = set(deps)
if dependencies == {''}:
dependencies = {}
deplist.append([name, dependencies])
sorted_deplist = topological_sort(deplist)
sorted = []
for entry in sorted_deplist:
for lib in selected_libs:
if lib['name'] == entry:
sorted.append(lib)
return sorted
def load_dependencies(library, selected_libs, config_dir):
lib_names = []
for lib in selected_libs:
lib_names.append(lib['name'])
dependencies = library['dependencies'].split(',')
for dep in dependencies:
if len(dep) > 1:
if dep not in lib_names:
config_file = config_dir + "/" + dep + ".json"
if os.path.exists(config_file):
dep_data = load_lib_data(config_file)
dep_data['version'] = dep_data['default version']
selected_libs.append(dep_data)
else:
print("Warning - library " + lib['name'] + " depends on " + dep + " for which no configration file was found. Presence in LD_LIBRARY_PATH is assumed.")
# load library data from names in argparse result
def load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps):
selected_libs = []
if install_all_libs:
for config_file in glob.glob(config_dir+"/*.json"):
data = load_lib_data(config_file)
#with open(cf, 'r') as f:
#data = json.load(f)
data['version'] = data['default version']
selected_libs.append(data)
else:
ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared", "ignore_deps"]
for lib_name in args:
if lib_name not in ignore_names and "version" not in lib_name:
install = getattr(arg_namespace, lib_name)
if install:
version = getattr(arg_namespace, lib_name + "_version")
lib_name = lib_name.replace('_','-')
config_file = config_dir + "/" + lib_name + ".json"
data = load_lib_data(config_file)
data['version'] = version
selected_libs.append(data)
if not ignore_deps:
# also add all dependencies to the install list
for lib in selected_libs:
load_dependencies(lib, selected_libs, config_dir)
return selected_libs

2
lib/init.py
View File

@ -29,6 +29,7 @@ def init():
config_parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true')
config_parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true')
config_parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true')
config_parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true')
parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config")
parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd())
@ -42,6 +43,7 @@ def init():
parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true')
parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true')
parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true')
parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true')
# run config parser and search config/*.json to add a build and version argument for it to the full parser
config_dir = config_parser.parse_known_args()[0].config

7
lib/io.py Normal file
View File

@ -0,0 +1,7 @@
import json
# load library data from json file
def load_lib_data(lib_path):
with open (lib_path, 'r') as config_file:
data = json.load(config_file)
return data

34
libinstaller
View File

@ -1,16 +1,14 @@
#!/usr/bin/env python3
import shutil
import json
import os
import glob
from lib.ui import progressbar, bordered, underlined, print_welcome
from lib.shell import get_from_command
from lib.ui import bordered, print_welcome
from lib.toolchain import get_compiler_version, get_mpi_version, set_toolchain
from lib.sort import sort_libs_by_dependencies
from lib.dependency import sort_libs_by_dependencies, load_selected_libs
from lib.installer import install_lib
from lib.init import init, check_python_version
SCRIPT_VERSION = "v0.8"
SCRIPT_VERSION = "v0.9"
# check if Python >=3.3.0 is used
check_python_version()
@ -31,15 +29,19 @@ verbose = arg_namespace.verbose
separate_lib64 = arg_namespace.separate_lib64
disable_shared = arg_namespace.disable_shared
install_all_libs = arg_namespace.all
ignore_deps = arg_namespace.ignore_deps
# extract libraries and versions selected for installation
selected_libs = load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps)
'''
selected_libs = []
if install_all_libs:
for cf in glob.glob(config_dir+"/*.json"):
with open(cf, 'r') as f:
data = json.load(f)
data['version'] = data['default version']
selected_libs.append(data)
for config_file in glob.glob(config_dir+"/*.json"):
data = load_lib_data(config_file)
#with open(cf, 'r') as f:
#data = json.load(f)
data['version'] = data['default version']
selected_libs.append(data)
else:
ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared"]
for lib_name in args:
@ -48,10 +50,14 @@ else:
if install:
version = getattr(arg_namespace, lib_name+"_version")
config_file = config_dir + "/" + lib_name + ".json"
with open(config_file, 'r') as cf:
data = json.load(cf)
data['version'] = version
selected_libs.append(data)
data = load_lib_data(config_file)
data['version'] = data['default version']
selected_libs.append(data)
#with open(config_file, 'r') as cf:
#data = json.load(cf)
#data['version'] = version
#selected_libs.append(data)
'''
# set up install directory name
compiler_version = get_compiler_version(compiler)