v0.9 - dependency management changed to enable automatic resolving of dependencies

README.md

@@ -3,7 +3,7 @@
 This Python tool has been developed to automate the installation of libraries frequently used by HPC applications.
 
 ## Basic Usage
-To install the library `<lib>` using compiler `<compiler>` and MPI `<mpi>` make sure to have a configuration file `<lib>.json` in the `config` directory and invoke the tool as follows:
+To install the library `<lib>` and all it's dependencies (if there are any) using compiler `<compiler>` and MPI `<mpi>`, make sure to have a configuration file `<lib>.json` in the `config` directory and invoke the tool as follows:
 ```
 ./libinstaller --compiler <compiler> --mpi <mpi> --<lib>
 ```
@@ -29,8 +29,9 @@ options:
   --separate-lib64     Do not create symbolic links of files from lib64 in lib
   --disable-shared     Disable building of shared libraries
   --all                Install all libraries with config file in config/
+  --ignore-deps        Do not add dependencies of libraries to the install process
 ```
-Per library configuration available in `config`, the corresponding `--<lib>` and `--<lib>-version`options are added automatically, eg:
+For each library configuration file available in `config`, the corresponding `--<lib>` and `--<lib>-version` options are added to the parser automatically, eg:
 ```
 --hdf5                          Enable build of hdf5
 --hdf5-version HDF5_VERSION     Set hdf5 version [1.13.2]
@@ -58,8 +59,9 @@ The `config` directory contains various json files describing how to obtain and
 On startup, the installer tool will read the configuration files in `config` and add coreesponding options to the argument parser. In case `--<lib>-version` is not set, the default version from the json file is used.
 
 ### dependencies
-Dependencies are used to determine the best installation order of selected librarie to satisfy all dependencies. Note that it is assumed that you supply needed dependencies via `LD_LIBRARY_PATH` in case they are not part of the current install.
+Dependencies are used to determine the best installation order of the selected libraries to satisfy all dependencies.
-Also note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH`
+When the installer encounters a library to install, all dependencies are also added to the installation process if not already selected. In case `--ignore-deps` is set, the dependencies need to be activated manually when invoking libinstaller or be present in `LD_LIBRARY_PATH`.
+Note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH`
 
 ### download
 Download command. This is being executed in a subshell. Make sure that it works in the shell used to start libinstaller.

config/netcdf-c.json

@@ -0,0 +1,10 @@
+{
+    "name" : "netcdf-c",
+    "default version" : "4.7.4",
+    "dependencies" : "zlib,szip,curl,hdf5",
+    "download" : "git clone --depth 1 --branch v$VERSION https://github.com/Unidata/netcdf-c.git netcdf-c-$VERSION",
+    "configure" : "./configure --prefix=$PREFIX CPPFLAGS=-I$PREFIX/include LDFLAGS=-L$PREFIX/lib $SHARED",
+    "build" : "make -j $BUILDTHREADS",
+    "install" : "make install",
+    "object files" : "435"
+}

config/netcdf-f.json

@@ -0,0 +1,10 @@
+{
+    "name" : "netcdf-f",
+    "default version" : "4.5.3",
+    "dependencies" : "zlib,szip,hdf5,curl,netcdf-c",
+    "download" : "git clone --depth 1 --branch v$VERSION https://github.com/Unidata/netcdf-fortran.git netcdf-f-$VERSION",
+    "configure" : "./configure --prefix=$PREFIX CPPFLAGS=-I$PREFIX/include LDFLAGS=-L$PREFIX/lib  $SHARED",
+    "build" : "make -j $BUILDTHREADS",
+    "install" : "make install",
+    "object files" : "60"
+}

lib/dependency.py

@@ -0,0 +1,103 @@
+import os
+from lib.io import load_lib_data
+
+def topological_sort(source):
+    pending = [(name, set(deps)) for name, deps in source]
+    emitted = []
+    while pending:
+        next_pending = []
+        next_emitted = []
+        for entry in pending:
+            name, deps = entry
+            deps.difference_update(set((name,)), emitted)
+            if deps:
+                next_pending.append(entry)
+            else:
+                yield name
+                emitted.append(name)
+                next_emitted.append(name)
+        if not next_emitted:
+            raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
+        pending = next_pending
+        emitted = next_emitted
+    return emitted
+
+
+def sort_libs_by_dependencies(selected_libs):
+    # no need for sorting in case of only one lib
+    if len(selected_libs) < 2:
+        return selected_libs
+
+    lib_names = []
+    for lib in selected_libs:
+        lib_names.append(lib['name'])
+
+    deplist = []
+    for lib in selected_libs:
+        name = lib['name']
+        # only sort after dependencies that are actually present to avoid cyclic dependencies
+        deps = []
+        for dep in lib['dependencies'].split(','):
+            if dep in lib_names:
+                deps.append(dep)
+        dependencies = set(deps)
+        if dependencies == {''}:
+            dependencies = {}
+        deplist.append([name, dependencies])
+    sorted_deplist = topological_sort(deplist)
+
+    sorted = []
+    for entry in sorted_deplist:
+        for lib in selected_libs:
+            if lib['name'] == entry:
+                sorted.append(lib)
+    return sorted
+
+
+def load_dependencies(library, selected_libs, config_dir):
+    lib_names = []
+    for lib in selected_libs:
+        lib_names.append(lib['name'])
+    
+    dependencies = library['dependencies'].split(',')
+    for dep in dependencies:
+        if len(dep) > 1:
+            if dep not in lib_names:
+                config_file = config_dir + "/" + dep + ".json"
+                if os.path.exists(config_file):
+                    dep_data = load_lib_data(config_file)
+                    dep_data['version'] = dep_data['default version']
+                    selected_libs.append(dep_data)
+                else:
+                    print("Warning - library " + lib['name'] + " depends on " + dep + " for which no configration file was found. Presence in LD_LIBRARY_PATH is assumed.")
+
+
+# load library data from names in argparse result
+def load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps):
+    selected_libs = []
+    if install_all_libs:
+        for config_file in glob.glob(config_dir+"/*.json"):
+            data = load_lib_data(config_file)
+            #with open(cf, 'r') as f:
+                #data = json.load(f)
+            data['version'] = data['default version']
+            selected_libs.append(data)
+    else:
+        ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared", "ignore_deps"]
+        for lib_name in args:
+            if lib_name not in ignore_names and "version" not in lib_name:
+                install = getattr(arg_namespace, lib_name)
+                if install:
+                    version = getattr(arg_namespace, lib_name + "_version")
+                    lib_name = lib_name.replace('_','-')
+                    config_file = config_dir + "/" + lib_name + ".json"
+                    data = load_lib_data(config_file)
+                    data['version'] = version
+                    selected_libs.append(data)
+    
+    if not ignore_deps:
+        # also add all dependencies to the install list
+        for lib in selected_libs:
+            load_dependencies(lib, selected_libs, config_dir)
+    
+    return selected_libs

lib/init.py

@@ -29,6 +29,7 @@ def init():
     config_parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true')
     config_parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true')
     config_parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true')
+    config_parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true')
 
     parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config")
     parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd())
@@ -42,6 +43,7 @@ def init():
     parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true')
     parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true')
     parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true')
+    parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true')
 
     # run config parser and search config/*.json to add a build and version argument for it to the full parser
     config_dir = config_parser.parse_known_args()[0].config

lib/io.py

@@ -0,0 +1,7 @@
+import json
+
+# load library data from json file
+def load_lib_data(lib_path):
+    with open (lib_path, 'r') as config_file:
+        data = json.load(config_file)
+        return data

libinstaller

@@ -1,16 +1,14 @@
 #!/usr/bin/env python3
 import shutil
-import json
 import os
 import glob
-from lib.ui import progressbar, bordered, underlined, print_welcome
+from lib.ui import bordered, print_welcome
-from lib.shell import get_from_command
 from lib.toolchain import get_compiler_version, get_mpi_version, set_toolchain
-from lib.sort import sort_libs_by_dependencies
+from lib.dependency import sort_libs_by_dependencies, load_selected_libs
 from lib.installer import install_lib
 from lib.init import init, check_python_version
 
-SCRIPT_VERSION = "v0.8"
+SCRIPT_VERSION = "v0.9"
 
 # check if Python >=3.3.0 is used
 check_python_version()
@@ -31,15 +29,19 @@ verbose = arg_namespace.verbose
 separate_lib64 = arg_namespace.separate_lib64
 disable_shared = arg_namespace.disable_shared
 install_all_libs = arg_namespace.all
+ignore_deps = arg_namespace.ignore_deps
 
 # extract libraries and versions selected for installation
+selected_libs = load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps)
+'''
 selected_libs = []
 if install_all_libs:
-    for cf in glob.glob(config_dir+"/*.json"):
+    for config_file in glob.glob(config_dir+"/*.json"):
-        with open(cf, 'r') as f:
+        data = load_lib_data(config_file)
-            data = json.load(f)
+        #with open(cf, 'r') as f:
-            data['version'] = data['default version']
+            #data = json.load(f)
-            selected_libs.append(data)
+        data['version'] = data['default version']
+        selected_libs.append(data)
 else:
     ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared"]
     for lib_name in args:
@@ -48,10 +50,14 @@ else:
             if install:
                 version = getattr(arg_namespace, lib_name+"_version")
                 config_file = config_dir + "/" + lib_name + ".json"
-                with open(config_file, 'r') as cf:
+                data = load_lib_data(config_file)
-                    data = json.load(cf)
+                data['version'] = data['default version']
-                    data['version'] = version
+                selected_libs.append(data)
-                    selected_libs.append(data)
+                #with open(config_file, 'r') as cf:
+                    #data = json.load(cf)
+                    #data['version'] = version
+                    #selected_libs.append(data)
+'''
 
 # set up install directory name
 compiler_version = get_compiler_version(compiler)