v1.0 - cleanup

README.md

@@ -3,13 +3,13 @@
 This Python tool has been developed to automate the installation of libraries frequently used by HPC applications.
 
 ## Basic Usage
-To install the library `<lib>` using compiler `<compiler>` and MPI `<mpi>` make sure to have a configuration file `<lib>.json` in the `config` directory and invoke the tool as follows:
+To install the library `<lib>` and all it's dependencies (if there are any) using compiler `<compiler>` and MPI `<mpi>`, make sure to have a configuration file `<lib>.json` in the `config` directory and invoke the tool as follows:
 ```
 ./libinstaller --compiler <compiler> --mpi <mpi> --<lib>
 ```
 To install all libraries with build recipes in `config`:
 ```
-./libinstaller --compiler <compiler> --mpi <mpi> --<lib>
+./libinstaller --compiler <compiler> --mpi <mpi> --all
 ```
 For further supported options and their default values, please have a look at `./libinstaller -h`:
 ```
@@ -29,8 +29,9 @@ options:
   --separate-lib64     Do not create symbolic links of files from lib64 in lib
   --disable-shared     Disable building of shared libraries
   --all                Install all libraries with config file in config/
+  --ignore-deps        Do not add dependencies of libraries to the install process
 ```
-Per library configuration available in `config`, the corresponding `--<lib>` and `--<lib>-version`options are added automatically, eg:
+For each library configuration file available in `config`, the corresponding `--<lib>` and `--<lib>-version` options are added to the parser automatically, eg:
 ```
 --hdf5                          Enable build of hdf5
 --hdf5-version HDF5_VERSION     Set hdf5 version [1.13.2]
@@ -58,8 +59,9 @@ The `config` directory contains various json files describing how to obtain and
 On startup, the installer tool will read the configuration files in `config` and add coreesponding options to the argument parser. In case `--<lib>-version` is not set, the default version from the json file is used.
 
 ### dependencies
-Dependencies are used to determine the best installation order of selected librarie to satisfy all dependencies. Note that it is assumed that you supply needed dependencies via `LD_LIBRARY_PATH` in case they are not part of the current install.
-Also note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH`
+Dependencies are used to determine the best installation order of the selected libraries to satisfy all dependencies.
+When the installer encounters a library to install, all dependencies are also added to the installation process if not already selected. In case `--ignore-deps` is set, the dependencies need to be activated manually when invoking libinstaller or be present in `LD_LIBRARY_PATH`.
+Note that `$PREFIX/lib` is automatically added to `LD_LIBRARY_PATH`
 
 ### download
 Download command. This is being executed in a subshell. Make sure that it works in the shell used to start libinstaller.

lib/dependency.py

@@ -0,0 +1,102 @@
+import os
+import glob
+from lib.io import load_lib_data
+
+def topological_sort(source):
+    pending = [(name, set(deps)) for name, deps in source]
+    emitted = []
+    while pending:
+        next_pending = []
+        next_emitted = []
+        for entry in pending:
+            name, deps = entry
+            deps.difference_update(set((name,)), emitted)
+            if deps:
+                next_pending.append(entry)
+            else:
+                yield name
+                emitted.append(name)
+                next_emitted.append(name)
+        if not next_emitted:
+            raise ValueError("cyclic dependancy detected: %s %r" % (name, (next_pending,)))
+        pending = next_pending
+        emitted = next_emitted
+    return emitted
+
+
+def sort_libs_by_dependencies(selected_libs):
+    # no need for sorting in case of only one lib
+    if len(selected_libs) < 2:
+        return selected_libs
+
+    lib_names = []
+    for lib in selected_libs:
+        lib_names.append(lib['name'])
+
+    deplist = []
+    for lib in selected_libs:
+        name = lib['name']
+        # only sort after dependencies that are actually present to avoid cyclic dependencies
+        deps = []
+        for dep in lib['dependencies'].split(','):
+            if dep in lib_names:
+                deps.append(dep)
+        dependencies = set(deps)
+        if dependencies == {''}:
+            dependencies = {}
+        deplist.append([name, dependencies])
+    sorted_deplist = topological_sort(deplist)
+
+    sorted = []
+    for entry in sorted_deplist:
+        for lib in selected_libs:
+            if lib['name'] == entry:
+                sorted.append(lib)
+    return sorted
+
+
+def load_dependencies(library, selected_libs, config_dir):
+    lib_names = []
+    for lib in selected_libs:
+        lib_names.append(lib['name'])
+
+    dependencies = library['dependencies'].split(',')
+    for dep in dependencies:
+        if len(dep) > 1:
+            if dep not in lib_names:
+                config_file = config_dir + "/" + dep + ".json"
+                if os.path.exists(config_file):
+                    dep_data = load_lib_data(config_file)
+                    dep_data['version'] = dep_data['default version']
+                    selected_libs.append(dep_data)
+                else:
+                    print("Warning - library " + lib['name'] + " depends on " + dep + " for which no configration file was found. Presence in LD_LIBRARY_PATH is assumed.")
+
+
+# load library data from names in argparse result
+def load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps):
+    selected_libs = []
+    if install_all_libs:
+        for config_file in glob.glob(config_dir+"/*.json"):
+            data = load_lib_data(config_file)
+            data['version'] = data['default version']
+            selected_libs.append(data)
+    else:
+        ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared", "ignore_deps"]
+        for lib_name in args:
+            if lib_name not in ignore_names and "version" not in lib_name:
+                install = getattr(arg_namespace, lib_name)
+                if install:
+                    version = getattr(arg_namespace, lib_name + "_version")
+                    lib_name = lib_name.replace('_','-')
+                    config_file = config_dir + "/" + lib_name + ".json"
+                    data = load_lib_data(config_file)
+                    data['version'] = version
+                    selected_libs.append(data)
+
+    if not ignore_deps:
+        # also add all dependencies to the install list
+        for lib in selected_libs:
+            load_dependencies(lib, selected_libs, config_dir)
+
+    return selected_libs

lib/init.py

@@ -29,6 +29,7 @@ def init():
     config_parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true')
     config_parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true')
     config_parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true')
+    config_parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true')
 
     parser.add_argument('--config', help='Path to config directory [$pwd/config]', default=os.getcwd()+"/config")
     parser.add_argument('--prefix', help='Path where install directory should be generated [$pwd]', default=os.getcwd())
@@ -42,6 +43,7 @@ def init():
     parser.add_argument('--separate-lib64', help='Do not create symbolic links of files from lib64 in lib', action='store_true')
     parser.add_argument('--disable-shared', help='Disable building of shared libraries', action='store_true')
     parser.add_argument('--all', help='Install all libraries with config file in config/', action='store_true')
+    parser.add_argument('--ignore-deps', help='Do not add dependencies of libraries to the install process', action='store_true')
 
     # run config parser and search config/*.json to add a build and version argument for it to the full parser
     config_dir = config_parser.parse_known_args()[0].config

lib/io.py

@@ -0,0 +1,8 @@
+import json
+
+
+# load library data from json file
+def load_lib_data(lib_path):
+    with open (lib_path, 'r') as config_file:
+        data = json.load(config_file)
+        return data

lib/toolchain.py

@@ -1,15 +1,22 @@
 import os
+import sys
 from lib.shell import get_from_command
 
 
 def get_compiler_version(compiler):
     if compiler == "gnu":
         cc = "gcc"
+        compstr = get_from_command([cc, "-dumpversion"]).strip()
     elif compiler == "intel":
         cc = "icc"
+        compstr = get_from_command([cc, "-dumpversion"]).strip()
     elif compiler == "aocc":
         cc = "clang"
-    compstr = get_from_command([cc, "-dumpversion"]).strip()
+        compstr = get_from_command([cc, "-dumpversion"]).strip()
+    elif compiler == "nec":
+        cc = "ncc"
+        rawstr = get_from_command([cc, "--version"])
+        compstr = rawstr.split()[2]
     return compstr
 
 
@@ -29,6 +36,9 @@ def get_mpi_version(mpi):
     elif mpi == "mpich":
         rawstr = get_from_command(["mpirun", "--version"])
         mpistr = rawstr.split()[4]
+    elif mpi == "necmpi":
+        rawstr = get_from_command(["mpirun", "--version"])
+        mpistr = rawstr.split()[4]
     return mpistr
 
 
@@ -42,6 +52,13 @@ def set_toolchain(compiler, mpi):
             cc = "mpicc"
             cxx = "mpicxx"
             fc = "mpif90"
+        elif mpi == "necmpi":
+            cc = "mpincc -vh"
+            cxx = "mpinc++ -vh"
+            fc = "mpinfort -vh"
+            os.environ["NMPI_CC_H"] = "gcc"
+            os.environ["NMPI_CXX_H"] = "g++"
+            os.environ["NMPI_FC_H"] = "gfortran"
     elif compiler == "aocc":
         if mpi == "hpcx" or mpi == "openmpi":
             cc = "mpicc"
@@ -53,9 +70,16 @@ def set_toolchain(compiler, mpi):
             os.environ["OMPI_MPIF90"] = "flang"
             os.environ["OMPI_MPIF77"] = "flang"
         elif (mpi == "intelmpi"):
-            cc = "\"mpicc -cc=clang\""
-            cxx = "\"mpicxx -cxx=clang++\""
-            fc = "\"mpif90 -fc=flang\""
+            cc = "mpicc -cc=clang"
+            cxx = "mpicxx -cxx=clang++"
+            fc = "mpif90 -fc=flang"
+        elif mpi == "necmpi":
+            cc = "mpincc -vh"
+            cxx = "mpic++ -vh"
+            fc = "mpinfort -vh"
+            os.environ["NMPI_CC_H"] = "clang"
+            os.environ["NMPI_CXX_H"] = "clang++"
+            os.environ["NMPI_FC_H"] = "flang"
     elif compiler == "intel":
         if mpi == "hpcx" or mpi == "openmpi":
             cc = "mpicc"
@@ -70,6 +94,18 @@ def set_toolchain(compiler, mpi):
             cc = "mpiicc"
             cxx = "mpiicpc"
             fc = "mpiifort"
+        elif mpi == "necmpi":
+            cc = "mpincc -vh"
+            cxx = "mpic++ -vh"
+            fc = "mpinfort -vh"
+            os.environ["NMPI_CC_H"] = "icc"
+            os.environ["NMPI_CXX_H"] = "icpc"
+            os.environ["NMPI_FC_H"] = "ifort"
+    elif compiler == "nec":
+        # only NEC MPI
+        cc = "mpincc"
+        cxx = "mpinc++"
+        fc = "mpinfort"
 
     # set environment variables
     os.environ["CC"] = cc

libinstaller

@@ -1,16 +1,14 @@
 #!/usr/bin/env python3
 import shutil
-import json
 import os
 import glob
-from lib.ui import progressbar, bordered, underlined, print_welcome
-from lib.shell import get_from_command
+from lib.ui import bordered, print_welcome
 from lib.toolchain import get_compiler_version, get_mpi_version, set_toolchain
-from lib.sort import sort_libs_by_dependencies
+from lib.dependency import sort_libs_by_dependencies, load_selected_libs
 from lib.installer import install_lib
 from lib.init import init, check_python_version
 
-SCRIPT_VERSION = "v0.8"
+SCRIPT_VERSION = "v1.0"
 
 # check if Python >=3.3.0 is used
 check_python_version()
@@ -31,27 +29,10 @@ verbose = arg_namespace.verbose
 separate_lib64 = arg_namespace.separate_lib64
 disable_shared = arg_namespace.disable_shared
 install_all_libs = arg_namespace.all
+ignore_deps = arg_namespace.ignore_deps
 
 # extract libraries and versions selected for installation
-selected_libs = []
-if install_all_libs:
-    for cf in glob.glob(config_dir+"/*.json"):
-        with open(cf, 'r') as f:
-            data = json.load(f)
-            data['version'] = data['default version']
-            selected_libs.append(data)
-else:
-    ignore_names = ["config", "mpi", "compiler", "prefix", "src", "work", "keep_work", "threads", "verbose", "version", "disable_shared"]
-    for lib_name in args:
-        if lib_name not in ignore_names and "version" not in lib_name:
-            install = getattr(arg_namespace, lib_name)
-            if install:
-                version = getattr(arg_namespace, lib_name+"_version")
-                config_file = config_dir + "/" + lib_name + ".json"
-                with open(config_file, 'r') as cf:
-                    data = json.load(cf)
-                    data['version'] = version
-                    selected_libs.append(data)
+selected_libs = load_selected_libs(config_dir, arg_namespace, args, install_all_libs, ignore_deps)
 
 # set up install directory name
 compiler_version = get_compiler_version(compiler)