{
    "andes": {
        "description"                 : "704 compute nodes",
        "notes"                       : "Requires RSA SecurID and uses local virtual env",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_endpoint"    : "slurm://andes.olcf.ornl.gov/",
            "filesystem_endpoint"     : "file://andes.olcf.ornl.gov/"
        },
        "forward_tunnel_endpoint"     : "andes.olcf.ornl.gov",
        "default_queue"               : "batch",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["SRUN"],
                                         "SRUN" : {}
                                        },
        "pre_bootstrap_0"             : [
            "source /sw/andes/python/3.7/anaconda-base/etc/profile.d/conda.sh"
        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "virtenv_mode"                : "local",
        "cores_per_node"              : 32
    },

    "andes_gpu": {
        "description"                 : "9 gpu nodes",
        "notes"                       : "Requires RSA SecurID and uses local virtual env",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_endpoint"    : "slurm://andes.olcf.ornl.gov/",
            "filesystem_endpoint"     : "file://andes.olcf.ornl.gov/"
        },
        "forward_tunnel_endpoint"     : "andes.olcf.ornl.gov",
        "default_queue"               : "gpu",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["SRUN"],
                                         "SRUN"  : {}
                                        },
        "pre_bootstrap_0"             : [
            "source /sw/andes/python/3.7/anaconda-base/etc/profile.d/conda.sh"
        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "virtenv_mode"                : "local",
        "cores_per_node"              : 28,
        "gpus_per_node"               : 2,
        "system_architecture"         : {"gpu": "k80"}
    },

    "crusher": {
        "description"                 : "2 cabinets: (1) 128 compute nodes; (2) 64 compute nodes",
        "notes"                       : "Requires RSA SecurID and uses prepared local virtual env",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_endpoint"    : "slurm://crusher.olcf.ornl.gov/",
            "filesystem_endpoint"     : "file://crusher.olcf.ornl.gov/"
        },
        "default_queue"               : "batch",
        "resource_manager"            : "SLURM",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["SRUN"],
                                         "SRUN" : {}
                                        },
        "pre_bootstrap_0"             : [
                                         "module load cray-python"
                                        ],
        "pre_bootstrap_1"             : [
                                        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "virtenv_mode"                : "local",
        "cores_per_node"              : 64,
        "gpus_per_node"               : 8,
        "lfs_path_per_node"           : "/tmp",
        "lfs_size_per_node"           : 0,
        "system_architecture"         : {
                                         "smt"           : 2,
                                       # "options"       : ["nvme"],
                                         "blocked_cores" : [],
                                         "blocked_gpus"  : []
                                        }
    },

    "spock": {
        "description"                 : "3 cabinets: each containing 12 compute nodes",
        "notes"                       : "",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_endpoint"    : "slurm://spock.olcf.ornl.gov/",
            "filesystem_endpoint"     : "file://spock.olcf.ornl.gov/"
        },
        "default_queue"               : "ecp",
        "resource_manager"            : "SLURM",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["SRUN"],
                                         "SRUN" : {}
                                        },
        "pre_bootstrap_0"             : [
                                         "module load cray-python"
                                        ],
        "pre_bootstrap_1"             : [
                                        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "virtenv_mode"                : "local",
        "cores_per_node"              : 64,
        "gpus_per_node"               : 4,
        "lfs_path_per_node"           : "/tmp",
        "lfs_size_per_node"           : 0,
        "system_architecture"         : {
                                         "smt"           : 2,
                                       # "options"       : ["nvme"],
                                         "blocked_cores" : [],
                                         "blocked_gpus"  : []
                                        }
    },

    "summit": {
        "description"                 : "4608 nodes with 2 IBM POWER9 CPUs and 6 NVIDIA Volta V100 GPUs",
        "notes"                       : "Requires RSA SecurID and uses local virtual env",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_hop"         : "fork://localhost/",
            "job_manager_endpoint"    : "lsf://localhost/",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "resource_manager"            : "LSF",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["MPIRUN"],
                                         "MPIRUN": {}
                                        },
        "pre_bootstrap_0"             : [
                                         "module unload xl xalt",
                                         "module load   gcc/9.1.0 libzmq/4.3.3",
                                         "module load   python/3.8-anaconda3"
                                        ],
        "pre_bootstrap_1"             : [
                                         "module load   gcc/9.1.0 libzmq/4.3.3",
                                         "module load   python/3.8-anaconda3",
                                         "ulimit -u 65536"
                                        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "python_dist"                 : "default",
        "virtenv_dist"                : "default",
        "virtenv_mode"                : "create",
        "rp_version"                  : "local",
        "cores_per_node"              : 42,
        "gpus_per_node"               : 6,
        "lfs_path_per_node"           : "/tmp",
        "lfs_size_per_node"           : 0,
        "system_architecture"         : {
                                         "smt"           : 1,
                                         "options"       : ["gpumps", "nvme"],
                                         "blocked_cores" : [],
                                         "blocked_gpus"  : []
                                        }
    },

    "summit_jsrun": {
        "description"                 : "4608 nodes with 2 IBM POWER9 CPUs and 6 NVIDIA Volta V100 GPUs",
        "notes"                       : "Requires RSA SecurID and uses local virtual env",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_hop"         : "fork://localhost/",
            "job_manager_endpoint"    : "lsf://localhost/",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "resource_manager"            : "LSF",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["JSRUN"],
                                         "JSRUN" : {
                                             "pre_exec_cached": [
                                               # "module unload xl xalt"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : [
                                         "module unload xl xalt",
                                         "module load   gcc/9.1.0 libzmq/4.3.3",
                                         "module load   python/3.8-anaconda3"
                                        ],
        "pre_bootstrap_1"             : [
                                         "module load   gcc/9.1.0 libzmq/4.3.3",
                                         "module load   python/3.8-anaconda3",
                                         "ulimit -u 65536"
                                        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "python_dist"                 : "default",
        "virtenv_dist"                : "default",
        "virtenv_mode"                : "create",
        "rp_version"                  : "local",
        "cores_per_node"              : 42,
        "gpus_per_node"               : 6,
        "lfs_path_per_node"           : "/tmp",
        "lfs_size_per_node"           : 0,
        "system_architecture"         : {
                                         "smt"           : 4,
                                         "options"       : ["gpumps", "nvme"],
                                         "blocked_cores" : [],
                                         "blocked_gpus"  : []
                                        }
    },

    "summit_interactive": {
        "description"                 : "4608 nodes with 2 IBM POWER9 CPUs and 6 NVIDIA Volta V100 GPUs",
        "notes"                       : "interactive job https://docs.olcf.ornl.gov/systems/summit_user_guide.html",
        "schemas"                     : ["interactive"],
        "interactive"                 : {
            "job_manager_endpoint"    : "fork://localhost/",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "resource_manager"            : "LSF",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["MPIRUN", "JSRUN"],
                                         "MPIRUN": {},
                                         "JSRUN" : {
                                             "pre_exec_cached": [
                                               # "module unload xl",
                                               # "module unload xalt"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : [
                                         "module unload xl xalt",
                                         "module load   gcc/9.1.0 libzmq/4.3.3",
                                         "module load   python/3.8-anaconda3"
                                        ],
        "pre_bootstrap_1"             : [
                                         "module load   gcc/9.1.0 libzmq/4.3.3",
                                         "module load   python/3.8-anaconda3",
                                         "ulimit -u 65536"
                                        ],
        "default_remote_workdir"      : "$MEMBERWORK/${LSB_PROJECT_NAME,,}",
        "virtenv_mode"                : "local",
        "cores_per_node"              : 42,
        "gpus_per_node"               : 6,
        "lfs_path_per_node"           : "/tmp",
        "lfs_size_per_node"           : 0
    },

    "summit_prte": {
        "description"                 : "4608 nodes with 2 IBM POWER9 CPUs and 6 NVIDIA Volta V100 GPUs",
        "notes"                       : "Requires RSA SecurID and uses local virtual env",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_hop"         : "fork://localhost/",
            "job_manager_endpoint"    : "lsf://localhost/",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "resource_manager"            : "LSF",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["PRTE", "SSH"],
                                         # for sub-agent
                                         "SSH"  : {},
                                         "PRTE" : {
                                             "pre_exec_cached": [
                                                 "module unload xl xalt spectrum-mpi",
                                                 "module load   gcc/9.1.0",
                                                 "module load   python/3.8-anaconda3",
                                                 "module use    /sw/summit/ums/ompix/gcc/9.1.0/modules",
                                               # "module use    /sw/summit/ums/ompix/DEVELOP/gcc/9.1.0/modules",
                                               # "module load   prrte/master"
                                                 "module load   prrte/2.0.2"
                                             ],
                                             "dvm_count": 1
                                         }
                                        },
        "pre_bootstrap_0"             : [
                                         "module unload xl xalt spectrum-mpi",
                                         "module load   gcc/9.1.0",
                                         "module load   python/3.8-anaconda3",
                                         "module use    /sw/summit/ums/ompix/gcc/9.1.0/modules",
                                         "module load   prrte/2.0.2"
                                        ],
        "pre_bootstrap_1"             : [
                                         "module unload xl xalt spectrum-mpi",
                                         "module load   gcc/9.1.0",
                                         "module load   python/3.8-anaconda3",
                                         "module use    /sw/summit/ums/ompix/gcc/9.1.0/modules",
                                         "module load   prrte/2.0.2",
                                         "ulimit -u 65536"
                                        ],
        "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "virtenv_mode"                : "local",
        "cores_per_node"              : 42,
        "gpus_per_node"               : 6,
        "lfs_path_per_node"           : "/tmp",
        "lfs_size_per_node"           : 0,
        "system_architecture"         : {
                                         "smt"           : 4,
                                         "options"       : ["gpumps", "nvme"],
                                         "blocked_cores" : [],
                                         "blocked_gpus"  : []
                                        }
    }
}