{
    "expanse": {
        "description"                 : "(https://www.sdsc.edu/support/user_guides/expanse.html).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription.",
        "schemas"                     : ["local", "ssh"],
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://expanse.sdsc.xsede.org",
            "filesystem_endpoint"     : "file://expanse.sdsc.xsede.org"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://expanse.sdsc.xsede.org",
            "filesystem_endpoint"     : "sftp://expanse.sdsc.xsede.org"
        },
        "default_remote_workdir"      :  "/expanse/lustre/scratch/$USER/temp_project",
        "default_queue"               : "compute",
        "resource_manager"            : "SLURM",
        "agent_config"                : "default",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "pre_bootstrap_0"             : ["module load slurm"],
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "launch_methods"              : {
                                         "order" : ["MPIRUN"],
                                         "MPIRUN": {"pre_exec_cached": ["module load slurm",
                                                                        "module load gcc",
                                                                        "module load openmpi"]}
                                        },
        "lfs_path_per_node"           : "${LOCAL}",
        "lfs_size_per_node"           : 1000000,
        "mem_per_node"                : 256000,
        "cores_per_node"              : 128
      # "gpus_per_node"               : 0,
      # "system_architecture"         : {"gpu": "v100"}
    },

    "stampede2_ssh": {
        "description"                 : "The XSEDE/ACCESS 'Stampede' cluster at TACC (https://www.tacc.utexas.edu/stampede/).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription or the pilot will fail.",
        "schemas"                     : ["local", "gsissh", "ssh"],
        "mandatory_args"              : ["project"],
        "gsissh"                      :
        {
            "job_manager_endpoint"    : "slurm+gsissh://stampede2.tacc.utexas.edu:2222/",
            "filesystem_endpoint"     : "gsisftp://stampede2.tacc.utexas.edu:2222/"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "sftp://stampede2.tacc.utexas.edu/"
        },
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "file://stampede2.tacc.utexas.edu/"
        },
        "default_queue"               : "normal",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["SSH", "IBRUN"],
                                         "SSH"  : {
                                             "pre_exec_cached": [
                                                 "module load TACC"
                                             ]
                                         },
                                         "IBRUN": {
                                             "pre_exec_cached": [
                                                 "module load TACC"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             :["module load TACC",
                                        "module load intel/18.0.0",
                                        "module load python3/3.7.0"
                                        ],
        "default_remote_workdir"      : "$WORK2",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "task_pre_exec"               : []
    },

    "stampede2_mpirun": {
        "description"                 : "The XSEDE/ACCESS 'Stampede' cluster at TACC (https://www.tacc.utexas.edu/stampede/).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription or the pilot will fail.",
        "schemas"                     : ["local", "gsissh", "ssh"],
        "mandatory_args"              : ["project"],
        "gsissh"                      :
        {
            "job_manager_endpoint"    : "slurm+gsissh://stampede2.tacc.utexas.edu:2222/",
            "filesystem_endpoint"     : "gsisftp://stampede2.tacc.utexas.edu:2222/"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "sftp://stampede2.tacc.utexas.edu/"
        },
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "file://stampede2.tacc.utexas.edu/"
        },
        "default_queue"               : "normal",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["FORK", "MPIRUN"],
                                         "FORK"  : {},
                                         "MPIRUN": {
                                             "pre_exec_cached": [
                                                 "module load TACC"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : ["module load TACC",
                                         "module load intel/18.0.0",
                                         "module load python3/3.7.0"
                                        ],
        "default_remote_workdir"      : "$WORK2",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "task_pre_exec"               : ["module restore"]
    },

    "stampede2_ibrun_repex": {
        "description"                 : "The XSEDE/ACCESS 'Stampede' cluster at TACC (https://www.tacc.utexas.edu/stampede/).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription or the pilot will fail.",
        "schemas"                     : ["local", "gsissh", "ssh"],
        "mandatory_args"              : ["project"],
        "gsissh"                      :
        {
            "job_manager_endpoint"    : "slurm+gsissh://stampede2.tacc.utexas.edu:2222/",
            "filesystem_endpoint"     : "gsisftp://stampede2.tacc.utexas.edu:2222/"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "sftp://stampede2.tacc.utexas.edu/"
        },
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "file://stampede2.tacc.utexas.edu/"
        },
        "cores_per_node"              : 68,
        "default_queue"               : "normal",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["FORK", "IBRUN"],
                                         "FORK"  : {},
                                         "IBRUN" : {
                                             "pre_exec_cached": [
                                                 "module load TACC"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : ["module load TACC",
                                         "module load intel/18.0.0",
                                         "module load python3/3.7.0"
                                        ],
        "default_remote_workdir"      : "$WORK2",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "system_architecture"         : {
                                       # "blocked_cores" : [64, 65, 66, 67],
                                       # "blocked_gpus"  : [0]
                                        }
    },

    "stampede2_ibrun": {
        "description"                 : "The XSEDE/ACCESS 'Stampede' cluster at TACC (https://www.tacc.utexas.edu/stampede/).",
        "notes"                       : "Always set the ``project`` attribute in the ComputePilotDescription or the pilot will fail.",
        "schemas"                     : ["local", "gsissh", "ssh"],
        "mandatory_args"              : ["project"],
        "gsissh"                      :
        {
            "job_manager_endpoint"    : "slurm+gsissh://stampede2.tacc.utexas.edu:2222/",
            "filesystem_endpoint"     : "gsisftp://stampede2.tacc.utexas.edu:2222/"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "sftp://stampede2.tacc.utexas.edu/"
        },
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "file://stampede2.tacc.utexas.edu/"
        },
        "cores_per_node"              : 68,
        "default_queue"               : "normal",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["IBRUN"],
                                         "IBRUN": {
                                             "pre_exec_cached": [
                                                 "module load TACC"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : [
                                         "module load TACC",
                                         "module load intel/18.0.0",
                                         "module load python3/3.7.0"
                                        ],
        "default_remote_workdir"      : "$WORK",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "task_pre_exec"               : ["module restore"]
    },

    "stampede2_srun": {
        "description"                 : "The XSEDE/ACCESS 'Stampede' cluster at TACC (https://www.tacc.utexas.edu/stampede/).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription or the pilot will fail.",
        "schemas"                     : ["local", "gsissh", "ssh"],
        "mandatory_args"              : ["project"],
        "gsissh"                      :
        {
            "job_manager_endpoint"    : "slurm+gsissh://stampede2.tacc.utexas.edu:2222/",
            "filesystem_endpoint"     : "gsisftp://stampede2.tacc.utexas.edu:2222/"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "sftp://stampede2.tacc.utexas.edu/"
        },
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://stampede2.tacc.utexas.edu/",
            "filesystem_endpoint"     : "file://stampede2.tacc.utexas.edu/"
        },
        "default_queue"               : "normal",
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order": ["SRUN"],
                                         "SRUN" : {
                                             "pre_exec_cached": [
                                                 "module load TACC",
                                                 "module load intel/18.0.0",
                                                 "module load python3/3.7.0"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : ["module load TACC",
                                         "module load intel/18.0.0",
                                         "module load python3/3.7.0"
                                        ],
        "default_remote_workdir"      : "$WORK2",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "task_pre_exec"               : []
    },

    "comet": {
        "description"                 : "The retired Comet HPC resource at SDSC 'HPC for the 99%%' (https://www.sdsc.edu/support/user_guides/comet.html).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription or the pilot will fail.",
        "schemas"                     : ["ssh"],
        "mandatory_args"              : ["project"],
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://comet.sdsc.xsede.org/",
            "filesystem_endpoint"     : "sftp://comet.sdsc.xsede.org/"
        },
        "default_queue"               : "compute",
        "lfs_path_per_node"           : "/scratch/$USER/$SLURM_JOB_ID",
        "lfs_size_per_node"           : 176105,
        "resource_manager"            : "SLURM",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["MPIRUN"],
                                         "MPIRUN": {
                                            "pre_exec_cached": [
                                               "module load gcc",
                                               "module load openmpi"
                                            ]
                                         }
                                        },
        "pre_bootstrap_0"             : ["module load python"],
        "default_remote_workdir"      : "/oasis/scratch/comet/$USER/temp_project",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "default",
        "cores_per_node"              : 24,
        "gpus_per_node"               : 8,
        "system_architecture"         : {"gpu": "k80"}
    },

    "bridges2": {
        "description"                 : "The XSEDE/ACCESS 'Bridges2' cluster at PSC (https://portal.xsede.org/psc-bridges/).",
        "notes"                       : "Always set the ``project`` attribute in the PilotDescription.",
        "schemas"                     : ["local", "interactive", "gsissh", "ssh"],
      # "mandatory_args"              : [],
        "local"                       :
        {
            "job_manager_endpoint"    : "slurm://bridges2.psc.xsede.org/",
            "filesystem_endpoint"     : "file://bridges2.psc.xsede.org/"
        },
        "interactive"                 :
        {
            "job_manager_endpoint"    : "fork://localhost/",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "gsissh"                      :
        {
            "job_manager_endpoint"    : "slurm+gsissh://bridges2.psc.xsede.org:2222/",
            "filesystem_endpoint"     : "gsisftp://bridges2.psc.xsede.org:2222/"
        },
        "ssh"                         :
        {
            "job_manager_endpoint"    : "slurm+ssh://bridges2.psc.xsede.org/",
            "filesystem_endpoint"     : "sftp://bridges2.psc.xsede.org/"
        },
        "default_queue"               : "RM",
        "resource_manager"            : "SLURM",
        "lfs_path_per_node"           : "${LOCAL}",
        "lfs_size_per_node"           : 2516582,
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "launch_methods"              : {
                                         "order" : ["MPIRUN"],
                                         "MPIRUN": {
                                             "pre_exec_cached": [
                                                 "module load gcc",
                                                 "module load openmpi"
                                             ]
                                         }
                                        },
        "pre_bootstrap_0"             : ["module load anaconda3"],
        "default_remote_workdir"      : "$PROJECT",
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "python_dist"                 : "anaconda",
        "task_pre_exec"               : [],
        "gpus_per_node"               : 8,
        "system_architecture"         : {"gpu": "v100"}
    }
}