Extends miniwdl to run workflows on SLURM clusters in singularity containers.
This SLURM backend plugin for miniwdl runs WDL task containers by creating a job script that is submitted to a SLURM cluster. In case the job description has a container, singularity will be used as container runtime.
For the latest stable version:
pip install miniwdl-slurm
For the development version:
pip install git+https://github.com/miniwdl-ext/miniwdl-slurm.git
The following miniwdl configuration example can be used to use miniwdl on a SLURM cluster:
[scheduler]
container_backend=slurm_singularity
# task_concurrency defaults to the number of processors on the system.
# since we submit the jobs to SLURM this is not necessary.
# higher numbers means miniwdl has to monitor more processes simultaneously
# which might impact performance.
task_concurrency=200
# This setting allows running tasks to continue, even if one other tasks fails.
# Useful in combination with call caching. Prevents wasting resources by
# cancelling jobs half-way that would probably succeed.
fail_fast = false
[call_cache]
# The following settings create a call cache under the current directory.
# This prevents wasting unnecessary resources on the cluster by rerunning
# jobs that have already succeeded.
put = true
get = true
dir = "$PWD/miniwdl_call_cache"
[task_runtime]
# Setting a 'maxRetries' default allows jobs that fail due to intermittent
# errors on the cluster to be retried.
defaults = {
"maxRetries": 2,
"docker": "ubuntu:20.04"
}
[singularity]
# This plugin wraps the singularity backend. Make sure the settings are
# appropriate for your cluster.
exe = ["singularity"]
# the miniwdl default options contain options to run as a fake root, which
# is not available on most clusters. So the run options do need to be
# overriden.
# --containall: isolates container environment, does not mount home, and
# isolates IPC and PID namespace.
# --no-mount hostfs: Prohibit mounting of any host filesystems unless
# explcitly mounted.
# --network none: Do not allow any network traffic inside and outside
# the container. This is a sane default for reproducible workflows,
# as "the internet" can vary, but if you have explicit download tasks
# you might need to remove this.
run_options = [
"--containall",
"--no-mount", "hostfs",
"--network", "none"
]
# Location of the singularity images (optional). The miniwdl-slurm plugin
# will set it to a directory inside $PWD. This location must be reachable
# for the submit nodes.
image_cache = "$PWD/miniwdl_singularity_cache"
[slurm]
# extra arguments passed to the sbatch command (optional).
extra_args="--partition heavy_users,gpu --comment 'run with miniwdl'"In some cases, you may wish to dynamically modify the SLURM submission arguments. This is possible through rule based arguments. Rules consist of one or more attributes, along with a comparator (lt/<, le/<=, gt/>, ge/>=, eq/==, ne/!=).
These results are evaluated sequentially until one matches, at which point the
value in args is appended to the sbatch command line. This can be used to
modify the partition, the comment, or something else.
[slurm]
# extra arguments passed to the sbatch command (optional).
dynamic_partition = [
{
"memory__ge": 20000000000,
"cpu__gt": 30,
"time_minutes__gt": 7200,
"args": "--partition highmem --comment highmem-long"
},
{
"memory__lt": 2000000000,
"time_minutes__lt": 60,
"args": "--comment short"
},
{
"args": "--comment default"
}
]