Lab tutorial

Note

During ICAPS 2020, we gave an online talk about Lab and Downward Lab (version 6.2). The first half of the presentation shows how to use Lab to run experiments for a solver. You can find the recording here.

Install Lab

Lab requires Python 3.6+ and Linux (e.g., Ubuntu). We recommend installing Lab in a Python virtual environment. This has the advantage that there are no modifications to the system-wide configuration, and that you can create multiple environments with different Lab versions (e.g., for different papers) without conflicts:

# Install required packages, including virtualenv.
sudo apt install python3 python3-venv

# Create a new directory for your experiments.
mkdir experiments-for-my-paper
cd experiments-for-my-paper

# If PYTHONPATH is set, unset it to obtain a clean environment.
unset PYTHONPATH

# Create and activate a Python 3 virtual environment for Lab.
python3 -m venv --prompt my-paper .venv
source .venv/bin/activate

# Install Lab in the virtual environment.
pip install -U pip wheel
pip install lab  # or preferably a specific version with lab==x.y

# Store installed packages and exact versions for reproducibility.
# Ignore pkg-resources package (https://github.com/pypa/pip/issues/4022).
pip freeze | grep -v "pkg-resources" > requirements.txt

Please note that before running an experiment script you need to activate the virtual environment with:

source .venv/bin/activate

We recommend clearing the PYTHONPATH variable before activating the virtual environment.

Run tutorial experiment

The following script shows a simple experiment that runs a naive vertex cover solver on a set of benchmarks.

../examples/vertex-cover/exp.py
#! /usr/bin/env python

"""
Example experiment using a simple vertex cover solver.
"""

import glob
import os
import platform

from downward.reports.absolute import AbsoluteReport
from lab.environments import BaselSlurmEnvironment, LocalEnvironment
from lab.experiment import Experiment
from lab.reports import Attribute


# Create custom report class with suitable info and error attributes.
class BaseReport(AbsoluteReport):
    INFO_ATTRIBUTES = ["time_limit", "memory_limit", "seed"]
    ERROR_ATTRIBUTES = [
        "domain",
        "problem",
        "algorithm",
        "unexplained_errors",
        "error",
        "node",
    ]


NODE = platform.node()
REMOTE = NODE.endswith(".scicore.unibas.ch") or NODE.endswith(".cluster.bc2.ch")
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
BENCHMARKS_DIR = os.path.join(SCRIPT_DIR, "benchmarks")
BHOSLIB_GRAPHS = sorted(glob.glob(os.path.join(BENCHMARKS_DIR, "bhoslib", "*.mis")))
RANDOM_GRAPHS = sorted(glob.glob(os.path.join(BENCHMARKS_DIR, "random", "*.txt")))
ALGORITHMS = ["2approx", "greedy"]
SEED = 2018
TIME_LIMIT = 1800
MEMORY_LIMIT = 2048

if REMOTE:
    ENV = BaselSlurmEnvironment(email="my.name@unibas.ch")
    SUITE = BHOSLIB_GRAPHS + RANDOM_GRAPHS
else:
    ENV = LocalEnvironment(processes=2)
    # Use smaller suite for local tests.
    SUITE = BHOSLIB_GRAPHS[:1] + RANDOM_GRAPHS[:1]
ATTRIBUTES = [
    "cover",
    "cover_size",
    "error",
    "solve_time",
    "solver_exit_code",
    Attribute("solved", absolute=True),
]

# Create a new experiment.
exp = Experiment(environment=ENV)
# Add solver to experiment and make it available to all runs.
exp.add_resource("solver", os.path.join(SCRIPT_DIR, "solver.py"))
# Add custom parser.
exp.add_parser("parser.py")

for algo in ALGORITHMS:
    for task in SUITE:
        run = exp.add_run()
        # Create a symbolic link and an alias. This is optional. We
        # could also use absolute paths in add_command().
        run.add_resource("task", task, symlink=True)
        run.add_command(
            "solve",
            ["{solver}", "--seed", str(SEED), "{task}", algo],
            time_limit=TIME_LIMIT,
            memory_limit=MEMORY_LIMIT,
        )
        # AbsoluteReport needs the following attributes:
        # 'domain', 'problem' and 'algorithm'.
        domain = os.path.basename(os.path.dirname(task))
        task_name = os.path.basename(task)
        run.set_property("domain", domain)
        run.set_property("problem", task_name)
        run.set_property("algorithm", algo)
        # BaseReport needs the following properties:
        # 'time_limit', 'memory_limit', 'seed'.
        run.set_property("time_limit", TIME_LIMIT)
        run.set_property("memory_limit", MEMORY_LIMIT)
        run.set_property("seed", SEED)
        # Every run has to have a unique id in the form of a list.
        run.set_property("id", [algo, domain, task_name])

# Add step that writes experiment files to disk.
exp.add_step("build", exp.build)

# Add step that executes all runs.
exp.add_step("start", exp.start_runs)

# Add step that collects properties from run directories and
# writes them to *-eval/properties.
exp.add_fetcher(name="fetch")

# Make a report.
exp.add_report(BaseReport(attributes=ATTRIBUTES), outfile="report.html")

# Parse the commandline and run the given steps.
exp.run_steps()

You can see the available steps with

./exp.py

Select steps by name or index:

./exp.py build
./exp.py 2
./exp.py 3 4

Here is the parser that the experiment uses:

../examples/vertex-cover/parser.py
#! /usr/bin/env python

"""
Solver example output:

Algorithm: 2approx
Cover: set([1, 3, 5, 6, 7, 8, 9])
Cover size: 7
Solve time: 0.000771s
"""

from lab.parser import Parser


def solved(content, props):
    props["solved"] = int("cover" in props)


def error(content, props):
    if props["solved"]:
        props["error"] = "cover-found"
    else:
        props["error"] = "unsolved"


if __name__ == "__main__":
    parser = Parser()
    parser.add_pattern(
        "node", r"node: (.+)\n", type=str, file="driver.log", required=True
    )
    parser.add_pattern(
        "solver_exit_code", r"solve exit code: (.+)\n", type=int, file="driver.log"
    )
    parser.add_pattern("cover", r"Cover: (\{.*\})", type=str)
    parser.add_pattern("cover_size", r"Cover size: (\d+)\n", type=int)
    parser.add_pattern("solve_time", r"Solve time: (.+)s", type=float)
    parser.add_function(solved)
    parser.add_function(error)
    parser.parse()

Find out how to create your own experiments by browsing the Lab API.