Run other planners

The script below shows how to run the FF planner on a number of classical planning benchmarks. You can see the available steps with

./ff.py

Select steps by name or index:

./ff.py build
./ff.py 2
./ff.py 3 4

You can use this file as a basis for your own experiments. For Fast Downward experiments, we recommend taking a look at the downward.tutorial.

../examples/ff/ff.py

#! /usr/bin/env python

"""
Example experiment for the FF planner
(http://fai.cs.uni-saarland.de/hoffmann/ff.html).
"""

import os
import platform

from ff_parser import FFParser

from downward import suites
from downward.reports.absolute import AbsoluteReport
from lab.environments import BaselSlurmEnvironment, LocalEnvironment
from lab.experiment import Experiment
from lab.reports import Attribute, geometric_mean


# Create custom report class with suitable info and error attributes.
class BaseReport(AbsoluteReport):
    INFO_ATTRIBUTES = ["time_limit", "memory_limit"]
    ERROR_ATTRIBUTES = [
        "domain",
        "problem",
        "algorithm",
        "unexplained_errors",
        "error",
        "node",
    ]


NODE = platform.node()
REMOTE = NODE.endswith(".scicore.unibas.ch") or NODE.endswith(".cluster.bc2.ch")
BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
if REMOTE:
    ENV = BaselSlurmEnvironment(email="my.name@unibas.ch")
else:
    ENV = LocalEnvironment(processes=2)
SUITE = ["grid", "gripper:prob01.pddl", "miconic:s1-0.pddl", "mystery:prob07.pddl"]
ATTRIBUTES = [
    "error",
    "plan",
    "times",
    Attribute("coverage", absolute=True, min_wins=False, scale="linear"),
    Attribute("evaluations", function=geometric_mean),
    Attribute("trivially_unsolvable", min_wins=False),
]
TIME_LIMIT = 1800
MEMORY_LIMIT = 2048


# Create a new experiment.
exp = Experiment(environment=ENV)
# Add custom parser for FF.
exp.add_parser(FFParser())

for task in suites.build_suite(BENCHMARKS_DIR, SUITE):
    run = exp.add_run()
    # Create symbolic links and aliases. This is optional. We
    # could also use absolute paths in add_command().
    run.add_resource("domain", task.domain_file, symlink=True)
    run.add_resource("problem", task.problem_file, symlink=True)
    # 'ff' binary has to be on the PATH.
    # We could also use exp.add_resource().
    run.add_command(
        "run-planner",
        ["ff", "-o", "{domain}", "-f", "{problem}"],
        time_limit=TIME_LIMIT,
        memory_limit=MEMORY_LIMIT,
    )
    # AbsoluteReport needs the following properties:
    # 'domain', 'problem', 'algorithm', 'coverage'.
    run.set_property("domain", task.domain)
    run.set_property("problem", task.problem)
    run.set_property("algorithm", "ff")
    # BaseReport needs the following properties:
    # 'time_limit', 'memory_limit'.
    run.set_property("time_limit", TIME_LIMIT)
    run.set_property("memory_limit", MEMORY_LIMIT)
    # Every run has to have a unique id in the form of a list.
    # The algorithm name is only really needed when there are
    # multiple algorithms.
    run.set_property("id", ["ff", task.domain, task.problem])

# Add step that writes experiment files to disk.
exp.add_step("build", exp.build)

# Add step that executes all runs.
exp.add_step("start", exp.start_runs)

# Add step that parses log output into "properties" files.
exp.add_step("parse", exp.parse)

# Add step that collects properties from run directories and
# writes them to *-eval/properties.
exp.add_fetcher(name="fetch")

# Make a report.
exp.add_report(BaseReport(attributes=ATTRIBUTES), outfile="report.html")

# Parse the commandline and run the specified steps.
exp.run_steps()

Here is a simple parser for FF:

../examples/ff/ff_parser.py

"""
FF example output:

[...]

ff: found legal plan as follows

step    0: UP F0 F1
        1: BOARD F1 P0
        2: DOWN F1 F0
        3: DEPART F0 P0


time spent:    0.00 seconds instantiating 4 easy, 0 hard action templates
               0.00 seconds reachability analysis, yielding 4 facts and 4 actions
               0.00 seconds creating final representation with 4 relevant facts
               0.00 seconds building connectivity graph
               0.00 seconds searching, evaluating 5 states, to a max depth of 2
               0.00 seconds total time
"""

import re

from lab.parser import Parser


def error(content, props):
    if props["planner_exit_code"] == 0:
        props["error"] = "plan-found"
    else:
        props["error"] = "unsolvable-or-error"


def coverage(content, props):
    props["coverage"] = int(props["planner_exit_code"] == 0)


def get_plan(content, props):
    # All patterns are parsed before functions are called.
    if props.get("evaluations") is not None:
        props["plan"] = re.findall(r"^(?:step)?\s*\d+: (.+)$", content, re.M)


def get_times(content, props):
    props["times"] = re.findall(r"(\d+\.\d+) seconds", content)


def trivially_unsolvable(content, props):
    props["trivially_unsolvable"] = int(
        "ff: goal can be simplified to FALSE. No plan will solve it" in content
    )


class FFParser(Parser):
    def __init__(self):
        super().__init__()
        self.add_pattern(
            "node", r"node: (.+)\n", type=str, file="driver.log", required=True
        )
        self.add_pattern(
            "planner_exit_code",
            r"run-planner exit code: (.+)\n",
            type=int,
            file="driver.log",
        )
        self.add_pattern("evaluations", r"evaluating (\d+) states")
        self.add_function(error)
        self.add_function(coverage)
        self.add_function(get_plan)
        self.add_function(get_times)
        self.add_function(trivially_unsolvable)