Source code for

import logging

from downward.reports.absolute import AbsoluteReport
from lab import reports

[docs] class ComparativeReport(AbsoluteReport): """Compare pairs of algorithms.""" def __init__(self, algorithm_pairs, **kwargs): """ See :py:class:`AbsoluteReport <downward.reports.absolute.AbsoluteReport>` for inherited parameters. *algorithm_pairs* is the list of algorithm pairs you want to compare. All columns in the report will be arranged such that the compared algorithms appear next to each other. After the two columns containing absolute values for the compared algorithms, a third column ("Diff") is added showing the difference between the two values. Algorithms may appear in multiple comparisons. Algorithms not mentioned in *algorithm_pairs* are not included in the report. If you want to compare algorithms A and B, instead of a pair ``('A', 'B')`` you may pass a triple ``('A', 'B', 'A vs. B')``. The third entry of the triple will be used as the name of the corresponding "Diff" column. For example, if the properties file contains algorithms A, B, C and D and *algorithm_pairs* is ``[('A', 'B', 'Diff BA'), ('A', 'C')]`` the resulting columns will be A, B, Diff BA (contains B - A), A, C , Diff (contains C - A). Example: >>> from downward.experiment import FastDownwardExperiment >>> exp = FastDownwardExperiment() >>> algorithm_pairs = [("default-lmcut", "issue123-lmcut", "Diff lmcut")] >>> exp.add_report(ComparativeReport(algorithm_pairs, attributes=["coverage"])) Example output: +----------+---------------+----------------+------------+ | coverage | default-lmcut | issue123-lmcut | Diff lmcut | +==========+===============+================+============+ | depot | 15 | 17 | 2 | +----------+---------------+----------------+------------+ | gripper | 7 | 6 | -1 | +----------+---------------+----------------+------------+ """ if "filter_algorithm" in kwargs: logging.critical( 'ComparativeReport doesn\'t support "filter_algorithm". ' 'Use "algorithm_pairs" to select and order algorithms.' ) if algorithm_pairs: algos = set() for tup in algorithm_pairs: for algo in tup[:2]: algos.add(algo) kwargs["filter_algorithm"] = algos AbsoluteReport.__init__(self, **kwargs) self._algorithm_pairs = algorithm_pairs def _get_empty_table(self, attribute=None, title=None, columns=None): table = AbsoluteReport._get_empty_table( self, attribute=attribute, title=title, columns=columns ) summary_functions = [sum, reports.arithmetic_mean] if title == "Summary": summary_functions = [] diff_module = DiffColumnsModule(self._algorithm_pairs, summary_functions) table.dynamic_data_modules.append(diff_module) return table
class DiffColumnsModule(reports.DynamicDataModule): """ Add multiple columns, each comparing the values of two algorithms. """ def __init__(self, algorithm_pairs, summary_functions): """ See :py:class:`.ComparativeReport` for how to choose the compared algorithms. *summary_functions* is a list of functions that will be calculated for all entries in the diff columns. Example:: algorithm_pairs = [ ('default-lmcut', 'issue123-lmcut', 'Diff (lmcut)'), ('default-ff', 'issue123-ff', 'Diff (ff)')] summary_functions = [sum, reports.arithmetic_mean] diff_module = DiffColumnsModule(algorithm_pairs, summary_functions) table.dynamic_data_modules.append(diff_module) """ self.header_names = [] diff_column_names = set() for tup in algorithm_pairs: diff_name = "Diff" if len(tup) == 3: diff_name = tup[2] # diff_name is printed in the column header and does not have to be unique. # To identify the column we thus calculate a uniqe name. uniq_count = 0 col_name = None while col_name is None or col_name in diff_column_names: uniq_count += 1 col_name = f"diff_column_{uniq_count}" diff_column_names.add(col_name) self.header_names.append(((tup[0], tup[1]), diff_name, col_name)) self.summary_functions = summary_functions @staticmethod def _get_function_name(function): return reports.function_name(function) + " of diffs" def collect(self, table, cells): """ Add cells for the specified diff columns and dynamically compute their values from the respective data columns. If one of the values is None, set the difference to the string '-'. Calculate the summary functions over all values were both columns have a value. Also add an empty header for a dummy column after every diff column. """ for col_names, diff_col_header, diff_col_name in self.header_names: non_none_values = [] cells[table.header_row][diff_col_name] = diff_col_header for row_name in table.row_names: values = [table[row_name].get(col_name, None) for col_name in col_names] try: diff = float(values[1]) - float(values[0]) except (ValueError, TypeError, OverflowError): diff = None if diff is not None: non_none_values.append(diff) cells[row_name][diff_col_name] = diff for func in self.summary_functions: func_name = self._get_function_name(func) cells[func_name][table.header_column] = func_name.capitalize() cells[func_name][diff_col_name] = ( func(non_none_values) if non_none_values else None ) return cells def format(self, table, formatted_cells): """ Format all columns added by this module. Diff values are green if they are better in the second column, red if they are worse and grey if there is no difference. "Better" and "worse" are with respect to the min_wins information of the table for each row. Do not format dummy columns and summary functions. """ for _, _, diff_col_name in self.header_names: for row_name in table.row_names: formatted_value = formatted_cells[row_name].get(diff_col_name) min_wins = table.get_min_wins(row_name) try: value = float(formatted_value) except (ValueError, TypeError): value = "-" if value == 0 or value == "-" or min_wins is None: color = "grey" elif (value < 0 and min_wins) or (value > 0 and not min_wins): color = "green" else: color = "red" # Add space in front of value to right-justify it. formatted_value = f" {{{value}|color:{color}}}" formatted_cells[row_name][diff_col_name] = formatted_value def modify_printable_column_order(self, table, column_order): """ Reorder algorithms in the order defined by algorithm_pairs. Hide all other columns. """ new_column_order = [table.header_column] for col_names, _, diff_col_name in self.header_names: if len(new_column_order) >= 4: new_column_order.append("DiffDummy") for col_name in col_names: new_column_order.append(col_name) new_column_order.append(diff_col_name) return new_column_order def modify_printable_row_order(self, table, row_order): """ Append lines for all summary functions that are not already used to the row order. """ for func in self.summary_functions: func_name = self._get_function_name(func) if func_name not in row_order: row_order.append(func_name) return row_order