Skip to content

CLI reference

mmm_eval.cli

CLI module for MMM evaluation.

Modules

evaluate

Classes
Functions
main(config_path: str, input_data_path: str, test_names: tuple[str, ...], framework: str, output_path: str, verbose: bool)

Evaluate MMM frameworks using the unified API.

Source code in mmm_eval/cli/evaluate.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
@click.command()
@click.option(
    "--framework",
    type=click.Choice(list(ADAPTER_REGISTRY.keys())),
    required=True,
    help="Open source MMM framework to evaluate",
)
@click.option(
    "--input-data-path",
    type=str,
    required=True,
    help="Path to input data file. Supported formats: CSV, Parquet",
)
@click.option(
    "--output-path",
    type=str,
    required=True,
    help="Directory to save evaluation results as a CSV file with name 'mmm_eval_<framework>_<timestamp>.csv'",
)
@click.option(
    "--config-path",
    type=str,
    required=True,
    help="Path to framework-specific JSON config file",
)
@click.option(
    "--test-names",
    type=click.Choice(ValidationTestNames.all_tests_as_str()),
    multiple=True,
    default=tuple(ValidationTestNames.all_tests_as_str()),
    help=(
        "Test names to run. Can specify multiple tests as space-separated values "
        "(e.g. --test-names accuracy cross_validation) or by repeating the flag "
        "(e.g. --test-names accuracy --test-names cross_validation). "
        "Defaults to all tests if not specified."
    ),
)
@click.option(
    "--verbose",
    "-v",
    is_flag=True,
    help="Enable verbose logging",
)
def main(
    config_path: str,
    input_data_path: str,
    test_names: tuple[str, ...],
    framework: str,
    output_path: str,
    verbose: bool,
):
    """Evaluate MMM frameworks using the unified API."""
    # logging
    log_level = logging.DEBUG if verbose else logging.INFO
    logging.basicConfig(level=log_level)

    logger.info("Loading config...")
    config = get_config(framework, config_path)

    logger.info("Loading input data...")
    data = DataLoader(input_data_path).load()

    # Run evaluation
    logger.info(f"Running evaluation suite for {framework} framework...")
    results = run_evaluation(framework, data, config, test_names)

    # Save results
    if results.empty:
        logger.warning("Results df empty, nothing to save.")
    else:
        save_results(results, framework, output_path)
Modules