1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
|
#!/usr/bin/env python3
# =============================================================================
# ProjT Launcher - Build Statistics Comparison Tool
# =============================================================================
# Compares build statistics between two builds/commits.
# Used by CI to detect performance regressions or improvements.
#
# Usage:
# python cmp-stats.py --explain before_stats/ after_stats/
# =============================================================================
import argparse
import json
import os
from pathlib import Path
from tabulate import tabulate
from typing import Final
def flatten_data(json_data: dict) -> dict:
"""
Extracts and flattens metrics from JSON data.
Handles nested structures by using dot notation.
Args:
json_data (dict): JSON data containing metrics.
Returns:
dict: Flattened metrics with keys as metric names.
"""
flat_metrics = {}
for key, value in json_data.items():
if isinstance(value, (int, float)):
flat_metrics[key] = value
elif isinstance(value, dict):
for subkey, subvalue in value.items():
if isinstance(subvalue, (int, float)):
flat_metrics[f"{key}.{subkey}"] = subvalue
elif isinstance(value, str):
flat_metrics[key] = value
return flat_metrics
def load_all_metrics(path: Path) -> dict:
"""
Loads all stats JSON files from the specified path.
Args:
path (Path): Directory or file containing JSON stats.
Returns:
dict: Dictionary with filenames as keys and metrics as values.
"""
metrics = {}
if path.is_dir():
for json_file in path.glob("**/*.json"):
try:
with json_file.open() as f:
data = json.load(f)
metrics[str(json_file.relative_to(path))] = flatten_data(data)
except (json.JSONDecodeError, IOError) as e:
print(f"Warning: Could not load {json_file}: {e}")
elif path.is_file():
try:
with path.open() as f:
metrics[path.name] = flatten_data(json.load(f))
except (json.JSONDecodeError, IOError) as e:
print(f"Warning: Could not load {path}: {e}")
return metrics
METRIC_EXPLANATIONS: Final[str] = """
### Metric Explanations
| Metric | Description |
|--------|-------------|
| build.time | Total build time in seconds |
| build.memory | Peak memory usage in MB |
| compile.units | Number of compilation units |
| link.time | Linking time in seconds |
| test.passed | Number of tests passed |
| test.failed | Number of tests failed |
| binary.size | Final binary size in bytes |
"""
def compare_metrics(before: dict, after: dict) -> tuple:
"""
Compare metrics between two builds.
Returns:
tuple: (changed_metrics, unchanged_metrics)
"""
changed = []
unchanged = []
# Get all metric keys from both
all_keys = sorted(set(list(before.keys()) + list(after.keys())))
for key in all_keys:
before_val = before.get(key)
after_val = after.get(key)
if before_val is None or after_val is None:
continue
if isinstance(before_val, (int, float)) and isinstance(after_val, (int, float)):
if before_val == after_val:
unchanged.append({
"metric": key,
"value": before_val
})
else:
diff = after_val - before_val
pct_change = (diff / before_val * 100) if before_val != 0 else float('inf')
changed.append({
"metric": key,
"before": before_val,
"after": after_val,
"diff": diff,
"pct_change": pct_change
})
return changed, unchanged
def format_results(changed: list, unchanged: list, explain: bool) -> str:
"""Format comparison results as markdown."""
result = ""
if unchanged:
result += "## Unchanged Values\n\n"
result += tabulate(
[[m["metric"], m["value"]] for m in unchanged],
headers=["Metric", "Value"],
tablefmt="github"
)
result += "\n\n"
if changed:
result += "## Changed Values\n\n"
result += tabulate(
[[
m["metric"],
f"{m['before']:.4f}" if isinstance(m['before'], float) else m['before'],
f"{m['after']:.4f}" if isinstance(m['after'], float) else m['after'],
f"{m['diff']:+.4f}" if isinstance(m['diff'], float) else m['diff'],
f"{m['pct_change']:+.2f}%" if isinstance(m['pct_change'], float) else "N/A"
] for m in changed],
headers=["Metric", "Before", "After", "Diff", "Change %"],
tablefmt="github"
)
result += "\n\n"
if explain:
result += METRIC_EXPLANATIONS
if not changed and not unchanged:
result = "No comparable metrics found.\n"
return result
def main():
parser = argparse.ArgumentParser(
description="Build statistics comparison for ProjT Launcher"
)
parser.add_argument(
"--explain",
action="store_true",
help="Include metric explanations"
)
parser.add_argument(
"before",
help="File or directory containing baseline stats"
)
parser.add_argument(
"after",
help="File or directory containing comparison stats"
)
args = parser.parse_args()
before_path = Path(args.before)
after_path = Path(args.after)
if not before_path.exists():
print(f"Error: {before_path} does not exist")
return 1
if not after_path.exists():
print(f"Error: {after_path} does not exist")
return 1
before_metrics = load_all_metrics(before_path)
after_metrics = load_all_metrics(after_path)
# Merge all metrics from all files
merged_before = {}
merged_after = {}
for metrics in before_metrics.values():
merged_before.update(metrics)
for metrics in after_metrics.values():
merged_after.update(metrics)
changed, unchanged = compare_metrics(merged_before, merged_after)
output = format_results(changed, unchanged, args.explain)
print(output)
return 0
if __name__ == "__main__":
exit(main())
|
