Setting the file. One moment. def load_graph_json
— line 203
This file
- Number
- 32.3
- Position
- 3 of 7
- Type
- Python
- Size
- 18 KB
- Lines
- 544
scripts/community_maintainers.py
Python·544 lines·18 KB
sys
14from collections import Counter, defaultdict
15from pathlib import Path
16from typing import Iterable
17
18
19def parse_args() -> argparse.Namespace:
20 parser = argparse.ArgumentParser(
21 description="Compute maintainers for a file's community over time."
22 )
23 parser.add_argument(
24 "--data-dir",
25 default="ownership-map-out",
26 help="Directory containing graph outputs",
27 )
28 parser.add_argument(
29 "--repo",
30 default=None,
31 help="Git repo path (required if commits.jsonl is missing)",
32 )
33 parser.add_argument(
34 "--file",
35 default=None,
36 help="File path (exact or substring) to locate community",
37 )
38 parser.add_argument(
39 "--community-id",
40 type=int,
41 default=None,
42 help="Community id to analyze",
43 )
44 parser.add_argument(
45 "--since",
46 default=None,
47 help="Filter commits since date (ISO or 'YYYY-MM-DD')",
48 )
49 parser.add_argument(
50 "--until",
51 default=None,
52 help="Filter commits until date (ISO or 'YYYY-MM-DD')",
53 )
54 parser.add_argument(
55 "--identity",
56 choices=("author", "committer"),
57 default="author",
58 help="Identity to attribute touches to",
59 )
60 parser.add_argument(
61 "--date-field",
62 choices=("author", "committer"),
63 default="author",
64 help="Date field to use for bucketing",
65 )
66 parser.add_argument(
67 "--include-merges",
68 action="store_true",
69 help="Include merge commits (excluded by default)",
70 )
71 parser.add_argument(
72 "--top",
73 type=int,
74 default=5,
75 help="Top maintainers per month",
76 )
77 parser.add_argument(
78 "--bucket",
79 choices=("month", "quarter"),
80 default="month",
81 help="Time bucket for grouping",
82 )
83 parser.add_argument(
84 "--touch-mode",
85 choices=("commit", "file"),
86 default="commit",
87 help="Count one touch per commit or one per file touched",
88 )
89 parser.add_argument(
90 "--window-days",
91 type=int,
92 default=0,
93 help="Use a rolling window of N days ending each month (0 = calendar month only)",
94 )
95 parser.add_argument(
96 "--weight",
97 choices=("touches", "recency"),
98 default="touches",
99 help="Weight touches by recency using exponential decay",
100 )
101 parser.add_argument(
102 "--half-life-days",
103 type=float,
104 default=180.0,
105 help="Half-life days for recency weighting",
106 )
107 parser.add_argument(
108 "--min-share",
109 type=float,
110 default=0.0,
111 help="Minimum share within a month to include a maintainer",
112 )
113 parser.add_argument(
114 "--ignore-author-regex",
115 default=None,
116 help="Regex to skip authors by name or email (e.g., '(bot|dependabot)')",
117 )
118 parser.add_argument(
119 "--min-touches",
120 type=int,
121 default=1,
122 help="Minimum touches per month to include a maintainer",
123 )
124 return parser.parse_args()
125
126
127def parse_date(value: str) -> dt.datetime:
128 try:
129 parsed = dt.datetime.fromisoformat(value)
130 except ValueError:
131 parsed = dt.datetime.fromisoformat(value + "T00:00:00")
132 if parsed.tzinfo is None:
133 parsed = parsed.replace(tzinfo=dt.timezone.utc)
134 return parsed
135
136
137def month_key(timestamp: dt.datetime) -> str:
138 return timestamp.strftime("%Y-%m")
139
140
141def quarter_key(timestamp: dt.datetime) -> str:
142 quarter = (timestamp.month - 1) // 3 + 1
143 return f"{timestamp.year}-Q{quarter}"
144
145
146def month_end(timestamp: dt.datetime) -> dt.datetime:
147 year = timestamp.year
148 month = timestamp.month
149 if month == 12:
150 next_month = dt.datetime(year + 1, 1, 1, tzinfo=dt.timezone.utc)
151 else:
152 next_month = dt.datetime(year, month + 1, 1, tzinfo=dt.timezone.utc)
153 return next_month - dt.timedelta(seconds=1)
154
155
156def quarter_start(timestamp: dt.datetime) -> dt.datetime:
157 quarter = (timestamp.month - 1) // 3
158 start_month = quarter * 3 + 1
159 return dt.datetime(timestamp.year, start_month, 1, tzinfo=dt.timezone.utc)
160
161
162def quarter_end(timestamp: dt.datetime) -> dt.datetime:
163 start = quarter_start(timestamp)
164 end_month = start.month + 2
165 end_year = start.year
166 if end_month > 12:
167 end_month -= 12
168 end_year += 1
169 end_anchor = dt.datetime(end_year, end_month, 1, tzinfo=dt.timezone.utc)
170 return month_end(end_anchor)
171
172
173def add_months(timestamp: dt.datetime, months: int) -> dt.datetime:
174 year = timestamp.year + (timestamp.month - 1 + months) // 12
175 month = (timestamp.month - 1 + months) % 12 + 1
176 return dt.datetime(year, month, 1, tzinfo=dt.timezone.utc)
177
178
179def recency_weight(age_days: float, half_life_days: float) -> float:
180 if half_life_days <= 0:
181 return 1.0
182 return math.exp(-age_days / half_life_days)
183
184
185def read_csv(path: Path) -> Iterable[dict[str, str]]:
186 with path.open("r", encoding="utf-8") as handle:
187 reader = csv.DictReader(handle)
188 yield from reader
189
190
191def load_people(data_dir: Path) -> dict[str, dict[str, str]]:
192 people_path = data_dir / "people.csv"
193 people = {}
194 for row in read_csv(people_path):
195 people[row.get("person_id", "")] = {
196 "name": row.get("name", ""),
197 "email": row.get("email", ""),
198 "primary_tz_offset": row.get("primary_tz_offset", ""),
199 }
200 return people
201
202
203def load_graph_json(data_dir: Path) -> dict[str, object] | None:
204 cochange_path = data_dir / "cochange.graph.json"
205 ownership_path = data_dir / "ownership.graph.json"
206 if cochange_path.exists():
207 return json.loads(cochange_path.read_text(encoding="utf-8"))
208 if ownership_path.exists():
209 return json.loads(ownership_path.read_text(encoding="utf-8"))
210 return None
211
212
213def find_file_node(nodes: list[dict[str, object]], query: str) -> dict[str, object]:
214 exact = [node for node in nodes if node.get("id") == query]
215 if exact:
216 return exact[0]
217 contains = [node for node in nodes if query in str(node.get("id", ""))]
218 if len(contains) == 1:
219 return contains[0]
220 if not contains:
221 raise ValueError(f"File not found in graph: {query}")
222 candidates = ", ".join(str(node.get("id")) for node in contains[:10])
223 raise ValueError(f"Multiple matches for file {query}: {candidates}")
224
225
226def load_community_files(
227 data_dir: Path, file_query: str | None, community_id: int | None
228) -> tuple[int, list[str]]:
229 graph = load_graph_json(data_dir)
230 if graph:
231 nodes = graph.get("nodes", [])
232 if file_query:
233 node = find_file_node(nodes, file_query)
234 community_id = int(node.get("community_id", -1))
235 if community_id is None:
236 raise ValueError("Provide --file or --community-id")
237 files = [node.get("id") for node in nodes if node.get("community_id") == community_id]
238 files = [entry for entry in files if entry]
239 if not files:
240 raise ValueError(f"No files found for community {community_id}")
241 return community_id, files
242
243 communities_path = data_dir / "communities.json"
244 if not communities_path.exists():
245 raise FileNotFoundError("Missing graph json and communities.json")
246 communities = json.loads(communities_path.read_text(encoding="utf-8"))
247 if file_query:
248 for entry in communities:
249 files = entry.get("files", [])
250 if any(file_query == f or file_query in f for f in files):
251 return int(entry.get("id", -1)), list(files)
252 raise ValueError("File not found in communities.json (list may be truncated)")
253 if community_id is None:
254 raise ValueError("Provide --file or --community-id")
255 for entry in communities:
256 if int(entry.get("id", -1)) == community_id:
257 return community_id, list(entry.get("files", []))
258 raise ValueError(f"Community id not found: {community_id}")
259
260
261def iter_commits_from_json(
262 commits_path: Path,
263 since: dt.datetime | None,
264 until: dt.datetime | None,
265 date_field: str,
266) -> Iterable[dict[str, object]]:
267 with commits_path.open("r", encoding="utf-8") as handle:
268 for line in handle:
269 entry = json.loads(line)
270 author_date = entry.get("author_date") or entry.get("date")
271 committer_date = entry.get("committer_date")
272 if author_date:
273 author_dt = parse_date(author_date)
274 else:
275 author_dt = None
276 if committer_date:
277 committer_dt = parse_date(committer_date)
278 else:
279 committer_dt = None
280 if date_field == "committer":
281 commit_date = committer_dt or author_dt
282 else:
283 commit_date = author_dt or committer_dt
284 if commit_date is None:
285 continue
286 if since and commit_date < since:
287 continue
288 if until and commit_date > until:
289 continue
290 yield {
291 "hash": entry.get("hash", ""),
292 "parents": entry.get("parents", []),
293 "is_merge": entry.get("is_merge", False),
294 "author_name": entry.get("author_name", ""),
295 "author_email": entry.get("author_email", ""),
296 "author_date": author_date,
297 "committer_name": entry.get("committer_name", ""),
298 "committer_email": entry.get("committer_email", ""),
299 "committer_date": committer_date,
300 "files": entry.get("files", []),
301 }
302
303
304def iter_commits_from_git(
305 repo: str, since: str | None, until: str | None, include_merges: bool
306) -> Iterable[dict[str, object]]:
307 cmd = [
308 "git",
309 "-C",
310 repo,
311 "log",
312 "--name-only",
313 "--no-renames",
314 "--date=iso-strict",
315 "--format=---%n%H%n%P%n%an%n%ae%n%ad%n%cn%n%ce%n%cd",
316 ]
317 if not include_merges:
318 cmd.append("--no-merges")
319 if since:
320 cmd.extend(["--since", since])
321 if until:
322 cmd.extend(["--until", until])
323
324 proc = subprocess.Popen(
325 cmd,
326 stdout=subprocess.PIPE,
327 stderr=subprocess.PIPE,
328 text=True,
329 )
330 assert proc.stdout is not None
331
332 block: list[str] = []
333 for line in proc.stdout:
334 line = line.rstrip("\n")
335 if line == "---":
336 if block:
337 yield from parse_git_block(block)
338 block = []
339 else:
340 block.append(line)
341 if block:
342 yield from parse_git_block(block)
343
344 stderr = proc.stderr.read() if proc.stderr else ""
345 exit_code = proc.wait()
346 if exit_code != 0:
347 raise RuntimeError(stderr.strip() or "git log failed")
348
349
350def parse_git_block(block: list[str]) -> Iterable[dict[str, object]]:
351 if len(block) < 8:
352 return []
353 commit_hash = block[0]
354 parents = [entry for entry in block[1].split(" ") if entry]
355 author_name = block[2]
356 author_email = block[3]
357 author_date = block[4]
358 committer_name = block[5]
359 committer_email = block[6]
360 committer_date = block[7]
361 files = [line for line in block[8:] if line]
362 return [
363 {
364 "hash": commit_hash,
365 "parents": parents,
366 "is_merge": len(parents) > 1,
367 "author_name": author_name,
368 "author_email": author_email,
369 "author_date": author_date,
370 "committer_name": committer_name,
371 "committer_email": committer_email,
372 "committer_date": committer_date,
373 "files": files,
374 }
375 ]
376
377
378def main() -> int:
379 args = parse_args()
380 data_dir = Path(args.data_dir)
381 if not data_dir.exists():
382 print(f"Data directory not found: {data_dir}", file=sys.stderr)
383 return 1
384
385 since = parse_date(args.since) if args.since else None
386 until = parse_date(args.until) if args.until else None
387
388 try:
389 community_id, community_files = load_community_files(data_dir, args.file, args.community_id)
390 except (ValueError, FileNotFoundError) as exc:
391 print(str(exc), file=sys.stderr)
392 return 2
393
394 people = load_people(data_dir)
395
396 ignore_re = re.compile(args.ignore_author_regex) if args.ignore_author_regex else None
397
398 commits_path = data_dir / "commits.jsonl"
399 if commits_path.exists():
400 commit_iter = iter_commits_from_json(commits_path, since, until, args.date_field)
401 else:
402 if not args.repo:
403 print("--repo is required when commits.jsonl is missing", file=sys.stderr)
404 return 2
405 commit_iter = iter_commits_from_git(args.repo, args.since, args.until, args.include_merges)
406
407 commit_rows: list[tuple[dt.datetime, str, int, str, str]] = []
408 for commit in commit_iter:
409 if commit.get("is_merge") and not args.include_merges:
410 continue
411 files = commit.get("files", [])
412 in_community = sum(1 for path in files if path in community_files)
413 if in_community == 0:
414 continue
415 identity_name = commit.get(f"{args.identity}_name", "")
416 identity_email = commit.get(f"{args.identity}_email", "")
417 date_value = commit.get(f"{args.date_field}_date")
418 if not date_value:
419 print(
420 "Missing committer fields in commits.jsonl. Re-run build or pass --repo.",
421 file=sys.stderr,
422 )
423 return 2
424 commit_date = parse_date(date_value)
425 person_id = identity_email or identity_name
426 if ignore_re and ignore_re.search(identity_name or ""):
427 continue
428 if ignore_re and ignore_re.search(identity_email or ""):
429 continue
430 touches = 1 if args.touch_mode == "commit" else in_community
431 commit_rows.append((commit_date, person_id, touches, identity_name, identity_email))
432 if person_id not in people:
433 people[person_id] = {
434 "name": identity_name,
435 "email": identity_email,
436 "primary_tz_offset": "",
437 }
438
439 if not commit_rows:
440 print("No commits touching community files for the selected window.", file=sys.stderr)
441 return 0
442
443 commit_rows.sort(key=lambda row: row[0])
444 period_counts: dict[str, Counter[str]] = defaultdict(Counter)
445 period_totals: dict[str, float] = defaultdict(float)
446
447 min_date = commit_rows[0][0]
448 max_date = commit_rows[-1][0]
449 if args.bucket == "quarter":
450 period_cursor = quarter_start(min_date)
451 period_end_anchor = quarter_start(max_date)
452 step_months = 3
453 key_func = quarter_key
454 end_func = quarter_end
455 else:
456 period_cursor = dt.datetime(min_date.year, min_date.month, 1, tzinfo=dt.timezone.utc)
457 period_end_anchor = dt.datetime(max_date.year, max_date.month, 1, tzinfo=dt.timezone.utc)
458 step_months = 1
459 key_func = month_key
460 end_func = month_end
461
462 while period_cursor <= period_end_anchor:
463 bucket_end = end_func(period_cursor)
464 bucket_key = key_func(bucket_end)
465 if args.window_days > 0:
466 window_start = bucket_end - dt.timedelta(days=args.window_days)
467
468 def in_bucket(commit_date: dt.datetime) -> bool:
469 return window_start <= commit_date <= bucket_end
470 else:
471 if args.bucket == "quarter":
472 bucket_start = quarter_start(period_cursor)
473
474 def in_bucket(commit_date: dt.datetime) -> bool:
475 return bucket_start <= commit_date <= bucket_end
476 else:
477
478 def in_bucket(commit_date: dt.datetime) -> bool:
479 return (
480 commit_date.year == bucket_end.year
481 and commit_date.month == bucket_end.month
482 )
483
484 for commit_date, person_id, touches, _name, _email in commit_rows:
485 if not in_bucket(commit_date):
486 continue
487 weight = 1.0
488 if args.weight == "recency":
489 age_days = (bucket_end - commit_date).total_seconds() / 86400.0
490 weight = recency_weight(age_days, args.half_life_days)
491 contribution = touches * weight
492 period_counts[bucket_key][person_id] += contribution
493 period_totals[bucket_key] += contribution
494
495 period_cursor = add_months(period_cursor, step_months)
496
497 writer = csv.writer(sys.stdout)
498 writer.writerow(
499 [
500 "period",
501 "rank",
502 "name",
503 "email",
504 "primary_tz_offset",
505 "community_touches",
506 "touch_share",
507 ]
508 )
509
510 for period in sorted(period_counts.keys()):
511 total = period_totals[period]
512 ranked = sorted(period_counts[period].items(), key=lambda item: item[1], reverse=True)
513 rank = 0
514 for person_id, touches in ranked:
515 if touches < args.min_touches:
516 continue
517 share = touches / total if total else 0.0
518 if share < args.min_share:
519 continue
520 rank += 1
521 if rank > args.top:
522 break
523 person = people.get(person_id, {})
524 if args.weight == "recency":
525 touches_value = f"{touches:.4f}"
526 else:
527 touches_value = f"{touches:.0f}"
528 writer.writerow(
529 [
530 period,
531 rank,
532 person.get("name", ""),
533 person.get("email", person_id),
534 person.get("primary_tz_offset", ""),
535 touches_value,
536 f"{share:.4f}",
537 ]
538 )
539
540 return 0
541
542
543if __name__ == "__main__":
544 raise SystemExit(main())