Source code for snippets.monitoring

"""Monitoring helpers for logging failures, output streams, and Git state."""

__all__ = [
    "append_git_state_snapshot",
    "git_state_context",
    "mail_traceback_wrapper",
    "pipe_output_and_git_state_log",
    "pipe_output_to_logfile",
]

from collections.abc import Sequence
from contextlib import contextmanager
from datetime import datetime, timezone
from email.message import EmailMessage
from pathlib import Path
import inspect
import smtplib
import subprocess
import sys
import traceback
from typing import Literal


[docs] def mail_traceback_wrapper( to_address: str, subject_tag: str = None, from_address: str = None ): """Return a decorator that emails tracebacks through localhost SMTP. Args: to_address: Recipient email address. subject_tag: Optional prefix added to the error subject. from_address: Optional sender address. """ # CREDIT: FBruzzesi, Nathan Davis https://stackoverflow.com/a/27500036 def decorate(f): def applicator(*args, **kwargs): try: return f(*args, **kwargs) except Exception as err: msg = EmailMessage() msg["To"] = to_address subject = str(err).strip().replace("\n", "; ") if subject_tag is not None and subject_tag != "": subject = f"[{subject_tag.strip()}] {subject}" msg["Subject"] = subject if from_address is not None: msg["From"] = from_address.replace(" ", "_") msg.set_content(traceback.format_exc()) print(str(err).strip().replace("\n", "; "), msg) with smtplib.SMTP("localhost") as s: s.send_message(msg) raise return applicator return decorate
[docs] @contextmanager def pipe_output_to_logfile(path: Path | str, mode: Literal["a", "w"]): """Redirect stdout and stderr to a logfile inside the context.""" with open(path, mode) as lf: orig_pipes = sys.stderr, sys.stdout sys.stderr = sys.stdout = lf try: yield finally: sys.stderr, sys.stdout = orig_pipes
def _run_git(args: Sequence[str], cwd: Path) -> tuple[bool, str, str]: try: proc = subprocess.run( ["git", *args], cwd=cwd, check=False, capture_output=True, text=True, encoding="utf-8", errors="replace", ) except OSError as err: return False, "", str(err) if proc.returncode != 0: error = proc.stderr.strip() or proc.stdout.strip() if error == "": error = f"git {' '.join(args)} failed with exit code {proc.returncode}" return False, "", error return True, proc.stdout, "" def _iter_parent_dirs(start: Path, max_parent_levels: int) -> list[Path]: levels = max(0, max_parent_levels) dirs = [start] current = start for _ in range(levels): parent = current.parent if parent == current: break dirs.append(parent) current = parent return dirs def _infer_caller_file(max_parent_levels: int) -> Path | None: module_file = Path(__file__).resolve() search_roots = [ path for path in _iter_parent_dirs(Path.cwd().resolve(), max_parent_levels) if path.parent != path ] stack = inspect.stack() try: for frame_info in stack: filename = frame_info.filename if filename.startswith("<"): continue try: candidate = Path(filename).resolve() except OSError: continue if candidate == module_file: continue if not candidate.exists(): continue if any( candidate == root or root in candidate.parents for root in search_roots ): return candidate finally: del stack return None def _dedupe_paths(paths: Sequence[Path]) -> list[Path]: unique_paths = [] seen = set() for path in paths: key = str(path) if key in seen: continue seen.add(key) unique_paths.append(path) return unique_paths def _default_anchor_paths(max_parent_levels: int) -> list[Path]: anchors = [] caller_file = _infer_caller_file(max_parent_levels=max_parent_levels) if caller_file is not None: anchors.append(caller_file) for parent_dir in _iter_parent_dirs(Path.cwd().resolve(), max_parent_levels): snakefile = parent_dir / "Snakefile" if snakefile.exists(): anchors.append(snakefile.resolve()) return _dedupe_paths(anchors) def _pathspec_candidates(anchor_path: Path, repo_root: Path) -> list[str]: pathspecs = [] if anchor_path.is_absolute(): try: rel = anchor_path.resolve().relative_to(repo_root) except (OSError, ValueError): return [] pathspecs.append(rel.as_posix()) return pathspecs pathspecs.append(anchor_path.as_posix()) try: rel_from_cwd = (Path.cwd() / anchor_path).resolve().relative_to(repo_root) except (OSError, ValueError): rel_from_cwd = None if rel_from_cwd is not None: rel_from_cwd_str = rel_from_cwd.as_posix() if rel_from_cwd_str not in pathspecs: pathspecs.append(rel_from_cwd_str) return pathspecs def _has_tracked_anchor(repo_root: Path, anchors: Sequence[Path]) -> bool: for anchor_path in anchors: for pathspec in _pathspec_candidates(anchor_path, repo_root): ok, _, _ = _run_git( ["ls-files", "--error-unmatch", "--", pathspec], cwd=repo_root ) if ok: return True return False def _discover_repo_root( repo_dir: Path | str | None, anchor_paths: Sequence[Path | str] | None, max_parent_levels: int, ) -> tuple[Path | None, list[str], list[Path]]: warnings = [] if anchor_paths is None: anchors = _default_anchor_paths(max_parent_levels=max_parent_levels) else: anchors = _dedupe_paths([Path(anchor).expanduser() for anchor in anchor_paths]) if repo_dir is not None: candidate_dirs = [Path(repo_dir).expanduser().resolve()] else: candidate_dirs = _iter_parent_dirs( start=Path.cwd().resolve(), max_parent_levels=max_parent_levels ) for candidate_dir in candidate_dirs: ok, output, error = _run_git( ["rev-parse", "--show-toplevel"], cwd=candidate_dir ) if not ok: if repo_dir is not None: warnings.append( f"Could not resolve git repo from {candidate_dir}: {error}" ) continue repo_root = Path(output.strip()).resolve() if anchors and not _has_tracked_anchor(repo_root=repo_root, anchors=anchors): warnings.append( "Anchor check failed for candidate " f"{repo_root}; none of the anchors appear to be tracked there." ) continue return repo_root, warnings, anchors if repo_dir is None: warnings.append( "Could not find a git repository in current directory or configured " f"parents (max_parent_levels={max(0, max_parent_levels)})." ) return None, warnings, anchors def _render_or_empty(text: str) -> str: if text == "": return "(empty)\n" if text.endswith("\n"): return text return f"{text}\n" def _append_reflog_activity_warning( log_path: Path | str, *, run_start: datetime, repo_dir: Path | str | None, anchor_paths: Sequence[Path | str] | None, max_parent_levels: int, encoding: str, ) -> None: """Append a warning if reflog shows HEAD activity since `run_start`.""" repo_root, _, _ = _discover_repo_root( repo_dir=repo_dir, anchor_paths=anchor_paths, max_parent_levels=max_parent_levels, ) if repo_root is None: return run_start_str = run_start.isoformat(timespec="seconds") reflog_ok, reflog_output, _ = _run_git( ["reflog", "--since", run_start_str, "--date=iso-strict"], cwd=repo_root, ) if not reflog_ok: return reflog_lines = [line for line in reflog_output.splitlines() if line.strip() != ""] if len(reflog_lines) == 0: return log_file = Path(log_path).expanduser() log_file.parent.mkdir(parents=True, exist_ok=True) with log_file.open("a", encoding=encoding) as handle: handle.write( "WARNING: Git reflog recorded " f"{len(reflog_lines)} action(s) since run start {run_start_str}. " "Repository state may have changed during execution.\n\n" )
[docs] def append_git_state_snapshot( log_path: Path | str, *, section: Literal["START", "END", "POINT"] = "POINT", repo_dir: Path | str | None = None, anchor_paths: Sequence[Path | str] | None = None, max_parent_levels: int = 3, max_diff_bytes: int = 262_144, encoding: str = "utf-8", ) -> None: """Append git state metadata to a plain-text logfile. Args: log_path (Path | str): Log file target. section (Literal["START", "END", "POINT"], optional): Snapshot label. repo_dir (Path | str | None, optional): Directory to start git discovery from. anchor_paths (Sequence[Path | str] | None, optional): Paths expected to be tracked in the target repository. max_parent_levels (int, optional): Number of parent directories to inspect when `repo_dir` is omitted. max_diff_bytes (int, optional): If staged + unstaged patch size is larger than this threshold, the function logs `--stat` output instead of full patch output. encoding (str, optional): File encoding used for writing and size accounting. """ timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") log_file = Path(log_path).expanduser() log_file.parent.mkdir(parents=True, exist_ok=True) repo_root, warnings, anchors = _discover_repo_root( repo_dir=repo_dir, anchor_paths=anchor_paths, max_parent_levels=max_parent_levels, ) with log_file.open("a", encoding=encoding) as handle: handle.write( f"===== GIT STATE SNAPSHOT [{section}] {timestamp} =====\n" ) if anchors: anchor_text = ", ".join(str(anchor) for anchor in anchors) handle.write(f"anchor_paths: {anchor_text}\n") else: handle.write("anchor_paths: (none)\n") for warning in warnings: handle.write(f"WARNING: {warning}\n") if repo_root is None: handle.write( "WARNING: Snapshot skipped because no valid git repository " "could be resolved.\n" ) handle.write("===== END GIT STATE SNAPSHOT =====\n\n") return head_ok, head_output, head_error = _run_git( ["rev-parse", "HEAD"], cwd=repo_root ) branch_ok, branch_output, branch_error = _run_git( ["branch", "--show-current"], cwd=repo_root ) staged_ok, staged_patch, staged_error = _run_git( ["diff", "--cached", "--no-ext-diff", "--"], cwd=repo_root ) unstaged_ok, unstaged_patch, unstaged_error = _run_git( ["diff", "--no-ext-diff", "--"], cwd=repo_root ) if not head_ok: handle.write(f"WARNING: Failed to read HEAD commit: {head_error}\n") if not branch_ok: handle.write(f"WARNING: Failed to read branch name: {branch_error}\n") if not staged_ok: handle.write(f"WARNING: Failed to read staged diff: {staged_error}\n") if not unstaged_ok: handle.write(f"WARNING: Failed to read unstaged diff: {unstaged_error}\n") combined_patch = f"{staged_patch}{unstaged_patch}" patch_bytes = len(combined_patch.encode(encoding, errors="replace")) diff_mode = "patch" staged_output = staged_patch unstaged_output = unstaged_patch if patch_bytes > max_diff_bytes: diff_mode = "stat_fallback" handle.write( "WARNING: Combined patch size exceeded threshold " f"({patch_bytes} > {max_diff_bytes}); writing --stat output.\n" ) staged_stat_ok, staged_stat, staged_stat_error = _run_git( ["diff", "--cached", "--no-ext-diff", "--stat", "--"], cwd=repo_root ) unstaged_stat_ok, unstaged_stat, unstaged_stat_error = _run_git( ["diff", "--no-ext-diff", "--stat", "--"], cwd=repo_root ) if staged_stat_ok: staged_output = staged_stat else: handle.write( "WARNING: Failed to read staged --stat diff: " f"{staged_stat_error}\n" ) if unstaged_stat_ok: unstaged_output = unstaged_stat else: handle.write( "WARNING: Failed to read unstaged --stat diff: " f"{unstaged_stat_error}\n" ) branch_name = branch_output.strip() if branch_ok else "" branch_value = branch_name if branch_name != "" else "(detached-or-unknown)" head_value = head_output.strip() if head_ok else "(unknown)" dirty = ( (staged_patch != "" if staged_ok else False) or (unstaged_patch != "" if unstaged_ok else False) ) handle.write(f"repo_root: {repo_root}\n") handle.write(f"head: {head_value}\n") handle.write(f"branch: {branch_value}\n") handle.write(f"dirty: {dirty}\n") handle.write( f"diff_mode: {diff_mode}; max_diff_bytes: {max_diff_bytes}; " f"measured_patch_bytes: {patch_bytes}\n" ) handle.write("--- STAGED DIFF START ---\n") handle.write(_render_or_empty(staged_output)) handle.write("--- STAGED DIFF END ---\n") handle.write("--- UNSTAGED DIFF START ---\n") handle.write(_render_or_empty(unstaged_output)) handle.write("--- UNSTAGED DIFF END ---\n") handle.write("===== END GIT STATE SNAPSHOT =====\n\n")
[docs] @contextmanager def git_state_context( log_path: Path | str, *, snapshot_timing: Literal["start", "end", "both"] = "both", repo_dir: Path | str | None = None, anchor_paths: Sequence[Path | str] | None = None, max_parent_levels: int = 3, max_diff_bytes: int = 262_144, encoding: str = "utf-8", ): """Context manager that writes START/END git state snapshots. Example: with git_state_context("run.log", snapshot_timing="both"): run_workflow() """ if snapshot_timing not in {"start", "end", "both"}: raise ValueError( "snapshot_timing must be one of 'start', 'end', or 'both', got " f"{snapshot_timing!r}." ) run_start = datetime.now(timezone.utc) if snapshot_timing in {"start", "both"}: append_git_state_snapshot( log_path=log_path, section="START", repo_dir=repo_dir, anchor_paths=anchor_paths, max_parent_levels=max_parent_levels, max_diff_bytes=max_diff_bytes, encoding=encoding, ) try: yield finally: if snapshot_timing in {"end", "both"}: _append_reflog_activity_warning( log_path=log_path, run_start=run_start, repo_dir=repo_dir, anchor_paths=anchor_paths, max_parent_levels=max_parent_levels, encoding=encoding, ) append_git_state_snapshot( log_path=log_path, section="END", repo_dir=repo_dir, anchor_paths=anchor_paths, max_parent_levels=max_parent_levels, max_diff_bytes=max_diff_bytes, encoding=encoding, )
[docs] @contextmanager def pipe_output_and_git_state_log( log_path: Path | str, mode: Literal["a", "w"] = "a", *, snapshot_timing: Literal["start", "end", "both"] = "both", repo_dir: Path | str | None = None, anchor_paths: Sequence[Path | str] | None = None, max_parent_levels: int = 3, max_diff_bytes: int = 262_144, encoding: str = "utf-8", ): """Pipe stdout/stderr and log git state snapshots in one context. Example: with pipe_output_and_git_state_log("run.log", mode="w"): print("workflow started") run_workflow() """ if mode not in {"a", "w"}: raise ValueError(f"mode must be 'a' or 'w', got {mode!r}.") log_file = Path(log_path).expanduser() log_file.parent.mkdir(parents=True, exist_ok=True) if mode == "w": log_file.write_text("", encoding=encoding) with git_state_context( log_path=log_file, snapshot_timing=snapshot_timing, repo_dir=repo_dir, anchor_paths=anchor_paths, max_parent_levels=max_parent_levels, max_diff_bytes=max_diff_bytes, encoding=encoding, ): with pipe_output_to_logfile(log_file, "a"): yield