init

thirdeye/scripts/test_m18.py

@@ -0,0 +1,206 @@
+"""
+Test Milestone 18: Jira Signal Agent.
+Seeds real signals and raises actual Jira tickets.
+Requires Milestone 17 (Jira client) to be passing.
+"""
+import asyncio
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+
+# ─── Sample signals ───────────────────────────────────────────────────────────
+
+SAMPLE_SIGNALS = [
+    {
+        "id": "test-signal-001",
+        "type": "recurring_bug",
+        "summary": "Checkout endpoint hits intermittent timeout — third time this sprint. Restarting the pod is the workaround.",
+        "raw_quote": "Sam: Timeout error AGAIN. That's the third time. We have a systemic issue here.",
+        "severity": "high",
+        "status": "open",
+        "sentiment": "negative",
+        "urgency": "high",
+        "entities": ["@Sam", "@Alex"],
+        "keywords": ["timeout", "checkout", "pod", "systemic"],
+        "timestamp": "2026-03-21T09:00:00Z",
+        "group_id": "acme_dev",
+        "lens": "dev",
+    },
+    {
+        "id": "test-signal-002",
+        "type": "tech_debt",
+        "summary": "JWT secret is hardcoded in auth service. Will move to Vault later, no timeline set.",
+        "raw_quote": "Alex: For the auth service, I'm hardcoding the JWT secret for now. We'll move to vault later.",
+        "severity": "medium",
+        "status": "open",
+        "sentiment": "neutral",
+        "urgency": "low",
+        "entities": ["@Alex"],
+        "keywords": ["jwt", "hardcode", "vault", "auth", "secret"],
+        "timestamp": "2026-03-21T09:05:00Z",
+        "group_id": "acme_dev",
+        "lens": "dev",
+    },
+    {
+        "id": "test-signal-003",
+        "type": "meet_blocker",
+        "summary": "Dashboard spec has been blocked waiting on design for two weeks. Dev cannot start work.",
+        "raw_quote": "Alex: Still no dashboard specs from design. This is blocking my entire sprint work.",
+        "severity": "high",
+        "status": "open",
+        "sentiment": "negative",
+        "urgency": "high",
+        "entities": ["@Alex", "@design"],
+        "keywords": ["dashboard", "blocked", "design", "specs", "sprint"],
+        "timestamp": "2026-03-21T10:00:00Z",
+        "group_id": "meet_sessions",
+        "lens": "meet",
+        "meeting_id": "sprint-planning-test",
+    },
+]
+
+# A signal type that should NOT be raised (raw chunk is not a raiseable type)
+NON_RAISEABLE_SIGNAL = {
+    "id": "test-signal-999",
+    "type": "meet_chunk_raw",
+    "summary": "Raw transcript chunk — should not be raised as a ticket",
+    "raw_quote": "...",
+    "severity": "low",
+    "status": "open",
+    "sentiment": "neutral",
+    "urgency": "none",
+    "entities": [],
+    "keywords": [],
+    "timestamp": "2026-03-21T10:00:00Z",
+    "group_id": "meet_sessions",
+    "lens": "meet",
+}
+
+
+async def test_ticket_generation():
+    """Test that LLM generates a valid ticket from a signal."""
+    from backend.agents.jira_agent import generate_ticket_content
+
+    print("Testing LLM ticket content generation...")
+    signal = SAMPLE_SIGNALS[0]  # recurring_bug
+    content = await generate_ticket_content(signal)
+
+    assert "summary" in content and len(content["summary"]) > 5, "Summary too short or missing"
+    assert len(content["summary"]) <= 100, f"Summary exceeds 100 chars: {len(content['summary'])}"
+    assert "description" in content and len(content["description"]) > 30, "Description too short"
+    assert "labels" in content and "thirdeye" in content["labels"], "Missing 'thirdeye' label"
+    assert "assignee_name" in content  # can be None, that's fine
+
+    print(f"  ✅ Summary ({len(content['summary'])} chars): {content['summary']}")
+    print(f"  ✅ Description ({len(content['description'])} chars)")
+    print(f"  ✅ Labels: {content['labels']}")
+    print(f"  ✅ Assignee hint: {content.get('assignee_name')}")
+
+
+async def test_raise_single_ticket():
+    """Test raising a single ticket for a real signal."""
+    from backend.agents.jira_agent import raise_ticket_for_signal
+
+    print("\nTesting raise_ticket_for_signal()...")
+    signal = SAMPLE_SIGNALS[0]  # recurring_bug, high severity
+    group_id = "test_jira_m18"
+
+    result = await raise_ticket_for_signal(signal, group_id, force=True)
+
+    assert result.get("ok"), f"raise_ticket_for_signal failed: {result}"
+    print(f"  ✅ Ticket raised: {result['key']}")
+    print(f"     URL: {result['url']}")
+    print(f"     Type: {result['issue_type']} | Priority: {result['priority']}")
+    print(f"     Summary: {result['summary'][:90]}")
+    return result["key"]
+
+
+async def test_dedup_prevents_double_raise():
+    """Test that the same signal cannot be raised twice."""
+    from backend.agents.jira_agent import raise_ticket_for_signal
+    from backend.db.chroma import mark_signal_as_raised
+
+    print("\nTesting dedup — cannot raise the same signal twice...")
+    signal = SAMPLE_SIGNALS[1]  # tech_debt
+    group_id = "test_jira_m18_dedup"
+
+    # First raise
+    result1 = await raise_ticket_for_signal(signal, group_id, force=True)
+    assert result1.get("ok"), f"First raise failed: {result1}"
+    print(f"  ✅ First raise succeeded: {result1['key']}")
+
+    # Second raise of the same signal — should be blocked
+    result2 = await raise_ticket_for_signal(signal, group_id, force=False)
+    assert not result2.get("ok"), "Expected second raise to be blocked"
+    assert result2.get("reason") == "already_raised", f"Expected 'already_raised', got: {result2.get('reason')}"
+    print(f"  ✅ Second raise correctly blocked: reason='{result2['reason']}'")
+
+
+async def test_non_raiseable_signal():
+    """Test that non-raiseable signal types are rejected."""
+    from backend.agents.jira_agent import raise_ticket_for_signal
+
+    print("\nTesting non-raiseable signal type rejection...")
+    result = await raise_ticket_for_signal(NON_RAISEABLE_SIGNAL, "test_group", force=True)
+    assert not result.get("ok")
+    assert result.get("reason") == "not_raiseable"
+    print(f"  ✅ Non-raiseable type correctly rejected: {NON_RAISEABLE_SIGNAL['type']}")
+
+
+async def test_bulk_raise():
+    """Test bulk raising multiple signals at once."""
+    from backend.agents.jira_agent import bulk_raise_for_group
+
+    print("\nTesting bulk_raise_for_group()...")
+    group_id = "test_jira_m18_bulk"
+
+    # Mix of raiseable and non-raiseable, different severities
+    all_signals = SAMPLE_SIGNALS + [NON_RAISEABLE_SIGNAL]
+    results = await bulk_raise_for_group(
+        group_id=group_id,
+        signals=all_signals,
+        min_severity="medium",  # low severity signals should be skipped
+        max_tickets=5,
+    )
+
+    raised = [r for r in results if r.get("ok")]
+    skipped_type = [r for r in results if r.get("reason") == "not_raiseable"]
+
+    assert len(raised) >= 1, "Expected at least 1 ticket raised from bulk"
+    print(f"  ✅ Bulk raised {len(raised)} ticket(s) from {len(all_signals)} signals")
+    for r in raised:
+        print(f"     [{r['key']}] {r.get('signal_type')} — {r.get('signal_summary', '')[:60]}")
+    if skipped_type:
+        print(f"  ✅ {len(skipped_type)} non-raiseable signal(s) correctly skipped")
+
+
+async def test_priority_mapping():
+    """Test that signal severity maps to correct Jira priority."""
+    from backend.agents.jira_agent import SEVERITY_TO_PRIORITY, SIGNAL_TYPE_MAP
+
+    print("\nTesting priority and type mapping...")
+    assert SEVERITY_TO_PRIORITY["critical"] == "Highest"
+    assert SEVERITY_TO_PRIORITY["high"] == "High"
+    assert SEVERITY_TO_PRIORITY["medium"] == "Medium"
+    assert SEVERITY_TO_PRIORITY["low"] == "Low"
+    print("  ✅ Severity → Priority mapping correct")
+
+    assert SIGNAL_TYPE_MAP["recurring_bug"] == ("Task", "High")
+    assert SIGNAL_TYPE_MAP["meet_blocker"] == ("Task", "Highest")
+    assert SIGNAL_TYPE_MAP["feature_request"] == ("Task", "Medium")
+    print("  ✅ Signal type → Jira type mapping correct")
+
+
+async def main():
+    print("Running Milestone 18 tests...\n")
+    await test_priority_mapping()
+    await test_ticket_generation()
+    key = await test_raise_single_ticket()
+    await test_dedup_prevents_double_raise()
+    await test_non_raiseable_signal()
+    await test_bulk_raise()
+    print(f"\n🎉 MILESTONE 18 PASSED — Jira Signal Agent working. First ticket: {key}")
+
+
+asyncio.run(main())