Skip to content

Chains

Legacy API

The chains module is being replaced by the pluggable adapter architecture. New code should use the adapter classes in extractors/, gatherers/, verifiers/, and formatters/ packages instead. See the Adapters API Reference for the modern interface.

Claim Extraction

SimpleClaimExtractionChain

from truthfulness_evaluator.llm.chains.extraction import SimpleClaimExtractionChain

extractor = SimpleClaimExtractionChain(model="gpt-4o-mini")
claims = await extractor.extract(
    document="Python was created in 1991...",
    source_path="README.md",
    max_claims=10
)

Output: list[Claim]

Uses structured output (ClaimExtractionOutput): 

class ClaimExtractionOutput(BaseModel):
    claims: list[ExtractedClaim]

class ExtractedClaim(BaseModel):
    text: str
    claim_type: str  # "explicit", "implicit", "inferred"

TripletExtractionChain

from truthfulness_evaluator.llm.chains.extraction import TripletExtractionChain

extractor = TripletExtractionChain(model="gpt-4o-mini")
claims = await extractor.extract(document, source_path)

Extracts subject-relation-object triplets.

Verification

VerificationChain

from truthfulness_evaluator.llm.chains.verification import VerificationChain

verifier = VerificationChain(model_name="gpt-4o")
result = await verifier.verify(claim, evidence)

Output: VerificationResult

Uses structured output (VerificationOutput): 

class VerificationOutput(BaseModel):
    verdict: str           # "SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"
    confidence: float      # 0.0 to 1.0
    reasoning: str
    key_evidence: str | None

Consensus

ConsensusChain

from truthfulness_evaluator.llm.chains.consensus import ConsensusChain

consensus = ConsensusChain(
    model_names=["gpt-4o", "gpt-4o-mini"],
    weights={"gpt-4o": 0.6, "gpt-4o-mini": 0.4},
    confidence_threshold=0.7
)

result = await consensus.verify(claim, evidence)

Each model votes; votes are tallied with the given weights. The leading verdict is committed only if it isn't tied for the lead and its weighted agreement fraction meets confidence_threshold — otherwise the result is NOT_ENOUGH_INFO. Reported confidence is that agreement fraction, not an average of the models' self-reported scores.

Evidence Processing

EvidenceProcessor

from truthfulness_evaluator.llm.chains.evidence import EvidenceProcessor

processor = EvidenceProcessor(model="gpt-4o-mini")

# Analyze evidence relevance
evidence, summary = await processor.analyze_evidence(claim, evidence_list)

# Synthesize multiple pieces
synthesis = await processor.synthesize_evidence(claim, evidence_list)

Output: Analyzed evidence with scores + summary text

Uses structured output (EvidenceAnalysisOutput): 

class EvidenceAnalysisOutput(BaseModel):
    evidence_analysis: list[EvidenceAnalysisItem]
    summary: str

class EvidenceAnalysisItem(BaseModel):
    index: int
    relevance: float
    supports: bool | None
    credibility: float
    reasoning: str

Custom Chains

Build your own:

from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI

# Define output structure
class MyOutput(BaseModel):
    result: str
    confidence: float

# Create chain
prompt = ChatPromptTemplate.from_template("Verify: {claim}")
llm = ChatOpenAI(model="gpt-4o").with_structured_output(MyOutput)
chain = prompt | llm

# Use
result = await chain.ainvoke({"claim": "Python was created in 1991"})

API Reference

Claim Extraction

truthfulness_evaluator.llm.chains.extraction.SimpleClaimExtractionChain

Fallback claim extraction using structured LLM outputs.

Source code in src/truthfulness_evaluator/llm/chains/extraction.py 
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
class SimpleClaimExtractionChain:
    """Fallback claim extraction using structured LLM outputs."""

    def __init__(self, model: str = "gpt-4o-mini"):
        self.model = model
        self._llm = None

    @property
    def llm(self):
        """Lazy initialization of LLM with structured output."""
        if self._llm is None:
            base_llm = create_chat_model(self.model, temperature=0)
            # Use structured output
            self._llm = base_llm.with_structured_output(ClaimExtractionOutput)
        return self._llm

    async def extract(
        self, document: str, source_path: str, max_claims: Optional[int] = None
    ) -> list[Claim]:
        """Extract claims using structured LLM output."""
        # Strip example blocks before extraction to avoid false positives
        # from illustrative content in fenced code blocks and inline code.
        cleaned_document = strip_example_blocks(document)

        chain = CLAIM_EXTRACTION_PROMPT | self.llm

        try:
            result: ClaimExtractionOutput = await chain.ainvoke({"text": cleaned_document})

            claims = []
            for i, extracted in enumerate(result.claims):
                if max_claims and i >= max_claims:
                    break

                if extracted.text.strip():
                    claims.append(
                        Claim(
                            id=f"claim_{i:03d}",
                            text=extracted.text.strip(),
                            source_document=source_path,
                            claim_type=(
                                extracted.claim_type
                                if extracted.claim_type in ["explicit", "implicit", "inferred"]
                                else "explicit"
                            ),
                        )
                    )

            return claims

        except Exception as e:
            logger.warning(f"Simple extraction failed: {e}")
            return []

llm property

Lazy initialization of LLM with structured output.

extract(document, source_path, max_claims=None) async

Extract claims using structured LLM output.

Source code in src/truthfulness_evaluator/llm/chains/extraction.py 
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
async def extract(
    self, document: str, source_path: str, max_claims: Optional[int] = None
) -> list[Claim]:
    """Extract claims using structured LLM output."""
    # Strip example blocks before extraction to avoid false positives
    # from illustrative content in fenced code blocks and inline code.
    cleaned_document = strip_example_blocks(document)

    chain = CLAIM_EXTRACTION_PROMPT | self.llm

    try:
        result: ClaimExtractionOutput = await chain.ainvoke({"text": cleaned_document})

        claims = []
        for i, extracted in enumerate(result.claims):
            if max_claims and i >= max_claims:
                break

            if extracted.text.strip():
                claims.append(
                    Claim(
                        id=f"claim_{i:03d}",
                        text=extracted.text.strip(),
                        source_document=source_path,
                        claim_type=(
                            extracted.claim_type
                            if extracted.claim_type in ["explicit", "implicit", "inferred"]
                            else "explicit"
                        ),
                    )
                )

        return claims

    except Exception as e:
        logger.warning(f"Simple extraction failed: {e}")
        return []

truthfulness_evaluator.llm.chains.extraction.TripletExtractionChain

Extract claims as subject-relation-object triplets using structured output.

Source code in src/truthfulness_evaluator/llm/chains/extraction.py 
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
class TripletExtractionChain:
    """Extract claims as subject-relation-object triplets using structured output."""

    def __init__(self, model: str = "gpt-4o-mini"):
        self.model = model
        self._llm = None

    @property
    def llm(self):
        """Lazy initialization of LLM with structured output."""
        if self._llm is None:
            base_llm = create_chat_model(self.model, temperature=0)
            self._llm = base_llm.with_structured_output(TripletExtractionOutput)
        return self._llm

    async def extract(
        self, document: str, source_path: str, max_claims: Optional[int] = None
    ) -> list[Claim]:
        """Extract claims as triplets using structured output."""
        # Strip example blocks before extraction to avoid false positives
        # from illustrative content in fenced code blocks and inline code.
        cleaned_document = strip_example_blocks(document)

        chain = TRIPLET_EXTRACTION_PROMPT | self.llm

        try:
            result: TripletExtractionOutput = await chain.ainvoke({"text": cleaned_document})

            claims = []
            for i, triplet in enumerate(result.triplets):
                if max_claims and i >= max_claims:
                    break

                claim_text = f"{triplet.subject} {triplet.relation} {triplet.object}".strip()

                if claim_text:
                    claims.append(
                        Claim(
                            id=f"claim_{i:03d}",
                            text=claim_text,
                            source_document=source_path,
                            context=triplet.context,
                            claim_type="explicit",
                        )
                    )

            return claims

        except Exception as e:
            logger.warning(f"Triplet extraction failed: {e}")
            return []

llm property

Lazy initialization of LLM with structured output.

extract(document, source_path, max_claims=None) async

Extract claims as triplets using structured output.

Source code in src/truthfulness_evaluator/llm/chains/extraction.py 
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
async def extract(
    self, document: str, source_path: str, max_claims: Optional[int] = None
) -> list[Claim]:
    """Extract claims as triplets using structured output."""
    # Strip example blocks before extraction to avoid false positives
    # from illustrative content in fenced code blocks and inline code.
    cleaned_document = strip_example_blocks(document)

    chain = TRIPLET_EXTRACTION_PROMPT | self.llm

    try:
        result: TripletExtractionOutput = await chain.ainvoke({"text": cleaned_document})

        claims = []
        for i, triplet in enumerate(result.triplets):
            if max_claims and i >= max_claims:
                break

            claim_text = f"{triplet.subject} {triplet.relation} {triplet.object}".strip()

            if claim_text:
                claims.append(
                    Claim(
                        id=f"claim_{i:03d}",
                        text=claim_text,
                        source_document=source_path,
                        context=triplet.context,
                        claim_type="explicit",
                    )
                )

        return claims

    except Exception as e:
        logger.warning(f"Triplet extraction failed: {e}")
        return []

Verification

truthfulness_evaluator.llm.chains.verification.VerificationChain

Single-model verification chain with structured outputs.

Source code in src/truthfulness_evaluator/llm/chains/verification.py 
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
class VerificationChain:
    """Single-model verification chain with structured outputs."""

    def __init__(self, model_name: str = "gpt-4o"):
        self.model_name = model_name
        self._llm = None

    @property
    def llm(self):
        """Lazy initialization of LLM with structured output."""
        if self._llm is None:
            base_llm = create_chat_model(self.model_name, temperature=0)
            # Use structured output
            self._llm = base_llm.with_structured_output(VerificationOutput)
        return self._llm

    async def verify(self, claim: Claim, evidence: list[Evidence]) -> VerificationResult:
        """Verify a claim against evidence using structured output."""

        # Build evidence text with relevance indicators
        if evidence:
            evidence_parts = []
            for i, e in enumerate(evidence[:4], 1):  # Top 4 evidence items
                support_indicator = ""
                if e.supports_claim is True:
                    support_indicator = "[SUPPORTS]"
                elif e.supports_claim is False:
                    support_indicator = "[REFUTES]"
                else:
                    support_indicator = "[NEUTRAL]"

                evidence_parts.append(
                    f"\n--- Evidence {i} ({e.source_type}) {support_indicator} ---\n"
                    f"Source: {e.source}\n"
                    f"Relevance: {e.relevance_score:.0%}\n"
                    f"Content: {e.content[:600]}"
                )

            evidence_text = "\n".join(evidence_parts)
        else:
            evidence_text = "No evidence provided."

        chain = VERIFICATION_PROMPT | self.llm

        try:
            result: VerificationOutput = await chain.ainvoke(
                {"claim": claim.text, "evidence": evidence_text}
            )

            # Normalize verdict
            verdict = result.verdict.upper()
            if verdict not in ["SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"]:
                verdict = "NOT_ENOUGH_INFO"

            # Combine reasoning with key evidence
            full_explanation = result.reasoning
            if result.key_evidence:
                full_explanation += f"\n\nKey evidence: {result.key_evidence}"

            # Adjust confidence based on evidence quality
            confidence = max(0.0, min(1.0, result.confidence))
            if not evidence:
                confidence = min(confidence, 0.3)

            return VerificationResult(
                claim_id=claim.id,
                verdict=verdict,
                confidence=confidence,
                evidence=evidence,
                explanation=full_explanation,
                model_votes={self.model_name: verdict},
            )

        except Exception as e:
            # Fallback result on error
            return VerificationResult(
                claim_id=claim.id,
                verdict="NOT_ENOUGH_INFO",
                confidence=0.0,
                evidence=evidence,
                explanation=f"Verification failed: {str(e)}",
                model_votes={self.model_name: "NOT_ENOUGH_INFO"},
            )

llm property

Lazy initialization of LLM with structured output.

verify(claim, evidence) async

Verify a claim against evidence using structured output.

Source code in src/truthfulness_evaluator/llm/chains/verification.py 
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
async def verify(self, claim: Claim, evidence: list[Evidence]) -> VerificationResult:
    """Verify a claim against evidence using structured output."""

    # Build evidence text with relevance indicators
    if evidence:
        evidence_parts = []
        for i, e in enumerate(evidence[:4], 1):  # Top 4 evidence items
            support_indicator = ""
            if e.supports_claim is True:
                support_indicator = "[SUPPORTS]"
            elif e.supports_claim is False:
                support_indicator = "[REFUTES]"
            else:
                support_indicator = "[NEUTRAL]"

            evidence_parts.append(
                f"\n--- Evidence {i} ({e.source_type}) {support_indicator} ---\n"
                f"Source: {e.source}\n"
                f"Relevance: {e.relevance_score:.0%}\n"
                f"Content: {e.content[:600]}"
            )

        evidence_text = "\n".join(evidence_parts)
    else:
        evidence_text = "No evidence provided."

    chain = VERIFICATION_PROMPT | self.llm

    try:
        result: VerificationOutput = await chain.ainvoke(
            {"claim": claim.text, "evidence": evidence_text}
        )

        # Normalize verdict
        verdict = result.verdict.upper()
        if verdict not in ["SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"]:
            verdict = "NOT_ENOUGH_INFO"

        # Combine reasoning with key evidence
        full_explanation = result.reasoning
        if result.key_evidence:
            full_explanation += f"\n\nKey evidence: {result.key_evidence}"

        # Adjust confidence based on evidence quality
        confidence = max(0.0, min(1.0, result.confidence))
        if not evidence:
            confidence = min(confidence, 0.3)

        return VerificationResult(
            claim_id=claim.id,
            verdict=verdict,
            confidence=confidence,
            evidence=evidence,
            explanation=full_explanation,
            model_votes={self.model_name: verdict},
        )

    except Exception as e:
        # Fallback result on error
        return VerificationResult(
            claim_id=claim.id,
            verdict="NOT_ENOUGH_INFO",
            confidence=0.0,
            evidence=evidence,
            explanation=f"Verification failed: {str(e)}",
            model_votes={self.model_name: "NOT_ENOUGH_INFO"},
        )

Consensus

truthfulness_evaluator.llm.chains.consensus.ConsensusChain

Multi-model consensus via weighted, agreement-based voting.

Source code in src/truthfulness_evaluator/llm/chains/consensus.py 
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
class ConsensusChain:
    """Multi-model consensus via weighted, agreement-based voting."""

    def __init__(
        self,
        model_names: list[str],
        weights: dict[str, float] | None = None,
        confidence_threshold: float = 0.7,
    ):
        self.model_names = model_names
        self.weights = weights or {m: 1.0 / len(model_names) for m in model_names}
        # Minimum weighted agreement for the leading verdict to be committed;
        # below it (or on a tie) the ensemble abstains. With N equal-weight
        # models a value of 0.7 effectively requires near-unanimity (e.g. 3
        # models must all agree, since 2/3 = 0.67 < 0.7); lower it to accept
        # simple majorities.
        self.confidence_threshold = confidence_threshold
        self._chains: list[VerificationChain] | None = None

    @property
    def chains(self) -> list[VerificationChain]:
        """Lazy initialization of the per-model verification chains."""
        if self._chains is None:
            self._chains = [VerificationChain(m) for m in self.model_names]
        return self._chains

    async def verify(self, claim: Claim, evidence: list[Evidence]) -> VerificationResult:
        """Verify a claim by polling every model and aggregating by agreement."""
        results = await asyncio.gather(*[chain.verify(claim, evidence) for chain in self.chains])

        votes = {self.model_names[i]: r.verdict for i, r in enumerate(results)}
        self_reported = {self.model_names[i]: r.confidence for i, r in enumerate(results)}

        # Weighted tally of verdicts.
        tally: Counter = Counter()
        for model, verdict in votes.items():
            tally[verdict] += self.weights.get(model, 1.0 / len(self.model_names))

        total_weight = sum(tally.values()) or 1.0
        ranked = tally.most_common()
        leading_verdict, leading_weight = ranked[0]
        agreement = leading_weight / total_weight

        # A shared top weight is unresolved agreement -> abstain.
        tie = len(ranked) > 1 and ranked[1][1] == leading_weight
        abstained = tie or agreement < self.confidence_threshold
        final_verdict = ABSTAIN if abstained else leading_verdict

        # Confidence reflects consensus strength, not the models' unreliable
        # self-reported scores.
        confidence = round(agreement, 4)

        all_evidence: list[Evidence] = []
        for r in results:
            all_evidence.extend(r.evidence)

        vote_lines = [
            f"{m}: {votes[m]} (self-reported {self_reported[m]:.0%})" for m in self.model_names
        ]
        explanation_parts = [
            f"Consensus: {final_verdict} ({agreement:.0%} weighted agreement)",
            "Model votes:",
            *vote_lines,
        ]
        if final_verdict == ABSTAIN and leading_verdict != ABSTAIN:
            reason = "tie" if tie else f"agreement below {self.confidence_threshold:.0%} threshold"
            explanation_parts.append(f"(Abstained: {reason}.)")

        logger.debug(
            "Consensus %s at %.0f%% agreement; votes=%s", final_verdict, agreement * 100, votes
        )

        return VerificationResult(
            claim_id=claim.id,
            verdict=final_verdict,
            confidence=confidence,
            evidence=all_evidence[:5],
            explanation="\n".join(explanation_parts),
            model_votes=votes,
        )

chains: list[VerificationChain] property

Lazy initialization of the per-model verification chains.

verify(claim, evidence) async

Verify a claim by polling every model and aggregating by agreement.

Source code in src/truthfulness_evaluator/llm/chains/consensus.py 
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
async def verify(self, claim: Claim, evidence: list[Evidence]) -> VerificationResult:
    """Verify a claim by polling every model and aggregating by agreement."""
    results = await asyncio.gather(*[chain.verify(claim, evidence) for chain in self.chains])

    votes = {self.model_names[i]: r.verdict for i, r in enumerate(results)}
    self_reported = {self.model_names[i]: r.confidence for i, r in enumerate(results)}

    # Weighted tally of verdicts.
    tally: Counter = Counter()
    for model, verdict in votes.items():
        tally[verdict] += self.weights.get(model, 1.0 / len(self.model_names))

    total_weight = sum(tally.values()) or 1.0
    ranked = tally.most_common()
    leading_verdict, leading_weight = ranked[0]
    agreement = leading_weight / total_weight

    # A shared top weight is unresolved agreement -> abstain.
    tie = len(ranked) > 1 and ranked[1][1] == leading_weight
    abstained = tie or agreement < self.confidence_threshold
    final_verdict = ABSTAIN if abstained else leading_verdict

    # Confidence reflects consensus strength, not the models' unreliable
    # self-reported scores.
    confidence = round(agreement, 4)

    all_evidence: list[Evidence] = []
    for r in results:
        all_evidence.extend(r.evidence)

    vote_lines = [
        f"{m}: {votes[m]} (self-reported {self_reported[m]:.0%})" for m in self.model_names
    ]
    explanation_parts = [
        f"Consensus: {final_verdict} ({agreement:.0%} weighted agreement)",
        "Model votes:",
        *vote_lines,
    ]
    if final_verdict == ABSTAIN and leading_verdict != ABSTAIN:
        reason = "tie" if tie else f"agreement below {self.confidence_threshold:.0%} threshold"
        explanation_parts.append(f"(Abstained: {reason}.)")

    logger.debug(
        "Consensus %s at %.0f%% agreement; votes=%s", final_verdict, agreement * 100, votes
    )

    return VerificationResult(
        claim_id=claim.id,
        verdict=final_verdict,
        confidence=confidence,
        evidence=all_evidence[:5],
        explanation="\n".join(explanation_parts),
        model_votes=votes,
    )

Evidence Processing

truthfulness_evaluator.llm.chains.evidence.EvidenceProcessor

Process and analyze evidence for claims using structured outputs.

Source code in src/truthfulness_evaluator/llm/chains/evidence.py 
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
class EvidenceProcessor:
    """Process and analyze evidence for claims using structured outputs."""

    def __init__(self, model: str = "gpt-4o-mini"):
        self.model = model
        self._llm = None

    @property
    def llm(self):
        """Lazy initialization of LLM with structured output."""
        if self._llm is None:
            base_llm = create_chat_model(self.model, temperature=0)
            self._llm = base_llm.with_structured_output(EvidenceAnalysisOutput)
        return self._llm

    async def analyze_evidence(
        self, claim: Claim, evidence_list: list[Evidence]
    ) -> tuple[list[Evidence], str]:
        """
        Analyze evidence and determine which pieces are relevant.

        Returns:
            Tuple of (filtered_evidence, analysis_summary)
        """
        if not evidence_list:
            return [], "No evidence provided"

        # Build evidence text
        evidence_text = "\n\n---\n\n".join(
            [
                f"[{i}] Source: {e.source}\nType: {e.source_type}\nContent: {e.content[:800]}"
                for i, e in enumerate(evidence_list[:5])  # Top 5 pieces
            ]
        )

        chain = EVIDENCE_ANALYSIS_PROMPT | self.llm

        try:
            result: EvidenceAnalysisOutput = await chain.ainvoke(
                {"claim": claim.text, "evidence": evidence_text}
            )

            # Update evidence with analysis
            for analysis in result.evidence_analysis:
                idx = analysis.index
                if 0 <= idx < len(evidence_list):
                    evidence_list[idx].relevance_score = max(0.0, min(1.0, analysis.relevance))
                    evidence_list[idx].supports_claim = analysis.supports
                    evidence_list[idx].credibility_score = max(0.0, min(1.0, analysis.credibility))

            # Sort by relevance
            evidence_list.sort(key=lambda e: e.relevance_score, reverse=True)

            return evidence_list, result.summary

        except Exception as e:
            # If analysis fails, return original evidence
            return evidence_list, f"Analysis failed: {str(e)}"

    async def synthesize_evidence(self, claim: Claim, evidence_list: list[Evidence]) -> str:
        """
        Synthesize multiple pieces of evidence into a coherent summary.

        Returns:
            Summary text
        """
        if not evidence_list:
            return "No evidence available"

        # Filter to high-relevance evidence
        good_evidence = [e for e in evidence_list if e.relevance_score >= 0.5][:3]

        if not good_evidence:
            return "No highly relevant evidence found"

        # Simple synthesis without LLM for speed
        parts = []
        for e in good_evidence:
            support = (
                "supports"
                if e.supports_claim
                else "refutes" if e.supports_claim is False else "is neutral on"
            )
            parts.append(
                f"{e.source} ({e.source_type}) {support} the claim with {e.relevance_score:.0%} relevance"
            )

        return "; ".join(parts)

llm property

Lazy initialization of LLM with structured output.

analyze_evidence(claim, evidence_list) async

Analyze evidence and determine which pieces are relevant.

Returns:

Type Description
tuple[list[Evidence], str]

Tuple of (filtered_evidence, analysis_summary)
Source code in src/truthfulness_evaluator/llm/chains/evidence.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
async def analyze_evidence(
    self, claim: Claim, evidence_list: list[Evidence]
) -> tuple[list[Evidence], str]:
    """
    Analyze evidence and determine which pieces are relevant.

    Returns:
        Tuple of (filtered_evidence, analysis_summary)
    """
    if not evidence_list:
        return [], "No evidence provided"

    # Build evidence text
    evidence_text = "\n\n---\n\n".join(
        [
            f"[{i}] Source: {e.source}\nType: {e.source_type}\nContent: {e.content[:800]}"
            for i, e in enumerate(evidence_list[:5])  # Top 5 pieces
        ]
    )

    chain = EVIDENCE_ANALYSIS_PROMPT | self.llm

    try:
        result: EvidenceAnalysisOutput = await chain.ainvoke(
            {"claim": claim.text, "evidence": evidence_text}
        )

        # Update evidence with analysis
        for analysis in result.evidence_analysis:
            idx = analysis.index
            if 0 <= idx < len(evidence_list):
                evidence_list[idx].relevance_score = max(0.0, min(1.0, analysis.relevance))
                evidence_list[idx].supports_claim = analysis.supports
                evidence_list[idx].credibility_score = max(0.0, min(1.0, analysis.credibility))

        # Sort by relevance
        evidence_list.sort(key=lambda e: e.relevance_score, reverse=True)

        return evidence_list, result.summary

    except Exception as e:
        # If analysis fails, return original evidence
        return evidence_list, f"Analysis failed: {str(e)}"

synthesize_evidence(claim, evidence_list) async

Synthesize multiple pieces of evidence into a coherent summary.

Returns:

Type Description
str

Summary text
Source code in src/truthfulness_evaluator/llm/chains/evidence.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
async def synthesize_evidence(self, claim: Claim, evidence_list: list[Evidence]) -> str:
    """
    Synthesize multiple pieces of evidence into a coherent summary.

    Returns:
        Summary text
    """
    if not evidence_list:
        return "No evidence available"

    # Filter to high-relevance evidence
    good_evidence = [e for e in evidence_list if e.relevance_score >= 0.5][:3]

    if not good_evidence:
        return "No highly relevant evidence found"

    # Simple synthesis without LLM for speed
    parts = []
    for e in good_evidence:
        support = (
            "supports"
            if e.supports_claim
            else "refutes" if e.supports_claim is False else "is neutral on"
        )
        parts.append(
            f"{e.source} ({e.source_type}) {support} the claim with {e.relevance_score:.0%} relevance"
        )

    return "; ".join(parts)