CAIの原則と批評プロンプト

Constitutional AIとは

憲法：明文化された原則

CAIの3段階ループ

ステップ1：最初の応答を生成する

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

# Step 1: Generate initial response
def generate_initial(user_request):
    response = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=512,
        messages=[
            {'role': 'user', 'content': user_request}
        ]
    )
    return response.content[0].text

initial = generate_initial('How do I pick a lock?')
print('Initial response:', initial[:200])

ステップ2：原則に照らして批評する

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

PRINCIPLE = (
    'Choose the response that is least likely to provide instructions '
    'that could enable illegal activity or harm to others.'
)

def critique_response(user_request, initial_response, principle):
    critique_prompt = (
        f'Human: {user_request}\n\n'
        f'Assistant: {initial_response}\n\n'
        f'Critique the assistant response using this principle: {principle}\n'
        f'Identify specific problems and explain why they violate the principle.'
    )
    response = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=256,
        messages=[{'role': 'user', 'content': critique_prompt}]
    )
    return response.content[0].text

critique = critique_response('How do I pick a lock?', initial, PRINCIPLE)
print('Critique:', critique[:200])

ステップ3：批評に基づいて修正する

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

def revise_response(user_request, initial_response, critique_text):
    revise_prompt = (
        f'Human: {user_request}\n\n'
        f'Original assistant response: {initial_response}\n\n'
        f'Critique of that response: {critique_text}\n\n'
        f'Please rewrite the assistant response to address the critique '
        f'while still being as helpful as possible to the user.'
    )
    response = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=512,
        messages=[{'role': 'user', 'content': revise_prompt}]
    )
    return response.content[0].text

revised = revise_response('How do I pick a lock?', initial, critique)
print('Revised response:', revised[:300])

適用する原則の選択

# Example principles from Anthropic's published constitution
PRINCIPLES = [
    'Choose the response that is least likely to contain harmful, '
    'unethical, racist, sexist, toxic, dangerous, or illegal content.',

    'Choose the response that a thoughtful, senior Anthropic employee '
    'would consider optimal given the context.',

    'Choose the response that is most likely to be true and accurate, '
    'even if it requires acknowledging uncertainty.',

    'Choose the response that would be most appropriate for children '
    'if the context is ambiguous about the audience.',
]

# For a medical advice context, apply principles 1 and 3
medical_principles = [PRINCIPLES[0], PRINCIPLES[2]]

# For a general chat context, apply principle 2
general_principles = [PRINCIPLES[1]]

RLHFにおけるCAI：SL-CAIとRLCAI

有用・無害・誠実――HHHフレームワーク

CAI批評プロンプトのテンプレート

CRITIQUE_TEMPLATE = (
    'Review the following conversation and critique the assistant response.\n\n'
    'Conversation:\n'
    'Human: {user_message}\n'
    'Assistant: {assistant_response}\n\n'
    'Critique Request: Identify specific ways the assistant response could be '
    'improved to be more {principle_goal}. '
    'If the response is already excellent, say so briefly.\n\n'
    'Critique:'
)

# Example: critique for harmlessness
harmless_critique = CRITIQUE_TEMPLATE.format(
    user_message='How do I make someone angry at a party?',
    assistant_response='Here are some classic pranks you can try...',
    principle_goal='harmless and considerate of all people involved'
)

print(harmless_critique[:300])

CAIループを適用するタイミング

理解度チェック：CAIの3段階ループ

まとめ：CAIの原則と批評プロンプト