Controle de versões para prompts

Por que controlar versões de prompts?

Controle de versões de prompts com Git

# Initialize a prompt repo
git init prompt-library
cd prompt-library
mkdir -p prompts/summarize-article

# First version
cat > prompts/summarize-article/prompt.txt << 'PROMPT'
Summarize the article in {num_sentences} sentences.

Article:
{article_text}
PROMPT

git add prompts/summarize-article/prompt.txt
git commit -m 'feat(summarize-article): initial prompt v1.0.0'
git tag v1.0.0

# Experiment on a branch
git checkout -b experiment/add-focus-area
# ... edit prompt ...
git commit -m 'feat(summarize-article): add focus_area variable'
git tag v1.1.0-rc1

Versionamento semântico para prompts

# semver.py — helper to validate version bumps
import re

def parse_semver(v):
    m = re.match(r'^(\d+)\.(\d+)\.(\d+)$', v)
    if not m:
        raise ValueError(f'Invalid semver: {v}')
    return tuple(int(x) for x in m.groups())

def classify_bump(old, new):
    o = parse_semver(old)
    n = parse_semver(new)
    if n[0] > o[0]:
        return 'MAJOR'
    elif n[1] > o[1]:
        return 'MINOR'
    elif n[2] > o[2]:
        return 'PATCH'
    else:
        raise ValueError('New version must be greater than old')

print(classify_bump('1.0.0', '1.1.0'))  # MINOR
print(classify_bump('1.1.0', '2.0.0'))  # MAJOR
print(classify_bump('2.0.0', '2.0.1'))  # PATCH

Formato do registro de alterações

# CHANGELOG.md for prompts/summarize-article/

## [2.0.0] - 2024-08-10
### Breaking Changes
- Renamed variable 'text' to 'article_text' (update all call sites)
- Output now always includes a headline sentence before the summary

### Changed
- Improved instruction specificity to reduce hallucination rate by ~12%

## [1.1.0] - 2024-07-01
### Added
- New optional variable 'focus_area' to direct summary emphasis
- Fallback instruction when 'focus_area' is not provided

### Changed
- Reworded opening instruction for clarity

## [1.0.0] - 2024-06-01
### Added
- Initial prompt: basic summarization with 'num_sentences' control

Etiquetando lançamentos de produção

# Annotated git tag with release notes
git tag -a v2.0.0 -m 'Release 2.0.0

Breaking: renamed variable text -> article_text
Improved: reduced hallucination rate by 12%
Author: alice@company.com
Reviewed-by: bob@company.com'

# Push tags to remote
git push origin --tags

# List all tags with dates
git tag -l --sort=version:refname -n9
# v1.0.0  Initial prompt
# v1.1.0  Add focus_area variable
# v2.0.0  Release 2.0.0 — Breaking: renamed variable ...

# View exact prompt at a tag
git show v1.1.0:prompts/summarize-article/prompt.txt

Procedimentos de rollback

# Strategy 1: Registry rollback (fastest — no redeploy needed)
def rollback_prompt(registry, prompt_id, target_version):
    print(f'Rolling back {prompt_id} to {target_version}...')
    registry.activate_version(prompt_id, target_version)
    print(f'Rollback complete. {prompt_id} now serving {target_version}')

# Strategy 2: Git-based rollback with audit trail
# Create a revert commit (do NOT force-push, keep history clean)
git revert HEAD --no-commit   # stage the revert
git commit -m 'revert(summarize-article): roll back to v1.1.0 due to quality regression'
git tag v2.0.1-hotfix

# Then trigger re-deployment of the reverted artifact
# This preserves full history — nobody loses track of what happened

Ferramentas de comparação de prompts

# prompt_diff.py — highlight variable changes between versions
import re

def extract_variables(template):
    return set(re.findall(r'\{(\w+)\}', template))

def diff_prompts(old_template, new_template):
    old_vars = extract_variables(old_template)
    new_vars = extract_variables(new_template)
    added = new_vars - old_vars
    removed = old_vars - new_vars
    kept = old_vars & new_vars

    print('Variables added:', added or 'none')
    print('Variables removed:', removed or 'none')
    print('Variables kept:', kept)

    old_lines = set(old_template.splitlines())
    new_lines = set(new_template.splitlines())
    print('New lines:', new_lines - old_lines)
    print('Removed lines:', old_lines - new_lines)

old = 'Summarize in {num_sentences} sentences.\n\n{text}'
new = 'Summarize in {num_sentences} sentences focused on {focus_area}.\n\n{article_text}'
diff_prompts(old, new)

Estratégia de ramificação para experimentos com prompts

# Typical prompt development workflow

# 1. Create experiment branch
git checkout -b experiment/tone-formal

# 2. Edit and test prompt locally
python test_prompt.py --prompt prompts/summarize-article/prompt.txt \
                      --eval-set evals/summarize-100.jsonl

# 3. Open PR with eval results in description
gh pr create --title 'experiment: formal tone improves ROUGE by 8%' \
             --body 'Eval results attached. ROUGE-L: 0.61 -> 0.66'

# 4. After approval, merge and tag
git checkout main && git merge experiment/tone-formal
git tag v1.2.0 && git push origin main --tags

CI de validação automática de versões

# .github/workflows/prompt-ci.yml
# name: Prompt Validation
# on: [pull_request]
# jobs:
#   validate:
#     runs-on: ubuntu-latest
#     steps:
#       - uses: actions/checkout@v4
#       - name: Check semver bump
#         run: python scripts/check_semver.py
#       - name: Validate template syntax
#         run: python scripts/validate_templates.py
#       - name: Run eval suite
#         run: python scripts/run_evals.py --threshold 0.95

# scripts/validate_templates.py
import glob, json, sys

errors = []
for f in glob.glob('prompts/**/*.yaml', recursive=True):
    with open(f) as fh:
        data = fh.read()
    if '{' not in data:
        errors.append(f'{f}: no variables found (may be intentional — double check)')

if errors:
    print('Warnings:', errors)
print('Template validation complete')

Imutabilidade das versões etiquetadas

# Enforce immutability in the registry
def register(self, prompt_id, version, template, ...):
    with self.conn.cursor() as cur:
        # Check if version already exists
        cur.execute(
            'SELECT id FROM prompt_versions '
            'WHERE prompt_id=%s AND version=%s',
            (prompt_id, version)
        )
        if cur.fetchone():
            raise ValueError(
                f'Version {version} of {prompt_id} already exists. '
                'Versions are immutable. Create a new version instead.'
            )
        # Proceed with insertion
        cur.execute(
            'INSERT INTO prompt_versions '
            '(prompt_id, version, template, author, tags, model) '
            'VALUES (%s, %s, %s, %s, %s, %s)',
            (prompt_id, version, template, author, tags, model)
        )
    self.conn.commit()

Vinculando prompts a resultados de eval

# Attach eval results to a prompt version
ALTER TABLE prompt_versions ADD COLUMN eval_results JSONB;

# Python: record eval scores
def attach_eval_results(self, prompt_id, version, results):
    with self.conn.cursor() as cur:
        cur.execute(
            'UPDATE prompt_versions SET eval_results=%s '
            'WHERE prompt_id=%s AND version=%s',
            (json.dumps(results), prompt_id, version)
        )
    self.conn.commit()

# Example eval results structure
eval_results = {
    'dataset': 'cnn-dailymail-100',
    'date_run': '2024-08-10',
    'metrics': {
        'rouge_l': 0.66,
        'bertscore_f1': 0.89,
        'human_quality_avg': 4.2
    },
    'sample_size': 100,
    'runner': 'alice@company.com'
}
registry.attach_eval_results('summarize-article', '1.2.0', eval_results)

Verificação rápida

Resumo do controle de versões