BERT를 활용한 텍스트 분류

정적 임베딩의 한계

BERT란 무엇인가

Hugging Face 트랜스포머 라이브러리

from transformers import AutoTokenizer, AutoModelForSequenceClassification

name = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(name)
model = AutoModelForSequenceClassification.from_pretrained(name)

AutoTokenizer

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
print(tokenizer.tokenize("unbelievable"))
# subwords like ["un", "##bel", "##ievable"]

AutoModelForSequenceClassification

from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased-finetuned-sst-2-english"
)

입력 토큰화하기

inputs = tokenizer(
    "This movie was fantastic!",
    truncation=True,
    padding=True,
    return_tensors="pt",
)
print(inputs["input_ids"].shape)

특수 토큰

모델 실행하기

import torch

with torch.no_grad():
    outputs = model(**inputs)
logits = outputs.logits
print(logits)

로짓에서 예측으로

import torch

probs = torch.softmax(logits, dim=-1)
pred = torch.argmax(probs, dim=-1).item()
print(model.config.id2label[pred])

작업에 맞게 미세 조정하기

from transformers import Trainer, TrainingArguments

args = TrainingArguments(output_dir="out", num_train_epochs=3)
trainer = Trainer(model=model, args=args, train_dataset=train_ds)
trainer.train()

pipeline 간편 기능

from transformers import pipeline

clf = pipeline("sentiment-analysis")
print(clf("I love this product!"))
# [{"label": "POSITIVE", "score": 0.99...}]

빠른 확인

복습