正規表現による AI 向けテキストクリーニング

raw = "<p>Contact us at info@shop.com or visit https://shop.com NOW!!!   Thanks</p>"

import re

text = re.sub("<[^>]+>", "", raw)
print(text)   # "Contact us at info@shop.com or visit https://shop.com NOW!!!   Thanks"

import re

text = re.sub("https?://\S+", "", text)
print(text)   # URL removed

import re

text = re.sub("\S+@\S+\.\S+", "", text)
print(text)   # email removed

import re

masked = re.sub("\S+@\S+\.\S+", "[EMAIL]", "reach me at a@b.com please")
print(masked)   # "reach me at [EMAIL] please"

import re

text = re.sub("[^A-Za-z0-9 ]", "", "Hello!! @world #2026")
print(text)   # "Hello world 2026"

import re

text = re.sub("\s+", " ", "Hello    world\n\n  again").strip()
print(text)   # "Hello world again"

text = text.lower()
# Pipeline order: tags -> urls -> emails -> specials -> whitespace -> lowercase

import re

def clean_text(s):
    s = re.sub("<[^>]+>", " ", s)          # html tags
    s = re.sub("https?://\S+", " ", s)     # urls
    s = re.sub("\S+@\S+\.\S+", " ", s)    # emails
    s = re.sub("[^A-Za-z0-9 ]", " ", s)     # special chars
    s = re.sub("\s+", " ", s).strip()       # whitespace
    return s.lower()

print(clean_text(raw))

import pandas as pd

df["clean"] = df["text"].apply(clean_text)
print(df[["text", "clean"]].head())