TF-IDF और पद-आवृत्ति विश्लेषण

library(tidytext)
library(dplyr)

docs <- tibble::tibble(
  document = c(rep('ML', 2), rep('Stats', 2), rep('DB', 2)),
  text = c(
    'machine learning neural gradient',
    'deep networks training epochs',
    'probability distributions variance covariance',
    'bayesian inference prior posterior',
    'sql database tables joins indexes',
    'queries transactions foreign primary'
  )
)

tfidf <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n)

cat('Columns added:', names(tfidf), '\n')
print(head(arrange(tfidf, desc(tf_idf)), 8))

library(tidytext)
library(dplyr)

docs <- tibble::tibble(
  document = c(rep('Python', 3), rep('R', 3), rep('Julia', 3)),
  text = c(
    'python pandas numpy scipy programming',
    'machine learning scikit tensorflow keras',
    'jupyter notebooks scripts debugging modules',
    'ggplot2 dplyr tidyverse statistics vectors',
    'shiny rmarkdown knitr cran bioconductor',
    'linear models factors dataframes packages',
    'julia multiple dispatch type system macros',
    'package ecosystem flux diffeq parallel',
    'scientific computing performance benchmarks'
  )
)

top_terms <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n) |>
  arrange(desc(tf_idf))

cat('Top distinctive terms per language:\n')
print(head(top_terms[, c('document', 'word', 'tf_idf')], 9))

library(tidytext)
library(dplyr)

docs <- tibble::tibble(
  document = c(rep('Cooking', 3), rep('Finance', 3), rep('Sports', 3)),
  text = c(
    'recipe ingredients oven bake flour butter',
    'cooking temperature boil simmer saute',
    'kitchen knife chopping herbs spices garnish',
    'investment portfolio returns dividends stocks bonds',
    'market equity risk hedge fund derivatives',
    'inflation interest rate treasury balance sheet',
    'goal tackle pass dribble goalkeeper penalty',
    'tournament league championship trophy season',
    'athlete training stamina sprint endurance'
  )
)

top5 <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n) |>
  group_by(document) |>
  slice_max(tf_idf, n = 3) |>
  ungroup()

print(top5[, c('document', 'word', 'tf_idf')])

library(tidytext)
library(dplyr)
library(ggplot2)

docs <- tibble::tibble(
  document = c(rep('AI', 3), rep('DB', 3), rep('Web', 3)),
  text = c(
    'neural networks gradient backpropagation',
    'deep learning transformer attention bert',
    'reinforcement policy reward exploration',
    'sql database indexes joins transactions',
    'relational schema normalization queries',
    'nosql mongodb redis cassandra document',
    'html css javascript react dom',
    'frontend api fetch async components',
    'responsive flexbox grid webpack bundle'
  )
)

plot_data <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n) |>
  group_by(document) |>
  slice_max(tf_idf, n = 4) |>
  ungroup()

ggplot(plot_data, aes(reorder(word, tf_idf), tf_idf, fill = document)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~document, scales = 'free_y') +
  coord_flip() +
  labs(x = NULL, y = 'TF-IDF', title = 'Top TF-IDF Terms per Topic') +
  theme_minimal()

library(tidytext)
library(dplyr)

docs <- tibble::tibble(
  document = c('A', 'B', 'C'),
  text = c(
    'data analysis statistics regression probability',
    'data engineering pipelines etl transformation',
    'data visualisation ggplot2 charts dashboards'
  )
)

tfidf <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n)

# 'data' appears in all 3 docs: tf_idf should be 0
data_rows <- filter(tfidf, word == 'data')
cat('TF-IDF for "data" across documents:\n')
print(data_rows[, c('document', 'word', 'idf', 'tf_idf')])

library(tidytext)
library(dplyr)

docs <- tibble::tibble(
  document = c(rep('Doc1', 3), rep('Doc2', 3)),
  text = c(
    'bayesian statistics prior posterior mcmc',
    'markov chain monte carlo sampling',
    'probability distributions conjugate inference',
    'convolutional neural network image classification',
    'pooling activation relu softmax batch',
    'convolution filter feature map stride padding'
  )
)

tfidf <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n)

# Top 3 unique terms per document
tfidf |>
  group_by(document) |>
  slice_max(tf_idf, n = 3) |>
  select(document, word, tf_idf) |>
  print()

library(tidytext)
library(dplyr)

# Requires janeaustenr package
# library(janeaustenr)
# austen_books() returns: book, text

# Simulated mini-Austen corpus
mini_austen <- tibble::tibble(
  book = c(rep('Sense', 3), rep('Pride', 3), rep('Emma', 3)),
  text = c(
    'elinor marianne dashwood willoughby colonel',
    'edward ferrars barton cottage sister sense',
    'brandon feelings attachment sensibility heart',
    'darcy bennet bingley netherfield wickham',
    'jane lizzy lydia longbourn pemberley',
    'pride prejudice proposal marriage happiness',
    'emma woodhouse knightley harriet weston',
    'highbury match social governess niece',
    'frank jane fairfax box hill picnic'
  )
)

top <- mini_austen |>
  unnest_tokens(word, text) |>
  count(book, word) |>
  bind_tf_idf(word, book, n) |>
  group_by(book) |>
  slice_max(tf_idf, n = 3) |>
  select(book, word, tf_idf)

print(top)

library(tidytext)
library(dplyr)

docs <- tibble::tibble(
  document = c(rep('Positive', 4), rep('Negative', 4)),
  text = c(
    'excellent product love recommend quality',
    'amazing fast delivery five star perfect',
    'great value money satisfied happy return',
    'wonderful experience purchase outstanding best',
    'terrible waste money broken arrived damaged',
    'horrible quality slow delivery disappointed awful',
    'poor value cheap plastic flimsy broken',
    'worst experience refund needed useless junk'
  )
)

# TF-IDF as features
tfidf_wide <- docs |>
  unnest_tokens(word, text) |>
  count(document, word) |>
  bind_tf_idf(word, document, n) |>
  cast_dtm(document, word, tf_idf)

cat('DTM dimensions:', dim(tfidf_wide), '\n')
cat('(rows=documents, cols=unique words)\n')

# TF-IDF strengths and weaknesses summary
criteria <- data.frame(
  Criterion = c('Speed', 'Interpretability', 'Context', 'Word order',
                'Rare words', 'Scalability'),
  TF_IDF    = c('Fast', 'High', 'None', 'Ignored',
                'High IDF (inflated)', 'Excellent'),
  BERT      = c('Slow', 'Low', 'Rich', 'Captured',
                'Handled', 'Moderate')
)

print(criteria, row.names = FALSE)

library(tidytext)
library(dplyr)

# Minimal TF-IDF pipeline
tibble::tibble(
  doc = c('A', 'A', 'B', 'B'),
  text = c('neural networks deep', 'learning training', 'sql database query', 'joins indexes')
) |>
  unnest_tokens(word, text) |>
  count(doc, word) |>
  bind_tf_idf(word, doc, n) |>
  arrange(desc(tf_idf)) |>
  print()