La loi de Zipf illustrée avec gutenbergr en R 🇫🇷¶

options(tidyverse.quiet = TRUE) # omettre les warnings pour ne pas encombrer le document
library(gutenbergr)
library(tidyverse)
library(gridExtra)
options(gridExtra.quiet = TRUE)
# center plot title
theme_update(plot.title = element_text(hjust = 0.5))

normalize_text <- function(id) {
  # This function takes a book id and returns normalized words vector #
  text <- gutenberg_download(id, mirror = "http://mirrors.xmission.com/gutenberg/")
  body <- text$text
  # tackle non english texts
  body <- iconv(body, "latin1", "UTF-8")
  body.lower <- tolower(body)
  # replace whitespace characters/non words by space for easy split
  body.lower <- gsub("[^a-zA-Z\\s]", " ", body.lower)
  # remove spaces at the beginning and end of texts
  body.lower <- gsub("(^ | +$)", "", body.lower)
  # remove multiple spaces
  body.lower <- gsub(" +", " ", body.lower)
  # remove empty lines
  body.lower <- body.lower[which(body.lower != "")]
  # make the whole text a vector of length 1
  body.v <- paste(body.lower, collapse = " ")
  # split the text to words
  body.words <- unlist(strsplit(body.v, "\\W+"))
  return(body.words)
}

get_freq <- function(text) {
  # This function takes a words vector and returns a dataframe
  # with word, count, relative freq, rank as columns

  # get frequency table and convert to tibble
  text.freq <- table(text)
  sorted_text.freq <- sort(text.freq, decreasing = TRUE)
  df <- as_tibble(sorted_text.freq)
  # reaarange the tibble to get rank info and relative frequency
  df <- df %>%
    rename(word = text) %>% # rename the default text column to word
    mutate(total = sum(n)) %>% # create a total n column
    mutate(
      rank = row_number(), # create rank column
      `term frequency` = n / total # create the relative frequency column
    )
  return(df)
}

make_plots <- function(df, title) {
  # This function takes a df and title
  # with 3 plots related to Zipf's law as output
  # Plot scatterplot of top20 words (word vs freq)
  p.top20 <- ggplot(df[1:20, ], aes(x = reorder(word, -n),
  y = n, group = 1, label = word)) +
    geom_text(check_overlap = TRUE) + # declare labels
    ggtitle(paste("Top 20 most common words in", title)) +
    xlab("Word") +
    ylab("Absolute frequency")
  # plot the non-log relative frequency distribution
  p.nonlog <- ggplot(df, aes(`term frequency`)) +
    geom_histogram(show.legend = FALSE) +
    xlim(NA, 0.0005) +
    ggtitle(paste("Relative frequency distribution of words in", title)) +
    xlab("Relative frequency") +
    ylab("Percentage")
  # plot the log zip's distribution
  p.log <- ggplot(df, aes(rank, `term frequency`)) +
    geom_line(size = 1.1, alpha = 0.8, show.legend = FALSE) +
    scale_x_log10() +
    scale_y_log10() +
    ggtitle(paste("Standard plot of Zipf's law using words in", title)) +
    xlab("Relative frequency (log)") +
    ylab("Rank")
  return(list(p.top20, p.nonlog, p.log))
}

main <- function(metas) {
  # This function asks the user to choose a book id
  # displays top 20 most common words
  # plot and save 3 plots illustrating the zipf's law

  welcome_message(metas)
  book.id <- readline()
  # convert the book id into integer
  book.id <- as.integer(book.id)
  # check id validity
  if (!(book.id %in% metas$gutenberg_id)) {
    cat("\n\n\U0001f609!Book id not found!\U0001f609
    Please type an id existing in our database.\n\n")
    main(metas)
  } else {
    book.chosen <- metas %>% filter(gutenberg_id == book.id)
    book.title <- book.chosen$title
    # normalize the long text and convert to words
    text <- normalize_text(book.id)
    # get frequency table as dataframe
    df.freq <- get_freq(text)
    # show top 20 most common words
    cat(paste0("\n\n\nThe book you chose was ", book.title, ".\n\n\n"))
    cat("Here is the list of the top 20 most frequent words in this text.\n\n\n")
    print(df.freq[1:20, ])
    # make plots
    plots <- make_plots(df.freq, book.title)
    # combine and show all 3 graphes
    grid.arrange(plots[[1]], plots[[2]], plots[[3]], nrow = 3)
    # save the plot we don't use it for the jupyter notebook illustrated on website
#     g <- arrangeGrob(plots[[1]], plots[[2]], plots[[3]], nrow = 3)
#     ggsave(file = paste0("zipf_", book.title, "_xiaoouWang.png"), g)
  }
}