Meertens scrape: name popularity lists

Set up

Libraries

# get packages needed
fpackage.check <- function(packages) { # (c) Jochem Tolsma
  package_list  <- lapply(packages, FUN = function(x) {
    if (!require(x, character.only = TRUE)) {
      install.packages(x, dependencies = TRUE)
      library(x, character.only = TRUE)
    }
  })
}
packages = c("rvest", "httr","polite", "tidyverse", 
             "data.table")
fpackage.check(packages)

Scrape

Function

Get the popularity list for each year and store the name data in a long file.

get_year_names <- function(session, year, file.name){
  #year = 2014
  if (file.exists(file.name)) {
  load(file.name)
  } else {
  yr_path = paste("nvb/topnamen/land/Nederland/", as.character(year),sep="") # set the path for the specific year's webpage
  
  year_session <-nod(session, path = yr_path) # agree changing of the path with the host (assuming I have already "bowed" for the higher-level path)
  
  year_page <- scrape(year_session) # get the page for this year
  
  all_names <- year_page %>%  # parse the page as a table. Turns out this is a list of three tables; we need numbers 2 and 3
    html_table()
  
  save(all_names, file = file.name)# save the raw html code
  }
  
  boy_names <- all_names[[2]] # second table from the list of three
  colnames(boy_names) <- c("rank", "name", "count") #seems like this should be easier...
  boy_names <- boy_names %>% 
    as.data.frame() %>% 
    mutate(is_girl_name = 0) # add a gender dummy
  
  girl_names <- all_names[[3]] # third table from the list of three
  colnames(girl_names) <- c("rank", "name", "count") #seems like this should be easier...
  girl_names <- girl_names %>% 
    as.data.frame() %>% 
    mutate(is_girl_name = 1)  # add a gender dummy
  
  all_names <- rbind(girl_names, boy_names) %>% # combine the two as a new data frame
    mutate(year = year)  
  
  return(all_names)
}

Initialize scrape

Scrape all the popularity lists for year 1920 untill 2014.

# check permissions and introduce myself to the host
session <- bow("https://www.meertens.knaw.nl/nvb/", user_agent =  "Thijmen Jeroense, Radboud Universiteit Nijmegen", delay = 1)
session

all_names <- data.frame( # initialize the empty data frame for the results
  rank = integer(),
  name = character(),
  count = integer(),
  is_girl_name = integer(),
  year = integer()
)

startyear = 1920
endyear = 2014

Scrape

Scrape all the pages and store in the dfs.

#set up archive
#in this I will store the raw html code for futur use. 
archive <- "data_analysis/data/data_raw/meertens_pop_by_year/year"

for(i in endyear:startyear){ # loop over all years
  print(paste("scraping year",i))
  
  file.name <- paste0(archive,"_", i, ".rda")
  
  names_year <- get_year_names(session, i, file.name)
  all_names <- rbind(all_names, names_year)
}

Export results

write.csv(all_names, 
          file = paste("data_analysis/data/data_processed/meertens_scrape/all_names_",as.character(startyear),as.character(endyear),".csv", sep=""),
          row.names=FALSE)

---

Copyright © 2024 Jeroense Thijmen