This is a test of the 8 billion parameter version of Llama 3.1, the new LLM from Meta, running locally via Ollama. I’m going to use it to try to classify articles on Google Scholar by their title and abstract.

Include some packages:

#devtools::install_github("hauselin/ollamar")
library(conflicted)
library(ollamar)
library(tictoc)
library(scholar)
library(beepr)
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.4     ✔ readr     2.1.5
## ✔ forcats   1.0.0     ✔ stringr   1.5.1
## ✔ ggplot2   3.5.1     ✔ tibble    3.2.1
## ✔ lubridate 1.9.3     ✔ tidyr     1.3.1
## ✔ purrr     1.0.2

Check that Ollama is running:

stopifnot(test_connection()$status_code == 200)
## Ollama local server running

Here are the models I currently have loaded on Ollama:

llms <- list_models()
llms

Install the model we want if it isn’t there (this took about 5 mins on my internet connection)…

if (!"llama3.1:8b" %in% llms$name) {
  tic()
  ollamar::pull("llama3.1:8b", stream = FALSE)
  toc()
  beep(3)
  llms <- list_models()
  llms
}

Grab data from Google Scholar (I’m using my own profile). Note the caching here using RDS files, so that I’m not continually re-requesting the same data (get_publications does its own caching but I’m not sure how, so I’ve switched it off).

get_abs <- Vectorize(function(pid) {
  res <- get_publication_abstract(id = "xrY7bFYAAAAJ", pub_id = pid)
  paste(res, collapse = "\n")
})

if (file.exists("scholar_stash.rds")) {
  papers <- readRDS("scholar_stash.rds")
} else {
  papers <- get_publications("xrY7bFYAAAAJ", flush = TRUE) |>
    mutate(abstract = get_abs(pubid))
  saveRDS(papers, "scholar_stash.rds")
}
papers

Here’s the prompt:

title_abstract_prompt <- function(t, a) {
  sprintf(
    "Instructions: I would like you to classify journal articles by academic discipline and subdiscipline please, based only on the article's title and abstract. If you don't know, answer 'NA'. Be concise, using a small number of words. If the article belongs to more than one category, separate each one with '|'. An example response would be 'psychology|reasoning|evaluation'. Another example response would be 'research methods|qualitative'. Your answer should be the category or categories, with no other text, no quotation marks, do not provide an explanation, and all lower case please. Use British English naming conventions. The input is:\n\nTitle: %s\n\nAbstract: %s",
    t,
    a
  )
}
title_abstract_prompt("Example title", "An example abstract") |> cat()
## Instructions: I would like you to classify journal articles by academic discipline and subdiscipline please, based only on the article's title and abstract. If you don't know, answer 'NA'. Be concise, using a small number of words. If the article belongs to more than one category, separate each one with '|'. An example response would be 'psychology|reasoning|evaluation'. Another example response would be 'research methods|qualitative'. Your answer should be the category or categories, with no other text, no quotation marks, do not provide an explanation, and all lower case please. Use British English naming conventions. The input is:
## 
## Title: Example title
## 
## Abstract: An example abstract

BBC’s Henry Cooke has written a neat article on designing good prompts – I read it after devising the mediocre prompt above by trial and error. Such is life :-)

This function calls Ollama:

classify_paper <- Vectorize(function(t, a) {
  generate("llama3.1:8b", title_abstract_prompt(t, a), output = "text") |> as.vector()
})

Do it for all papers in the Scholar stash (when I was developing the code, I used this line to select two or three papers before running on all):

papers_to_analyse <- papers

I’m using beepr to let me know when it’s done (note I’m using caching again as this can take 1 to 2 minutes per paper):

tic()
if (file.exists("llama_out.rds")) {
  papers_to_analyse <- readRDS("llama_out.rds")
} else {
  papers_to_analyse$res <- classify_paper(papers_to_analyse$title,
                                          papers_to_analyse$abstract)
  saveRDS(papers_to_analyse, "llama_out.rds")
}
toc()
## 0 sec elapsed
beep(2)

Take a look:

result <- papers_to_analyse |>
  mutate(
    res = ifelse(res == "na", NA, res)
  )
result |>
  mutate(title = str_trunc(title, 30),
         res   = str_trunc(res, 30)) |>
  select(title, res)

Now I want to reshape the data to tidy format:

wide_topics_mat <- result$res |> str_split_fixed("\\|", n = Inf)
colnames(wide_topics_mat) <- paste0("t_",1:ncol(wide_topics_mat))
wide_topics <- as_tibble(wide_topics_mat)
wide_topics
res_topics <- bind_cols(result |> select(title), wide_topics) |>
  pivot_longer(
    cols = starts_with("t_"),
    values_to = "class",
    names_prefix = "t_",
    names_to = "topic_num"
  ) |>
  dplyr::filter(class != "") |>
  mutate(class = as_factor(class))

Tidy up the levels a little:

levels(res_topics$class) |> sort()
##  [1] "aphasiology"                "child development"         
##  [3] "cognition"                  "cognitive psychology"      
##  [5] "cognitive science"          "communication studies"     
##  [7] "conditional logic"          "education"                 
##  [9] "educational studies"        "epistemology"              
## [11] "ethics"                     "evaluation"                
## [13] "gender studies"             "healthcare"                
## [15] "healthcare policy"          "human-computer interaction"
## [17] "information technology"     "intelligence testing"      
## [19] "law"                        "linguistics"               
## [21] "literacy"                   "logic"                     
## [23] "mathematics"                "mental health"             
## [25] "mhc"                        "mhc (mental health care)"  
## [27] "neuroscience"               "philosophy"                
## [29] "pragmatics"                 "psycholinguistics"         
## [31] "psychology"                 "public health"             
## [33] "qualitative"                "qualitative research"      
## [35] "quantitative"               "reasoning"                 
## [37] "research methods"           "social sciences"           
## [39] "social work"                "sociology"                 
## [41] "special educational needs"  "speech therapy"            
## [43] "statistics"                 "teacher education"
res_topics_clean <- res_topics |>
  mutate(class = fct_recode(class,
                             "mental health care" = "mhc",
                             "mental health care" = "mhc (mental health care)",
                             "HCI" = "human-computer interaction",
                             "logic" = "conditional logic"))
levels(res_topics_clean$class) |> sort()
##  [1] "aphasiology"               "child development"        
##  [3] "cognition"                 "cognitive psychology"     
##  [5] "cognitive science"         "communication studies"    
##  [7] "education"                 "educational studies"      
##  [9] "epistemology"              "ethics"                   
## [11] "evaluation"                "gender studies"           
## [13] "HCI"                       "healthcare"               
## [15] "healthcare policy"         "information technology"   
## [17] "intelligence testing"      "law"                      
## [19] "linguistics"               "literacy"                 
## [21] "logic"                     "mathematics"              
## [23] "mental health"             "mental health care"       
## [25] "neuroscience"              "philosophy"               
## [27] "pragmatics"                "psycholinguistics"        
## [29] "psychology"                "public health"            
## [31] "qualitative"               "qualitative research"     
## [33] "quantitative"              "reasoning"                
## [35] "research methods"          "social sciences"          
## [37] "social work"               "sociology"                
## [39] "special educational needs" "speech therapy"           
## [41] "statistics"                "teacher education"
res_topics_clean |>
  mutate(title = str_trunc(title, 30)) |>
  select(title, class)

Summarise:

res_topics_clean |>
  group_by(class) |>
  tally() |>
  arrange(desc(n))

Lob it at a cluster analysis:

res_topics_binary <- res_topics_clean |>
  mutate(val = 1) |>
  dplyr::select(-topic_num) |>
  pivot_wider(names_from = "class",
              values_from = val,
              values_fill = 0)

topics_mat <- res_topics_binary |>
  dplyr::select(-title) |>
  as.matrix()
rownames(topics_mat) <- res_topics_binary$title |> str_trunc(50)

dist_mat <- dist(topics_mat, method = "binary")
hc <- hclust(dist_mat, method = "ward.D")

old_par <- par(mar = c(2, 0, 0, 20))
plot(hc |> as.dendrogram(), horiz = TRUE)

par(old_par)

This doesn’t quite look the way I would have done it manually, e.g., I would have put “How people interpret conditionals” alongside “A process model of the understanding of uncertain conditionals” and “Probabilistic theories of reasoning need pragmatics too”. But it’s not horrendous and impressive given that all I had to do was come up with a prompt.