Project: Analyzing Car Reviews with LLMs

Car-ing is sharing, an auto dealership company for car sales and rental, is taking their services to the next level thanks to Large Language Models (LLMs).

As their newly recruited AI and NLP developer, you've been asked to prototype a chatbot app with multiple functionalities that not only assist customers but also provide support to human agents in the company.

The solution should receive textual prompts and use a variety of pre-trained Hugging Face LLMs to respond to a series of tasks, e.g. classifying the sentiment in a car’s text review, answering a customer question, summarizing or translating text, etc.

Install Hugging Face libraries `transformers` and `evaluate`.

!pip install transformers
#!pip uninstall evaluate
!pip install evaluate==0.4.0
from transformers import logging
logging.set_verbosity(logging.WARNING)

# load the car reviews
import pandas as pd

car_review_df = pd.read_csv("data/car_reviews.csv",sep=";")
car_review_df.info()

Task 1: classify the reviews in a sentiment analysis

#use pretrained LLM for classification
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from transformers import pipeline

#text_classifier = pipeline(task="text-classification", model="nlptown/bert-base-multilingual-uncased-sentiment")
reviews = car_review_df['Review'].tolist()
labels = car_review_df['Class'].tolist()
text_classifier = pipeline(task="sentiment-analysis")
#print(car_review_df['Review'].tolist())
predicted_labels = text_classifier(reviews)
predictions = [1 if pred["label"] == "POSITIVE" else 0 for pred in predicted_labels]
print([sent['label'] for sent in predicted_labels])
print("predictions: ", predictions)
true_labels = [1 if truelab == "POSITIVE" else 0 for truelab in labels]
print("true labels: ", true_labels)
#compute accuracy and F1:
import evaluate
from sklearn.metrics import accuracy_score,f1_score

accuracy = evaluate.load("accuracy")
f1 = evaluate.load("f1")

accuracy_result = accuracy.compute(references=true_labels, predictions=predictions)['accuracy']
f1_result = f1.compute(references=true_labels, predictions=predictions)['f1']

#accuracy_result = float(accuracy_score(true_labels, predictions))
#f1_result = float(f1_score(true_labels, predictions))

print(accuracy_result, f1_result)

Task 2: translate the first two sentences of the first review to Spanish and evaluate the translation with BLEU

#Translate English to Spanish:
#first two sentences of the first review
sentences = ['I am very satisfied with my 2014 Nissan NV SL. I use this van for my business deliveries and personal use.']
import evaluate
bleu = evaluate.load("bleu")
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-en-es")
translated_outputs = translator(sentences)
translated_review = translated_outputs[0]['translation_text']
print(translated_review)

with open("data/reference_translations.txt", 'r') as file:
    lines = file.readlines()
references = [line.strip() for line in lines]


#from nltk.translate.bleu_score import sentence_bleu
#bleu_number = sentence_bleu(references, translated_review)
#bleu_score = {'bleu': bleu_number, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6}
bleu_score = bleu.compute(predictions=[translated_review], references=[references])
print("bleu score:",bleu_score)

Task 3: Extract answer to a question from the context of the second review

#Extract answers from the second review:
question = "What did he like about the brand?"
context = reviews[1]

model_ckp = "deepset/minilm-uncased-squad2"
tokenizer = AutoTokenizer.from_pretrained(model_ckp)
inputs = tokenizer(question, context, return_tensors="pt")

from transformers import AutoModelForQuestionAnswering
import torch
model = AutoModelForQuestionAnswering.from_pretrained(model_ckp)
with torch.no_grad():
    outputs = model(**inputs)
start_idx = torch.argmax(outputs.start_logits)
end_idx = torch.argmax(outputs.end_logits) + 1
answer_span = inputs["input_ids"][0][start_idx:end_idx]
answer = tokenizer.decode(answer_span)
print("answer: ",answer)

Task 4: summarize the text of the last review to less than 55 words

#summarize last review
from transformers import AutoModelForSeq2SeqLM
model_name = "t5-small"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
article = reviews[-1]
#input_ids = tokenizer.encode( article, return_tensors="pt", max_length=1024, truncation=True)
#summary_ids = model.generate(input_ids, max_length=53)
#summarized_text = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
summarizer = pipeline("summarization", model=model_name)
outputs = summarizer(article, max_length=53)
summarized_text = outputs[0]['summary_text']
print("summary: ")
print(summarized_text)