Skip to content
# import required machine learning packages
import torchvision
from gensim.models import KeyedVectors
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize
import numpy as np
import nltk
import pandas as pd
from sklearn.cluster import KMeans  # for words preprocessing
from sklearn.decomposition import TruncatedSVD  # word clustering
from sklearn.feature_extraction.text import TfidfVectorizer  # keywords detection
# for frequency calculation
from sklearn.feature_extraction.text import CountVectorizer
from wordcloud import WordCloud  # for visulization
import matplotlib.pyplot as plt  # for visualization # words structure analyse
import pandas as pd  # data manipulation
import re
import numpy as np  # vectorazation
import codecs
import warnings
from collections import Counter
from pprint import pprint
import scipy as sp
import sklearn.datasets as sk_data  # machine learning package
import sklearn.metrics as metrics  # evaluation methods
import scipy.sparse.linalg as linalg
from sklearn.preprocessing import Normalizer
from sklearn.pipeline import make_pipeline
from sklearn.decomposition import TruncatedSVD
import os.path
import re
import nltk.tokenize.punkt
from sklearn.metrics.pairwise import cosine_similarity
from gensim.models import Word2Vec
from scipy import spatial
import torch
from transformers import T5Tokenizer, T5ForConditionalGeneration
import sentencepiece
from transformers import BertTokenizer, BertModel
from bert_score import score
import concurrent.futures
from typing import List, Tuple
from transformers import BertTokenizer, BertModel
from typing import List, Tuple
from scipy.spatial.distance import cosine
from transformers import BertTokenizerFast
from transformers import AlbertModel
import numpy as np
from typing import List, Tuple
from transformers import BertTokenizerFast, AlbertModel
import torch
from scipy.spatial.distance import cosine
import concurrent.futures
import os