# pip install nltk
import nltk
nltk.download('punkt', quiet=True)
text = "This is a sample sentence for testing NLTK library."
tokens = nltk.tokenize.word_tokenize(text)
print(tokens)
# ['This', 'is', 'a', 'sample', 'sentence', 'for', 'testing', 'NLTK', 'library', '.']

# pip install spacy
import spacy
nlp = spacy.load("en_core_web_sm")
text = "This is a sample sentence for testing spaCy library."
doc = nlp(text)
spacy_tokens = [token.text for token in doc]
print(spacy_tokens)
# ['This', 'is', 'a', 'sample', 'sentence', 'for', 'testing', 'spaCy', 'library', '.']

# pip install gensim
import gensim
from gensim.utils import simple_preprocess
text = "This is a sample sentence for testing gensim library."
tokenized = simple_preprocess(text, deacc=True)
print(tokenized)
# ['this', 'is', 'sample', 'sentence', 'for', 'testing', 'spacy', 'library']

# Код для замера скорости токенизации библиотек
import time
import nltk
import spacy
import gensim
import matplotlib.pyplot as plt

nltk.download('punkt', quiet=True)
nlp = spacy.load("en_core_web_sm", disable=["parser", "ner", "lemmatizer", "tagger", 'tok2vec', 'attribute_ruler'])

text = "This is a sample sentence for testing speed of NLP libraries. " * 10000

# Замер NLTK
start = time.time()
tokens = nltk.tokenize.word_tokenize(text)
nltk_time = time.time() - start

# Замер spaCy
start = time.time()
doc = nlp(text)
spacy_tokens = [token.text for token in doc]
spacy_time = time.time() - start

# Замер gensim
start = time.time()
gensim_tokens = list(gensim.utils.tokenize(text))
gensim_time = time.time() - start

libraries = ["nltk", "spacy", "gensim"]
times = [nltk_time, spacy_time, gensim_time]

plt.bar(libraries, times, color=['orange', 'blue', 'green'])
plt.ylabel('Время (секунды)')
plt.noscript('Сравнение скорости токенизации')
plt.show()