import re
pre_processed = re.sub(r'[^\w\s]', '', text)  # keep only whitespace & words

tokens_1 = pre_processed.split()
# ['إن', 'التاريخ', 'العسكري', 'سوف', 'يتوقف', ...]

PATTERN = r"\w+"
tokens_2 = re.findall(PATTERN, pre_processed)

import nltk
nltk.download('punkt_tab')
from nltk.tokenize import word_tokenize
tokens_3 = word_tokenize(pre_processed)

def segment(text):
    segmenter = nltk.data.load("tokenizers/punkt/english.pickle")
    sents = segmenter.tokenize(text)
    for sent in sents:
        print(sent)

from nltk.corpus import stopwords
stopwords = stopwords.words('arabic')
X_set = {w for w in word_tokenize(X) if w not in stopwords}