#Unicode Normalization
from camel_tools.utils.normalize import normalize_unicode
 
sentence = 'ﷺ'
print(sentence)
 
sent_norm = normalize_unicode(sentence)
print(sent_norm)

ﷺ
صلى الله عليه وسلم

#Orthographic Normalization
from camel_tools.utils.normalize import normalize_alef_maksura_ar
from camel_tools.utils.normalize import normalize_alef_ar
from camel_tools.utils.normalize import normalize_teh_marbuta_ar
 
sentence = "هل ذهبت إلى المكتبة؟"
print(sentence)
 
sent_norm = normalize_alef_ar(sentence)
sent_norm = normalize_alef_maksura_ar(sent_norm)
sent_norm = normalize_teh_marbuta_ar(sent_norm)
print(sent_norm)

هل ذهبت إلى المكتبة؟
هل ذهبت الي المكتبه؟

#Dediacritization
from camel_tools.utils.dediac import dediac_ar
 
sentence = "هَلْ ذَهَبْتَ إِلَى المَكْتَبَةِ؟"
print(sentence)
 
sent_dediac = dediac_ar(sentence)
print(sent_dediac)

هَلْ ذَهَبْتَ إِلَى المَكْتَبَةِ؟
هل ذهبت إلى المكتبة؟

#Tokenaization
from camel_tools.tokenizers.word import simple_word_tokenize
 
sentence = "هَلْ ذَهَبْتَ إِلَى المَكْتَبَةِ؟"
print(sentence)
 
sent_split = simple_word_tokenize(sentence)
print(sent_split)

هَلْ ذَهَبْتَ إِلَى المَكْتَبَةِ؟
['هَلْ', 'ذَهَبْتَ', 'إِلَى', 'المَكْتَبَةِ', '؟']

#Translitration
from camel_tools.utils.charmap import CharMapper
 
sentence = "ذهبت إلى المكتبة."
print(sentence)
 
# Instantiate the builtin ar2bw (Arabic to Buckwalter) CharMapper
ar2bw = CharMapper.builtin_mapper('ar2bw')
 
sent_bw = ar2bw(sentence)
print(sent_bw)

ذهبت إلى المكتبة.
*hbt <lY Almktbp.

#Morphological Analysis
from camel_tools.morphology.database import MorphologyDB
from camel_tools.morphology.analyzer import Analyzer
 
db = MorphologyDB.builtin_db()
analyzer = Analyzer(db)
 
# To analyze a word, we can use the analyze() method
analyses = analyzer.analyze('وسيكتبونها')
print(analyses)

[{'diac': 'وَسَيَكْتُبُونَها', 'lex': 'كَتَب', 'bw': 'وَ/PART+سَ/FUT_PART+يَ/IV3MP+كْتُب/IV+ُونَ/IVSUFF_SUBJ:MP_MOOD:I+ها/IVSUFF_DO:3FS', 'gloss': '[part.]_+_will_+_they_(people)+write+it;them;her', 'pos': 'verb', 'prc3': '0', 'prc2': 'wa_part', 'prc1': 'sa_fut', 'prc0': '0', 'per': '3', 'asp': 'i', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '3fs_dobj', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 'p', 'd3seg': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'caphi': 'w_a_s_a_y_a_k_t_u_b_uu_n_a_h_aa', 'd1tok': 'وَ+_سَيَكْتُبُونَها', 'd2tok': 'وَ+_سَ+_يَكْتُبُونَها', 'pos_logprob': -1.023208, 'd3tok': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'd2seg': 'وَ+_سَ+_يَكْتُبُونَها', 'pos_lex_logprob': -3.648503, 'num': 'p', 'ud': 'PART+AUX+VERB+PRON', 'gen': 'm', 'catib6': 'PRT+PRT+VRB+NOM', 'root': 'ك.ت.ب', 'bwtok': 'وَ+_سَ+_يَ+_كْتُب_+ُونَ_+ها', 'pattern': 'وَسَيَ1ْ2ُ3ُونَها', 'lex_logprob': -3.648503, 'atbtok': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'atbseg': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'd1seg': 'وَ+_سَيَكْتُبُونَها', 'stem': 'كْتُب', 'stemgloss': 'write', 'stemcat': 'IV'}, {'diac': 'وَسَيَكْتُبُونَها', 'lex': 'كَتَب', 'bw': 'وَ/CONJ+سَ/FUT_PART+يَ/IV3MP+كْتُب/IV+ُونَ/IVSUFF_SUBJ:MP_MOOD:I+ها/IVSUFF_DO:3FS', 'gloss': 'and_+_will_+_they_(people)+write+it;them;her', 'pos': 'verb', 'prc3': '0', 'prc2': 'wa_conj', 'prc1': 'sa_fut', 'prc0': '0', 'per': '3', 'asp': 'i', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '3fs_dobj', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 'p', 'd3seg': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'caphi': 'w_a_s_a_y_a_k_t_u_b_uu_n_a_h_aa', 'd1tok': 'وَ+_سَيَكْتُبُونَها', 'd2tok': 'وَ+_سَ+_يَكْتُبُونَها', 'pos_logprob': -1.023208, 'd3tok': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'd2seg': 'وَ+_سَ+_يَكْتُبُونَها', 'pos_lex_logprob': -3.648503, 'num': 'p', 'ud': 'CCONJ+ADP+VERB+PRON', 'gen': 'm', 'catib6': 'PRT+PRT+VRB+NOM', 'root': 'ك.ت.ب', 'bwtok': 'وَ+_سَ+_يَ+_كْتُب_+ُونَ_+ها', 'pattern': 'وَسَيَ1ْ2ُ3ُونَها', 'lex_logprob': -3.648503, 'atbtok': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'atbseg': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'd1seg': 'وَ+_سَيَكْتُبُونَها', 'stem': 'كْتُب', 'stemgloss': 'write', 'stemcat': 'IV'}, {'diac': 'وَسَيُكْتِبُونَها', 'lex': 'أَكْتَب', 'bw': 'وَ/SUB_CONJ+سَ/FUT_PART+يُ/IV3MP+كْتِب/IV+ُونَ/IVSUFF_SUBJ:MP_MOOD:I+ها/IVSUFF_DO:3FS', 'gloss': 'while_+_will_+_they_(people)+dictate;make_write+it;them;her', 'pos': 'verb', 'prc3': '0', 'prc2': 'wa_sub', 'prc1': 'sa_fut', 'prc0': '0', 'per': '3', 'asp': 'i', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '3fs_dobj', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 'p', 'd3seg': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'caphi': 'w_a_s_a_y_u_k_t_i_b_uu_n_a_h_aa', 'd1tok': 'وَ+_سَيُكْتِبُونَها', 'd2tok': 'وَ+_سَ+_يُكْتِبُونَها', 'pos_logprob': -1.023208, 'd3tok': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'd2seg': 'وَ+_سَ+_يُكْتِبُونَها', 'pos_lex_logprob': -99.0, 'num': 'p', 'ud': 'CCONJ+ADP+VERB+PRON', 'gen': 'm', 'catib6': 'PRT+PRT+VRB+NOM', 'root': 'ك.ت.ب', 'bwtok': 'وَ+_سَ+_يُ+_كْتِب_+ُونَ_+ها', 'pattern': 'وَسَيُ1ْ2ِ3ُونَها', 'lex_logprob': -99.0, 'atbtok': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'atbseg': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'd1seg': 'وَ+_سَيُكْتِبُونَها', 'stem': 'كْتِب', 'stemgloss': 'dictate;make_write', 'stemcat': 'IV_yu'}, {'diac': 'وَسَيَكْتُبُونَها', 'lex': 'كَتَب', 'bw': 'وَ/SUB_CONJ+سَ/FUT_PART+يَ/IV3MP+كْتُب/IV+ُونَ/IVSUFF_SUBJ:MP_MOOD:I+ها/IVSUFF_DO:3FS', 'gloss': 'while_+_will_+_they_(people)+write+it;them;her', 'pos': 'verb', 'prc3': '0', 'prc2': 'wa_sub', 'prc1': 'sa_fut', 'prc0': '0', 'per': '3', 'asp': 'i', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '3fs_dobj', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 'p', 'd3seg': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'caphi': 'w_a_s_a_y_a_k_t_u_b_uu_n_a_h_aa', 'd1tok': 'وَ+_سَيَكْتُبُونَها', 'd2tok': 'وَ+_سَ+_يَكْتُبُونَها', 'pos_logprob': -1.023208, 'd3tok': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'd2seg': 'وَ+_سَ+_يَكْتُبُونَها', 'pos_lex_logprob': -3.648503, 'num': 'p', 'ud': 'CCONJ+ADP+VERB+PRON', 'gen': 'm', 'catib6': 'PRT+PRT+VRB+NOM', 'root': 'ك.ت.ب', 'bwtok': 'وَ+_سَ+_يَ+_كْتُب_+ُونَ_+ها', 'pattern': 'وَسَيَ1ْ2ُ3ُونَها', 'lex_logprob': -3.648503, 'atbtok': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'atbseg': 'وَ+_سَ+_يَكْتُبُونَ_+ها', 'd1seg': 'وَ+_سَيَكْتُبُونَها', 'stem': 'كْتُب', 'stemgloss': 'write', 'stemcat': 'IV'}, {'diac': 'وَسَيُكْتِبُونَها', 'lex': 'أَكْتَب', 'bw': 'وَ/CONJ+سَ/FUT_PART+يُ/IV3MP+كْتِب/IV+ُونَ/IVSUFF_SUBJ:MP_MOOD:I+ها/IVSUFF_DO:3FS', 'gloss': 'and_+_will_+_they_(people)+dictate;make_write+it;them;her', 'pos': 'verb', 'prc3': '0', 'prc2': 'wa_conj', 'prc1': 'sa_fut', 'prc0': '0', 'per': '3', 'asp': 'i', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '3fs_dobj', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 'p', 'd3seg': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'caphi': 'w_a_s_a_y_u_k_t_i_b_uu_n_a_h_aa', 'd1tok': 'وَ+_سَيُكْتِبُونَها', 'd2tok': 'وَ+_سَ+_يُكْتِبُونَها', 'pos_logprob': -1.023208, 'd3tok': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'd2seg': 'وَ+_سَ+_يُكْتِبُونَها', 'pos_lex_logprob': -99.0, 'num': 'p', 'ud': 'CCONJ+ADP+VERB+PRON', 'gen': 'm', 'catib6': 'PRT+PRT+VRB+NOM', 'root': 'ك.ت.ب', 'bwtok': 'وَ+_سَ+_يُ+_كْتِب_+ُونَ_+ها', 'pattern': 'وَسَيُ1ْ2ِ3ُونَها', 'lex_logprob': -99.0, 'atbtok': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'atbseg': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'd1seg': 'وَ+_سَيُكْتِبُونَها', 'stem': 'كْتِب', 'stemgloss': 'dictate;make_write', 'stemcat': 'IV_yu'}, {'diac': 'وَسَيُكْتِبُونَها', 'lex': 'أَكْتَب', 'bw': 'وَ/PART+سَ/FUT_PART+يُ/IV3MP+كْتِب/IV+ُونَ/IVSUFF_SUBJ:MP_MOOD:I+ها/IVSUFF_DO:3FS', 'gloss': '[part.]_+_will_+_they_(people)+dictate;make_write+it;them;her', 'pos': 'verb', 'prc3': '0', 'prc2': 'wa_part', 'prc1': 'sa_fut', 'prc0': '0', 'per': '3', 'asp': 'i', 'vox': 'a', 'mod': 'i', 'stt': 'na', 'cas': 'na', 'enc0': '3fs_dobj', 'rat': 'n', 'source': 'lex', 'form_gen': 'm', 'form_num': 'p', 'd3seg': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'caphi': 'w_a_s_a_y_u_k_t_i_b_uu_n_a_h_aa', 'd1tok': 'وَ+_سَيُكْتِبُونَها', 'd2tok': 'وَ+_سَ+_يُكْتِبُونَها', 'pos_logprob': -1.023208, 'd3tok': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'd2seg': 'وَ+_سَ+_يُكْتِبُونَها', 'pos_lex_logprob': -99.0, 'num': 'p', 'ud': 'PART+AUX+VERB+PRON', 'gen': 'm', 'catib6': 'PRT+PRT+VRB+NOM', 'root': 'ك.ت.ب', 'bwtok': 'وَ+_سَ+_يُ+_كْتِب_+ُونَ_+ها', 'pattern': 'وَسَيُ1ْ2ِ3ُونَها', 'lex_logprob': -99.0, 'atbtok': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'atbseg': 'وَ+_سَ+_يُكْتِبُونَ_+ها', 'd1seg': 'وَ+_سَيُكْتِبُونَها', 'stem': 'كْتِب', 'stemgloss': 'dictate;make_write', 'stemcat': 'IV_yu'}]

#Morphological Generation
from camel_tools.morphology.database import MorphologyDB
from camel_tools.morphology.generator import Generator
 
# Initialize database in generation mode
db = MorphologyDB.builtin_db(flags='g')
 
# Create generator instance
generator = Generator(db)
 
# Specify lemma and features to generate for
lemma = 'مُوَظَّف'
features = {
    'pos': 'noun',
    'gen': 'm',
    'num': 'p'
}
 
# Generate analyses for lemma and features
analyses = generator.generate(lemma, features)
 
for diac in set([a['diac'] for a in analyses]):
    print(diac)

مُوَظَّفُونَ
مُوَظَّفِي
مُوَظَّفُو
مُوَظَّفِينَ

#Morphological Reinflection
from camel_tools.morphology.database import MorphologyDB
from camel_tools.morphology.reinflector import Reinflector
 
# Initialize database in reinflection mode
db = MorphologyDB.builtin_db(flags='r')
 
# Create reinflector instance
reinflector = Reinflector(db)
 
# Specify word and features to generate for
word = 'شوارع'
features = {
    'num': 'd',
    'prc1': 'bi_prep'
}
 
# Generate analyses for lemma and features
analyses = reinflector.reinflect(word, features)
 
for diac in set([a['diac'] for a in analyses]):
    print(diac)

بِشارِعَيْنِ
بِشارِعَيْ

#Morphological Disambiguation
from camel_tools.disambig.mle import MLEDisambiguator
 
mle = MLEDisambiguator.pretrained()
sentence = 'سوف نقرأ الكتب'.split()
 
disambig = mle.disambiguate(sentence)
 
# Let's, for example, use the top disambiguations to generate a diacritized
# version of the above sentence.
# Note that, in practice, you'll need to make sure that each word has a
# non-zero list of analyses.
diacritized = [d.analyses[0].analysis['diac'] for d in disambig]
pos_tags = [d.analyses[0].analysis['pos'] for d in disambig]
lemmas = [d.analyses[0].analysis['lex'] for d in disambig]
 
for triplet in zip(diacritized, pos_tags, lemmas):
    print(triplet)

('سَوْفَ', 'part_fut', 'سَوْفَ')
('نَقْرَأ', 'verb', 'قَرَأ')
('الكُتُبِ', 'noun', 'كِتاب')

#POS Tagging
from camel_tools.disambig.mle import MLEDisambiguator
from camel_tools.tagger.default import DefaultTagger
 
mled = MLEDisambiguator.pretrained()
tagger = DefaultTagger(mled, 'pos')
 
pos_tags = tagger.tag('ذهبت الى المدرسة'.split())
print(pos_tags)

['verb', 'prep', 'noun']

#Morphological Tokenization
from camel_tools.disambig.mle import MLEDisambiguator
from camel_tools.tokenizers.morphological import MorphologicalTokenizer
 
# Initialize disambiguators
mle_msa = MLEDisambiguator.pretrained('calima-msa-r13')
mle_egy = MLEDisambiguator.pretrained('calima-egy-r13')
 
# We expect a sentence to be whitespace/punctuation tokenized beforehand.
# We provide a simple whitespace and punctuation tokenizer as part of camel_tools.
# See camel_tools.tokenizers.word.simple_word_tokenize.
sentence_msa = ['فتنفست', 'الصعداء']
sentence_egy = ['وكاتباله', 'مكتوبين']
 
# Create different morphological tokenizer instances
msa_d3_tokenizer = MorphologicalTokenizer(disambiguator=mle_msa, scheme='d3tok', split='True', diac=True)
msa_atb_tokenizer = MorphologicalTokenizer(disambiguator=mle_msa, scheme='atbtok')
msa_bw_tokenizer = MorphologicalTokenizer(disambiguator=mle_msa, scheme='bwtok')
egy_bw_tokenizer = MorphologicalTokenizer(disambiguator=mle_egy, scheme='bwtok')
 
# Generate tokenizations
# Note that our Egyptian resources currently provide bwtok tokenization only.
msa_d3_tok = msa_d3_tokenizer.tokenize(sentence_msa)
msa_atb_tok = msa_atb_tokenizer.tokenize(sentence_msa)
msa_bw_tok = msa_bw_tokenizer.tokenize(sentence_msa)
egy_bw_tok = egy_bw_tokenizer.tokenize(sentence_egy)
 
# Print results
print('D3 tokenization (MSA):', msa_d3_tok)
print('ATB tokenization (MSA):', msa_atb_tok)
print('BW tokenization (MSA):', msa_bw_tok)
print('BW tokenization (EGY):', egy_bw_tok)

D3 tokenization (MSA): ['فَ+', 'تَنَفَّسَت', 'ال+', 'صُعَداءَ']
ATB tokenization (MSA): ['ف+_تنفست', 'الصعداء']
BW tokenization (MSA): ['ف+_تنفس_+ت', 'ال+_صعداء']
BW tokenization (EGY): ['و+_كاتب_+ة_+ل_+ه', 'مكتوب_+ين']

#Sentiment Analysis
from camel_tools.sentiment import SentimentAnalyzer
 
sa = SentimentAnalyzer.pretrained()
 
# Predict the sentiment of a single sentence
sentiment = sa.predict_sentence('أنا بخير')
print(sentiment)
 
# Predict the sentiment of multiple sentences
sentences = [
    'أنا بخير',
    'أنا لست بخير'
]
sentiments = sa.predict(sentences)
print(sentiments)

C:\Users\Asus\anaconda3\lib\site-packages\numpy\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:
C:\Users\Asus\anaconda3\lib\site-packages\numpy\.libs\libopenblas.4SP5SUA7CBGXUEOC35YP2ASOICYYEQZZ.gfortran-win_amd64.dll
C:\Users\Asus\anaconda3\lib\site-packages\numpy\.libs\libopenblas64__v0.3.21-gcc_10_3_0.dll
  warnings.warn("loaded more than 1 DLL from .libs:"
positive
['positive', 'negative']

#Named Entity Recognition
from camel_tools.ner import NERecognizer
 
ner = NERecognizer.pretrained()
 
# Predict the labels of a single sentence.
# The sentence must be pretokenized by whitespace and punctuation.
sentence = 'إمارة أبوظبي هي إحدى إمارات دولة الإمارات العربية المتحدة السبع .'.split()
labels = ner.predict_sentence(sentence)
 
# Print the list of token-label pairs
print(list(zip(sentence, labels)))

Some weights of the model checkpoint at C:\Users\Asus\AppData\Roaming\camel_tools\data\ner\arabert were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']
- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
[('إمارة', 'O'), ('أبوظبي', 'B-LOC'), ('هي', 'O'), ('إحدى', 'O'), ('إمارات', 'O'), ('دولة', 'O'), ('الإمارات', 'B-LOC'), ('العربية', 'I-LOC'), ('المتحدة', 'I-LOC'), ('السبع', 'O'), ('.', 'O')]

#install library first 
# pip install pyaramorph
import pyaramorph
 
analyzer = pyaramorph.Analyzer()
 
print(analyzer.analyze_word('كتب'))
print(analyzer.analyze_word('ktb'))
print(analyzer.analyze_word('katab'))
print(analyzer.analyze_text('AlktAb'))
 
print(analyzer.analyze_word('الكتاب'))
print(analyzer.analyze_text('الكتب العربية الكبيرة'))

loading dictPrefixes ... loaded 299 entries
loading dictStems ... loaded 38600 lemmas and 82158 entries
loading dictSuffixes ... loaded 618 entries
[]
['    solution: (كَتَبَ kataba) [katab-u_1]\n         pos: katab/VERB_PERFECT+a/PVSUFF_SUBJ:3MS\n       gloss: ___ + write + he/it <verb>\n', '    solution: (كُتِبَ kutiba) [katab-u_1]\n         pos: kutib/VERB_PERFECT+a/PVSUFF_SUBJ:3MS\n       gloss: ___ + be written;be fated;be destined + he/it <verb>\n', '    solution: (كُتُب kutub) [kitAb_1]\n         pos: kutub/NOUN\n       gloss: ___ + books + ___\n']
[]
[]
[]
[['analysis for: الكتب Alktb', '    solution: (الكُتُب Alkutub) [kitAb_1]\n         pos: Al/DET+kutub/NOUN\n       gloss: the + books + ___\n'], ['analysis for: العربية AlErbyp', '    solution: (العَرَبِيَّة AlEarabiy~ap) [Earabiy~_1]\n         pos: Al/DET+Earabiy~/ADJ+ap/NSUFF_FEM_SG\n       gloss: the + Arabic;Arab + [fem.sg.]\n', '    solution: (العَرَبِيَّة AlEarabiy~ap) [Earabiy~_2]\n         pos: Al/DET+Earabiy~/ADJ+ap/NSUFF_FEM_SG\n       gloss: the + Arab + [fem.sg.]\n'], ['analysis for: الكبيرة Alkbyrp', '    solution: (الكَبِيرَة Alkabiyrap) [kabiyr_1]\n         pos: Al/DET+kabiyr/ADJ+ap/NSUFF_FEM_SG\n       gloss: the + large;great;important + [fem.sg.]\n', '    solution: (الكَبِيرَة Alkabiyrap) [kabiyrap_1]\n         pos: Al/DET+kabiyr/NOUN+ap/NSUFF_FEM_SG\n       gloss: the + grave offense + [fem.sg.]\n']]