-iw$dZedkrddlZejdddlZddlmZddlm Z ddl Z ddl m ZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZGddZdS)zm Arabic text morphological analyzer. Provides routins to alanyze text. Can treat text as verbs or as nouns. __main__Nz..) analex_const) stem_noun) stem_verb) stem_unknown) stem_stop)stem_pounct_const)disambig)wordcase) stemmedwordcache)cache_pickledbceZdZdZ d&dZdZdZdZdZd'd Z d Z d Z d Z dZ dZdZdZd(dZdZdZd)dZdZdZd'dZdZdZdZdZdZd Zd!Zd"Ze d*d#Z!d$Z"e d%Z#dS)+Analexz} Arabic text morphological analyzer. Provides routins to alanyze text. Can treat text as verbs or as nouns. FTcZtj|_tj|_t j|_tj |_ ||_ |o||_ d|_|j r#tj|_|j rt'j|_d|_d|_d|_dt4j}t9jddt4jdt8j|_d|_ tCj"dtBj#|_$d|_%||_&|rtOj(||_'nd |_'d|_)d|_*i|_+d S) z) Create Analex instance. TFi'rz([\wz\s]+)wordfreqN),r NounStemmer nounstemmerr VerbStemmer verbstemmerrUnknownStemmerunknownstemmerstem_stopwordsStopWordStemmerstopwordsstemmerallow_tag_guessingallow_disambiguationallow_syntax_lastmark naftawayhwordtag WordTaggertaggerr Disambiguator disambiguatordebuglimit wordcounterjoinarabyTASHKEELrecompileUNICODEclause_patternpartial_vocalization_supportwordfreqdictionaryclassWordFreqDictionaryWORDFREQ_DICTIONARY_INDEXrallow_cache_use cache_pathrCachefully_vocalized_input error_codewordfreq_cache)selfr7rr markss L/var/www/html/speakWrite/venv/lib/python3.11/site-packages/qalsadi/analex.py__init__zAnalex.__init__Is%022$022*9;; . > @ @#5%9$O=O!&*"  " 9#+6688DK  $ :!)!7!9!9D   ''!jj ggen5555 7   -1)0B /I    %$  Z00DJJDJ&+" chd|_d|_d|_d|_d|_d|_d|_dS)z1 Delete instance and clear cache N)rrrrrr%r'r<s r>__del__zAnalex.__del__s@ " $ !r@c0|xjdz c_|jS)zqcount input words. Used just for profiling and tests. @return: counter. @rtype: integer. r)r*rBs r> count_wordzAnalex.count_words! Ar@c|||jvr |j|S|j||}||j|<|S)z' Return word_frequency )r;rget_freq)r<wordwordtypefreqs r>rGzAnalex.get_freqsI 4& & &&t, ,=))$99D(,D  %Kr@cnd|jd|jS)z? Return error code when word is not recognized Nz-V)rget_error_coderrBs r>rMzAnalex.get_error_codes=   + + - - - -   + + - - -  r@rc*tj|S)z Tokenize text into words @param text: the input text. @type text: unicode. @return: list of words. @rtype: list. )r,tokenizer<texts r>rOzAnalex.tokenizes~d###r@c"|r|j|}|rnd}g}|D]e}|j|s/|dkr||d}:||xx|z cc<K|||dz }f|SgSgS)z Split Text into clauses @param text: input text @type text: unicode @return: list of clauses @rtype: list of unicode rr)r1splitmatchappend)r<rQ list_phrasejnewlistphrs r>split_into_phraseszAnalex.split_into_phrasess  -33D99K &  C.44S99 q55#NN3/// !AA$AJJJ#-JJJJs+++Q  r@cH||}d|D}|S)z Tokenize text into words, after treatement. @param text: the input text. @type text: unicode. @return: list of words. @rtype: list. cg|]}||Sr^).0rHs r> z(Analex.text_tokenize..s888d48T888r@)rO)r<rQ list_words r> text_tokenizezAnalex.text_tokenizes.MM$'' 88i888 r@c|||_|j||j|dS)z Set the debug attribute to allow printing internal analysis results. @param debug: the debug value. @type debug: True/False. N)r(r set_debugr)r<r(s r>rdzAnalex.set_debugs?   ""5))) ""5)))))r@cxd|_|j|jdS)zX Enable the syntaxic last mark attribute to allow use of I'rab harakat. TN)r!renable_syntax_lastmarkrrBs r>rfzAnalex.enable_syntax_lastmarks<&*" //111 //11111r@cd|_dS)zS Enable fully vocalized input in case of vocalized text analysis. TN)r9rBs r>enable_fully_vocalized_inputz#Analex.enable_fully_vocalized_input s&*"""r@cxd|_|j|jdS)zY Disable the syntaxic last mark attribute to allow use of I'rab harakat. FN)r!rdisable_syntax_lastmarkrrBs r>rjzAnalex.disable_syntax_lastmarks<&+" 00222 0022222r@c||_dS)z Set the number of word treated in text. @param limit: the word number limit. @type limit: integer. N)r))r<r)s r> set_limitzAnalex.set_limits  r@Nc||_dS)z: Use a cache system to qalsadi to be used Nr)r<cachers r> set_cacherzAnalex.set_cacher!s r@cnd|_|js&t|j|_dSdS)zA Allow the analex to use Cache to reduce calcul. TN)r6r cachemanagerr8r7rBs r>enable_allow_cache_usezAnalex.enable_allow_cache_use's; $z =%++DO<disable_allow_cache_usezAnalex.disable_allow_cache_use/s %r@allc ||}dgt|z}|jrM|j|}t|t|krdgt|z}|jr=|j||}t|t|kr|}g}|dkrttt|d|j D]]}||}| ||} | || } d| D} | | ^n|dkrI|d|j D]8}||} d| D} | | 9nN|dkrH|d|j D]8}||} d | D} | | 9|S) a0 Analyze text morphologically. @param text: the input text. @type text: unicode. @param mode: the mode of analysis as 'verbs', 'nouns', or 'all'. @type mode: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. rnvruNc6g|]}tj|Sr^r StemmedWordr_ws r>r`z%Analex.check_text..b0)))34K+A..)))r@nounsc6g|]}tj|Sr^ryr{s r>r`z%Analex.check_text..jr}r@verbsc6g|]}tj|Sr^ryr{s r>r`z%Analex.check_text..rr}r@)rblenrr% word_taggingr r'disambiguate_wordslistranger)rE check_wordrVcheck_word_as_nouncheck_word_as_verb) r<rQmoderalist_guessed_tag newwordlist resulted_datairH guessedtag one_data_liststemmed_one_data_lists r> check_textzAnalex.check_text5sh&&t,, 4#i..0  " ;#{77 BB #$$I66 %)6C NN#:  $ (,??+K;3y>>11'   5==%Il l$; < <==>> < < |!!!-a0 $j A A ))8E)))%$$%:;;;; <W__!,DJ,/ < < $ 7 7 = = ))8E)))%$$%:;;;;  <W__!,DJ,/ < < $ 7 7 = = ))8E)))%$$%:;;;;r@cltj|rdS|D]}|dvrdS|tjvrdSdSz tag words as verbs or nouns according to some features Some letters are forbiden in some types like TehMarbuta in verbs stopuإةnonverbr stopwordsis_stopr,TANWINr<rHcs r> light_tagzAnalex.light_tagyY  T " " 6 ! !AF{{ yyEL   yy!rr@cltj|rdS|D]}|dvrdS|tjvrdSdSrrrs r> light_tag2zAnalex.light_tag2rr@c tj|}|}tj|}tj|}|jr6|j|r|j|}ng}|||z }tj |r3| |dkr|| |z }|j r|j |s|j |r|||z }|j |s|j |r|||z }nb| |dkr|||z }| |dkr|||z }t'|dkr|||z }|||}||||j}||}d|D}|jr|j|||jr|||}t'|dkr`|}|t=j|d|||||d|zdd d | |dd d |S) z Analyze one word morphologically as verbs @param word: the input word. @type word: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. rrnonnounrcg|] }|j Sr^)__dict__r{s r>r`z%Analex.check_word..s%H%H%HQaj%H%H%Hr@rrrrz%sunknownr) rHaffixstemoriginallemma vocalized semivocalizedtagstyperoottemplaterJsyntax)!r, strip_tatweelstrip_tashkeel strip_harakatr6ris_already_checked get_checkedcheck_word_as_pounct is_arabicwordrcheck_word_as_stopwordrr% has_verb_tagis_stopword_tagr has_noun_tagrrcheck_word_as_unknowncheck_normalized check_shaddar9add_word_frequency add_checkedr2check_partial_vocalizedrMrVr WordCaserG) r<rHrword_vocalisedword_nmword_nm_shaddardata_list_to_serializer:s r>rzAnalex.check_wordsX"4((&t,,,T22  : HDJ$A$A'$J$J: H J227;;MMM T66w?? ?M "7++ J>>$''611!T%@%@%I%IIM*J{//"J44Z@@J&)@)@)I)II {//"J44Z@@J&)@)@)I)II ~~d++y88%)@)@)I)II ~~d++y88%)@)@)I)II =!!Q&&!;!;G!D!DD !11'=IIM!-- t/IM !33MBBM&I%H-%H%H%H "# H &&w0FGGG  , X 88WWM }   " ",,..J  ! $!1 $$(!%%))- $z 1 ) "$& $ dI > >"$   &r@c.||dS)z Analyze text morphologically as nouns @param text: the input text. @type text: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. r~rrPs r>check_text_as_nounszAnalex.check_text_as_nounstW---r@c.||dS)z Analyze text morphologically as verbs @param text: the input text. @type text: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. rrrPs r>check_text_as_verbszAnalex.check_text_as_verbsrr@ct|D]\}}|j}|j}|dkrd}n|dkrd}n |dkrd}nd}|r|jr<|j||r!|j|||_n@|||}|jr|j|||||_|||<|S)uO If the entred word is like the found word in dictionary, to treat some normalized cases, the analyzer return the vocalized like words ُIf the word is ذئب, the normalized form is ذءب, which can give from dictionary ذئبـ ذؤب. this function filter normalized resulted word according the given word, and give ذئب. @param resulted_data: the founded resulat from dictionary. @type resulted_data: list of dict. @return: list of dictionaries of analyzed words with tags. @rtype: list. freqverbverbfreqnounnoun freqstopwordstopwordr) enumeraterrJr6rexists_cache_freqrGadd_freq)r<rritemrfreqtyperIrJs r>rzAnalex.add_word_frequencys&!//( $( $GAt }H yH:%%!Z''!^++% %'%DJ,H,Hh--%!% 3 3Hh G GDII ==8<rzAnalex.check_word_as_stopwordLs$66t<<rzAnalex.check_word_as_pounctVs #" " >>     " "! $!1 "$( $%) ( ( !"$ "     $  D,999:  " "! $!1 "$(!%%) 1 >tAw G O ( !"$ "     $r@c6|j|S)z Analyze the word as verb. @param verb: the input word. @type verb: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. )r stemming_verb)r<rs r>rzAnalex.check_word_as_verb--d333r@c6|j|S)z Analyze the word as noun. @param noun: the input word. @type noun: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. )r stemming_nounr<rs r>rzAnalex.check_word_as_nounrr@c6|j|S)z Analyze the word as unknown. @param noun: the input word. @type noun: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. )rrrs r>rzAnalex.check_word_as_unknowns"00666r@c@|rfd|DSfd|DS)a if the entred word is like the found word in dictionary, to treat some normalized cases, the analyzer return the vocalized like words. This function treat the Shadda case. @param word_nm_shadda: a word without harakat, but shadda @type word_nm_shadda: unicode @param resulted_data: the founded resulat from dictionary. @type resulted_data: list of dict. @param fully_vocalized_input: if the two words must resect the shadda and vocalized. @type fully_vocalized_input: Boolean, default is False. @return: list of dictionaries of analyzed words with tags. @rtype: list. crg|]3}tjtj|jk1|4Sr^)r,rrr_xrs r>r`z'Analex.check_shadda..sQ&~66&q{3344444r@cHg|]}tj|j|Sr^)r, shaddalikerrs r>r`z'Analex.check_shadda..s>#NAK@@r@r^)rrr9s` r>rzAnalex.check_shaddasc ! & & r@c fd|DS)u If the entred word is like the found word in dictionary, to treat some normalized cases, the analyzer return the vocalized like words ُIf the word is ذئب, the normalized form is ذءب, which can give from dictionary ذئبـ ذؤب. this function filter normalized resulted word according the given word, and give ذئب. @param word_nm the input word. @type word_nm: unicode. @param word_unvocalised: the input word unvocalized. @type word_unvocalised: unicode. @param resulted_data: the founded resulat from dictionary. @type resulted_data: list of dict. @return: list of dictionaries of analyzed words with tags. @rtype: list. c*g|]}|jk |Sr^) unvocalized)r_drs r>r`z+Analex.check_normalized..s%EEEaAMW,D,D,D,D,Dr@r^)r<rrs ` r>rzAnalex.check_normalizeds $FEEE=EEEEr@c*g}tj|s|S|D]}d|vr|d}d|dv}tj||r3|dxxdtjzz cc<||`|r|tjrv|tjrWtj|dd|ddr2|dxxdtjzz cc<|||S)a if the entred word is vocalized fully or partially, the analyzer return the vocalized like words This function treat the partial vocalized case. @param word_vocalised: the input word. @type word_vocalised: unicode. @param resulted_data: the founded resulat from dictionary. @type resulted_data: list of dict. @return: list of dictionaries of analyzed words with tags. @rtype: list. rVerbrr:NrS) r, is_vocalized vocalizedlikerPARTIAL_VOCALIZED_TAGrVendswithKASRASUKUN)rr filtred_dataroutputis_verbs r>rzAnalex.check_partial_vocalizeds; !.11 6 & 6 6$&&!+.F$V 4G*>6BB 6V l.P(PP $++D1111 6*33EK@@6#OOEK886 !.~crc/BF3B3KPP6 LLLC,2T,TTLLL(//555r@)FFT)r)N)ru)F)$__name__ __module__ __qualname____doc__r?rCrErGrMrOr[rbrdrfrhrjrlrorrrtrrrrrrrrrrrr staticmethodrrrr^r@r>rrBsBPTE!E!E!E!N " " "          $$$$@   ***222*** 333 ===%%% BBBBH      ccccJ......<<<|===777r444444777\BFFF8##\###r@r)rrsyspathrVr.pyarabic.arabyr, arramooz.wordfreqdictionaryclassr3naftawayh.wordtagr"arabicstopwords.arabicstopwordsarabicstopwordsrrrrrrr rr r r r rqrqalsadi.cachemanagerrrr^r@r>r s zJJJHOOD   CBBBBB333333))))))888888M M M M M M M M M M r@