-idZddlmZddlmZddlZddlm Z ddl Z ddl ZddlmZddlmZddlmZddlmZGdd Zd Zed kr edSdS) z Arabic verb stemmer N)arepr) print_table)custom_dictionary)wordcaseceZdZdZddZdZdZdZdZdZ d Z d Z d Z d Z d ZdZdZdZdZdZdZejdddfdZedZedZedZedZdZdZdS) VerbStemmerz Arabic verb stemmer Fc tj|_|jt j|jt jtj|_ |j t j |j t j tj |_d|_i|_i|_||_t'jd|_t-jd|_t j|_i|_i|_i|_i|_i|_d|_dS)NTverbs) tashaphynestemmingArabicLightStemmer comp_stemmerset_prefix_listSVCCOMP_PREFIX_LISTset_suffix_listCOMP_SUFFIX_LIST conj_stemmerCONJ_PREFIX_LISTCONJ_SUFFIX_LISTalyahmor verb_affixer generatorallow_syntax_lastmarkcompatibility_cacheverb_dict_cachedebugarabicdictionaryArabicDictionaryverb_dictionaryrcustom_verb_dictionaryVERB_STAMP_PATverb_stamp_pat stamp_cache verb_cacheverbclass_cachestripped_words_cacheverb_conj_cache error_codeselfrs O/var/www/html/speakWrite/venv/lib/python3.11/site-packages/qalsadi/stem_verb.py__init__zVerbStemmer.__init__"s>&/BBDD ))#*>??? ))#*>???'/BBDD ))#*>??? ))#*>???".;;==%)"$& !  0@II'8&I'&R&R#!0!$&!!c|jS)z? Return error code when word is not recognized r+r-s r.get_error_codezVerbStemmer.get_error_codeOs r0c&|js ||_dSdS)z< set error code when word is not recognized Nr2)r-r+s r.set_error_codezVerbStemmer.set_error_codeWs# )(DOOO ) )r0c||jvr |j|S|j|}||j|z }||j|<|S)) lookup for word in dict )r'r"lookup_by_stampr#)r-wordresults r.r9zVerbStemmer.lookup_by_stamp^s` 4? " "?4( ()99$??F d1AA$GG GF$*DOD ! r0cj|j|}|tjd}t |dkrdS||jvrA|j|}||j|z }||j|<|j |dS)r8r F) r" word_stampreplacearTEHlenr&exists_as_stampr#get)r-r:stampr;s r.rCzVerbStemmer.exists_as_stampjs$//55 bfb)) u::>>5 ( ( ()99$??F d1AA$GG GF&,D U ###E5111r0c|sdSg}|g||z}tt|}|j}g}|D]}|j|}|D]}|d|d} ||d|d} ||dd} | t jvrt j| d} | } | g|| | z} t| }| D]}|| | ||d}| | |rtd|r8tt|tt|g}|D]}|d}|j|}|||t j}|D]}|d|dz dkrh|}|d|d|d <||d|d|d <||dd|d <| ||}g}|D]s}|d } |d } d|d |d g}|| | |r'| |t|}g}|D]D}||d r'| |E|rtd|r8tt|tt||}g}|D]N}||d |d}|rtd|r8tt|tt|||d |}|rtd|r8tt|tt||D]t}|}|d|d<|d|d<|dd|d<t|ddv|d<| |uP|rtd|r8tt|tt||}g}|D]}||d|d|d dz|d z|d|d |d |d}|D]B}|}||d<| |C|rtd|rDtt|d|D}tt||}|D]}|d}||d|d |d }d}|drd nd!} |D]\}!}"}#||| | }$| t7jid"|dd#|d |d |d |d fd$|d dt;j|ddd%|dd&|dd|!d'|"d(|$d)|d*|d+d*dd,|d+d,dd-|d+d-dd.|d/d0dd1|d/d1dd2|d/d2dd3|d/d3d|d|d0|d4d5| dd6|S)7z Stemming verb @param verb_in: given verb @type verb_in: unicode @return : stemmed words @rtype: Nrrfirst)verbproenc stem_comp trans_compzafter first levelrKprefix stem_conjsuffixrIrJ-zafter second levelrLzinfinitive candidat verbszvalid infinitive candidat verbsrHinfharakarootr transitiveyrzafter lookup dictconjzafter generating conjugationcg|] }|d S)rX).0items r. z-VerbStemmer.stemming_verb..2s777dT&\777r0 vocalizedVerbrWnr:affixstemoriginallemma semivocalizedtagstypenumber pronoun_tagsgenderpersontense2 tense_tagstensevoicemood confirmedpronounfreqverb)rUrnrrfreq originaltagssyntax)get_verb_variantslistsetrrsegmentrTABLE_DOUBLE_TRANSITIVE_SUFFIXget_in_stem_variantsboolappendprintrrr verify_affixVERBAL_CONJUGATION_AFFIXcopyjoin _VerbStemmer__check_clitic_affixrC)_VerbStemmer__get_infinitive_verb_by_stem%_VerbStemmer__verify_infinitive_verbsrD&_VerbStemmer__generate_possible_conjugvocalize prepare_tagsrWordCaser@normalize_hamza)%r-verb_indetailed_result verb_listrword_segmented_listrH list_seg_compseg procliticrbenclitic firstsuffix list_stemtransitive_compstmword_segtmp_listverb2 list_seg_conjseg_conj word_seg_l2 affix_conj infverb_dictr\ word_seg_l3one_correct_conjrX word_seg_l4conjsvocal_tuple_listtag_type original_tagsr^re__rfs% r. stemming_verbzVerbStemmer.stemming_verbzsQ  4   " "7 + +, Y((    9 9D -55d;;M$ 9 9 3q6N CFSVO,A>sAAA"%"DX"Nw"WK*H!FT%>%>tX%N%NN "&x..$99C $('%(&5   H(..x88889 9.  ' % & & &  4 %.. ! ! ! +122 3 3 3+ 1 1H[)E!-55e<>%"S(8H+=="&   ) - -&mmoo &*iikk F# ,,,, -  2 0 1 1 1  & %.. ! ! !77h777E +e$$ % % %',/ / HF#D#}}[!8E?HUO   H#' #5>CC3M0@& & , ="((y(CC&&%"HV$4# ( ( 2 ( 2 ( &#H[$9#B$6x||FB7O7O$P$P'V $T&\(,]#D#H %d>&:&>&>x&L&L!"%d>&:&>&>x&L&L#$%d>&:&>&>x&L&L%&%d<&8&<&.ts%%%a1%%%%r0:)extendrCOMP_PREFIX_LIST_TAGSrDCOMP_SUFFIX_LIST_TAGSr)r-rXrrrfs r.rzVerbStemmer.prepare_tagslsW tI/ C-11)R@@DDVRPPQQQ C-11(B??CCFBOOPPP&%4%%%xx~~ r0c d|t|g}||jvr |j|Sg}||}t |r>|D];}||d|d|d|d|dd<|}g}|D]#}|ddvs|s||$||j|<|S) a0 Get the infinitive verb form by given stem, and transitivity @param verb: the given verb @type verb; unicode @param transitive: tranitive or intransitive @type transitive: boolean @return : list of infinitive verbs @rtype: list of unicode rr^rU future_typestampedrT)rHrUrSrErTrV)rstrrr9rBr~) r-rHrUverb_keyliste verb_id_list verb_tuple listetempr\s r.__get_infinitive_verb_by_stemz)VerbStemmer.__get_infinitive_verb_by_stemxs88T3z??344 t+ + +'1 1++D11 |   *    *; 7&0&>",]";!+I!6 *6 2   # #DL!X--Z- T""").X& r0c||_dS)z Set the debug attribute to allow printing internal analysis results. @param debug: the debug value. @type debug: True/False. N)rr,s r. set_debugzVerbStemmer.set_debugs  r0cd|_dS)zX Enable the syntaxic last mark attribute to allow use of I'rab harakat. TNrr3s r.enable_syntax_lastmarkz"VerbStemmer.enable_syntax_lastmarks&*"""r0cd|_dS)zY Disable the syntaxic last mark attribute to allow use of I'rab harakat. FNrr3s r.disable_syntax_lastmarkz#VerbStemmer.disable_syntax_lastmarks&+"""r0cg}||}|D]#}|d|kr||$|S)a verify if given infinitive verbs are compatible with stem_conj @param stem_conj: the stemmed verbs without conjugation affixes. @type stem_conj: unicode. @param infverb_dict: list of given infinitive verbs, each item contain 'verb' and 'type'. @type infverb_dict: list of dicts. @return: filtred infinitive verbs @rtype: list of dict rE) verb_stampr~)r-rOrtmp stem_stampr\s r.__verify_infinitive_verbsz%VerbStemmer.__verify_infinitive_verbssR__Y//   ! !DG} ** 4    r0ctj|}tj|}|tjr |dd}|dd|ddkr |dd}|jd|S)a> generate a stamp for a verb, the verb stamp is different of word stamp, by hamza noralization remove all letters which can change form in the word : - ALEF, - YEH, - WAW, - ALEF_MAKSURA - SHADDA @return: stamped word rNr )r@strip_tashkeelr startswithHAMZAr%subr-r:s r.rzVerbStemmer.verb_stamps &&!$'' ??28 $ $ 8D 9RU # #9D"&&r4000r0cd|||g}||jvr |j|S|s|sdSd}|sd}nh|tjvrZ|dkrd}nQtj|gD].}|dtj|dvrd}n/d}|r|s d|j|<dS|tjvrw|dkr d|j|<dStj|gD],}|dtj|dvrn -d|j|<dSd|j|<dSd|j|<dS)a Verify if proaffixes (sytaxic affixes) are compatable with affixes (conjugation) @param proclitic: first level prefix. @type proclitic: unicode. @param enclitic: first level suffix. @type enclitic: unicode. @param affix: second level affix. @type affix: unicode. @return: compatible. @rtype: True/False. rTFrQrr r)rrrEXTERNAL_PREFIX_TABLE TABLE_AFFIXrDEXTERNAL_SUFFIX_TABLE)r-rrracomp_keyproclitic_compatibler\s r.__check_clitic_affixz VerbStemmer.__check_clitic_affixs88Y%899 t/ / /+H5 5% $% $4#(  5'+$$c777C<<+/(( # 3 3E2 > >557c&?&C&CIr&R&RRR370!ES05,# $$9=D,X64!:::||=A0:#t$'O$7$7r$B$B))D#Aw#*C*G*GRT*U*UUU % VBGD4X>#(5=A0:#t-2 *ur0c d||||t|g}||jvr |j|S|s|tjjvr d|j|<dS|s|s d|j|<dS|r|tjjvr d|j|<dS|r"|tj|dvr0|r"|tj|dvr d|j|<dSd|j|<dS)zI test if the given tenses are compatible with proclitics rFTr ) rrrrqutrubVerbConstTablePassiveTenserrDr)r-rrrnrrrUrs r.__check_clitic_tensez VerbStemmer.__check_clitic_tenses/ 88Y%#j//RSS t/ / /+H5 5 es':'LLL16D $X .5  15D $X .4  !4!FFF16D $X .5 "c&?&C&CIr&R&RRRS&#*C*G*GRT*U*UUU15D $X .427D $X .5r0cd||t|g}||jvr |j|Stj|||}||j|<|S)zG return the verb class, used to reduce verb class init rQ)rrr( libqutrub classverb VerbClass)r-infinitive_verbrUrkeyvbcs r._get_verbclasszVerbStemmer._get_verbclassDsqhhc*ooFGG $& & &', ,%//[C),D  %Jr0c|j||}|s|||}||}||fS)z4 return the conjugation from a verb ) conj_displayget_conjconjugate_tense_for_pronoun_strip_tashkeel)r- verb_objectrnrrrXconj_nms r._get_conjugationzVerbStemmer._get_conjugationRsW'00@@ K::5'JJD&&t,,W}r0rQTc g}|dks |dks|dkrtS||||} |tjr |dd}g} |t jvr0t j|D]} | | dtt| } |t jvrt j|D]} | d} | d} | ||| | |}|rd| | | | \}}||krD||| | | | | | |||d|S)zL generate possible conjugation for given verb to be stemmed r rNr)rHrnrrrirmr^ unvocalizedrU) ryrrr@ALEFrrr~rx _VerbStemmer__check_clitic_tenserget_pronoun_featuresget_tense_features)r-r unstemed_verbrar extern_prefix extern_suffixrUlist_correct_conjrtensespairrnrrtestconj_vocalizedrs r.__generate_possible_conjugz&VerbStemmer.__generate_possible_conjug_s b MR$7$75B;;55L!!/:{KK   BG $ $ !""IE CO # #. ' ' d1g&&&&c&kk"" CO # #.  Qq'00!=%* .2.C.CCPW.X.X+NG -//)00(7).+2030H0H0Q0Q.1.D.DU.K.K-;/6.8     ! r0c$fd|DS)ad Verify possible affixes in the resulted segments according to the given affixes list. @param word: the input word. @type word: unicode. @param list_seg: list of word segments indexes (numbers). @type list_seg: list of pairs. @return: list of acceped segments. @rtype: list of pairs. c~g|]9}dd|d|ddgv7|:S)rQNrr)r)r[s affix_listr:s r.r]z,VerbStemmer.verify_affix..sS   388T&AaD&\4!<,H#I#IZ#W#WA#W#W#Wr0rZ)r:list_segrs` `r.rzVerbStemmer.verify_affixs4         r0c:g}tj|vr||tjtjdz||tjtjtjz|S)z#return modified forms of input verb)r@ ALEF_MADDAr~r?ALEF_HAMZA_ABOVErr)rHrs r.rwzVerbStemmer.get_verb_variantssw =D   T\\"-9Lq9PQQ R R R   T\\"-BG9KLL M M Mr0cg}|r|tjtjztjzr||ddn|tjr#||tjznI|tjr*||ddtjztj|vr|| tjtj dz|| tjtj tjz|| tjtj tjz|S)z#return modified forms of input stemNrr) endswithr@rAMEEMWAWr~r ALEF_MAKSURArr?rr)rbrrs r.r|z VerbStemmer.get_in_stem_variantss\  >}}RVbg-677 >  crc++++rv&& >  0000rw'' >  crcR_!<=== =D   T\\"-9Lq9PQQ R R R   T\\"-BG9KLL M M M   T\\"-9Lrw9VWW X X Xr0c|tjtjzkrW|tjs|tjrtjtjz}|S)u Get the enclitic variant to be joined to the word. For example: word = أرجِهِ , enclitic = هُ. The enclitic is convert to HEH+ KAsra. اعبارة في مثل أرجه وأخاه إلى يم الزينة @param word: word found in dictionary. @type word: unicode. @param enclitic: first level suffix vocalized. @type enclitic: unicode. @return: variant of enclitic. @rtype: unicode. )r@HEHDAMMArKASRAYEH)r:rs r.get_enclitic_variantz VerbStemmer.get_enclitic_variantsY rv( ( ( MM"( # # )'+}}RV'<'< )v(Hr0c:|j|||S)a Join the verb and its affixes, and get the vocalized form @param verb: verb found in dictionary. @type verb: unicode. @param proclitic: first level prefix. @type proclitic: unicode. @param enclitic: first level suffix. @type enclitic: unicode. @return: (vocalized word, semivocalized). @rtype: (unicode, unicode). ) rrrrr rrr@rrrrstrip_lastharaka)r-rHrr enclitic_voc proclitic_vocr^res r.rzVerbStemmer.vocalizes~&&tYAAAr0cf||jvrtj||j|<|j|S)z; reduce amount of calculate strip tashkeel )r)r@rrs r.rzVerbStemmer._strip_tashkeels8t000.0.?.E.ED %d +(..r0N)F)__name__ __module__ __qualname____doc__r/r4r6r9rCrrrrrrrrrrrrr@FATHAr staticmethodrrwr|r rrrZr0r.r r s++++Z)))   222 pppd   ***X*** +++ *111.888t$$$L      $H=!=!=!=!~  \ "  \ \*\(***</////r0r c *gd}t}|d|D]B}|j|t |jC|D]}||}|D]}|jD]Q}t d |t|j|g dRt t dS)z Test main)u يضربهu يضربكu استقلuويستخدمونهاu اتركنيT utf8N) r rrrzrget_affix_listr__dict__keysrunicodeencode)wordlist verbstemmerr:r;analyzedrs r.mainlyr"s.H--K$99 ((... k&55778888**400  H (--// X Xdiigh.?.D&E&E FGGNNvVVWWWW GGG GGGG  r0__main__)rpyarabic.arabyarabyr@pyarabic.arabreprrtashaphyne.stemmingr alyahmor.aly_stem_verb_constaly_stem_verb_constralyahmor.verb_affixerrlibqutrub.classverbrarramooz.arabicdictionaryr print_debugrr rrr r"rrZr0r.r.s######******444444$$$$$$n /n /n /n /n /n /n /n /b< z FHHHHHr0