-idZddlZddlmZddlmZddlZddl m Z ddl Z ddlmZddlmZddlmZddlmZGdd Zd Zed kr edSdS) z Arabic noun stemmer N)arepr) print_table)custom_dictionary)wordcaseceZdZdZddZdZdZdZdZdZ d Z d Z d Z e d Ze dd Ze dZe dZdZe dZe dZe dZe dZdS) NounStemmerz Arabic noun stemmer Fctj|_|jt j|jt jtj|_ |j t j |j t j tj |_d|_t!jd|_t'jd|_i|_i|_i|_i|_||_d|_dS)NTnouns) tashaphynestemmingArabicLightStemmer comp_stemmerset_prefix_listSNCCOMP_PREFIX_LISTset_suffix_listCOMP_SUFFIX_LIST conj_stemmerCONJ_PREFIX_LISTCONJ_SUFFIX_LISTalyahmor noun_affixer generatorallow_syntax_lastmarkarabicdictionaryArabicDictionarynoun_dictionaryrcustom_noun_dictionarycache_dict_searchcache_affixes_verification noun_cachenoun_vocalize_cachedebug error_codeselfr%s O/var/www/html/speakWrite/venv/lib/python3.11/site-packages/qalsadi/stem_noun.py__init__zNounStemmer.__init__"s&/BBDD ))#*>??? ))#*>???&/BBDD ))#*>??? ))#*>???".;;==%)" 0@II&7&I'&R&R#"$*,'#%  c|jS)z? Return error code when word is not recognized r&r(s r)get_error_codezNounStemmer.get_error_code@s r+c&|js ||_dSdS)z< set error code when word is not recognized Nr-)r(r&s r)set_error_codezNounStemmer.set_error_codeHs# )(DOOO ) )r+c||jvr |j|S|j|}||j|z }||j|<|S)z) lookup for word in dict )r#rlookupr )r(wordresults r) lookup_dictzNounStemmer.lookup_dictOs` 4? " "?4( ()0066F d188>> >F$*DOD ! r+c||d|s|ddS|j}g}|g||z}tt |}g}|D]}|j|}|||tj }|D]s}|d|d} ||d|d} ||dd} | g| | | z} | D]} || | | d} | | t|s|dg}|rtd|r8tt|tt||D]X} |j| d }|| d |tj}|D] }| d d|d}| d |dd}t#j|}|||}|D]} | }| |d <||d <tt tj|d d tj|dd ztj|d d z|d<| | Z|rtd|r8tt|tt||s|d|}g}|D]} | d }||jvr|j|}n||}||j|<|D]H}| }|d|d<t5||d<| |I|rtd|rDtt|d|D}tt||s|d|}g}|D]>} || d| d| d | d| d rtj| d dD]}tj| ddD]}tj| d dD]}|| d|||rtj|d tj|d ztj|d z}| }||d<||d<||d<| |Č@|rtd|rDtt|d|D}tt||s|d|}g}|D]} || dd| d | d| d}|D]\} }!}"g}#| dddkr|# dd tj| dvrtj| dd }$nd!}$|$D]G}%| d|%fz}&d"|&D}&| dd#}'|'d$vr.| dd%}(|(r| dd%})n| dd})n| dd})| t=j|| d d| d| df| d t#j| d d&d| dd|)| |!d'!|&d'!d(| dd)g|'| dd*d+d'!|#dd,I|s|d-|rtd.|rttE||S)/z Analyze word morphologically as noun @param noun_in: the input noun. @type noun_in: unicode. @return: list of dictionaries of analyzed words with tags. @rtype: list. r z Empty wordNrr)noun stem_compproencz First level segmentation errorzafter first levelr9 stem_conjsuffixr:tagsr; affix_tagszafter second levelz Second level segmentation error vocalizedoriginal noun_tuplezafter lookup dictcg|] }|d SrB.0items r) z-NounStemmer.stemming_noun..CCC$4 -CCCr+zNot exists in dictionarysuf_vocenc_voczafter check compatibilitycg|] }|d SrDrErFs r)rIz-NounStemmer.stemming_noun..rJr+zAffixes not compatiblemankousTku منقوصcases)r cg|]}||SrErE)rGvacs r)rIz-NounStemmer.stemming_noun..-s%K%K%Kcs%Kc%K%K%Kr+numberجمعuجمع تكسيرsingleroot:Nounwordtypegenderfreqnoun)r4affixstemrWrAlemmar@ semivocalizedr>typerSr[freq originaltagssyntaxzForms are not generatedzafter generate result)#r1r%get_noun_variantslistsetrsegment verify_affixrCOMP_NOUN_AFFIXESget_input_stem_variantsappendprintrrrNOMINAL_CONJUGATION_AFFIXarnormalize_hamzaget_stem_variantscopyCOMP_PREFIX_LIST_TAGSCOMP_SUFFIX_LIST_TAGSCONJ_SUFFIX_LIST_TAGSr!r6dict validate_tags _NounStemmer__check_clitic_affixvocalizerWordCasegetjoinlen)*r(noun_inr%detailed_result noun_listword_segmented_listr8 list_seg_compseg proclitic_nmr^ enclitic_nm list_stemword_segtmp_list list_seg_conjseg_conjr<r=stem_conj_list word_seg_l2inf_nouninfnoun_foundlistrB word_seg_l3 noun_tuplespro_vocrLrKaffix_tags_voc word_seg_l4voca_tuple_listr@semi_vocalized_ original_tags list_casescasevoc_affix_caserS single_formr_s* r) stemming_nounzNounStemmer.stemming_noun[s  B     - - -4    " "7 + +, Y((   9 9D -55d;;M --mS%:M% 9 9#Hc!fH~ CFSVO,"3q688n 00{CCD &99D $%)+*   H (..x88889 9$# C    A B B B  ' % & & &  4 %.. ! ! ! +122 3 3 3+$ 1$ 1H!-55h{6KLLM!--%}c6SM * 1 1$[1-HQK-@ !+.x{}}=.y99 !%!7!7 6!J!J+ 1 1D"*--//K/3K ,,2K)045k%6HI&Q!7 E8JKFST!7 H8MNvVW11K -OOK0000 1 1.  ( & ' ' '  ) %.. ! ! ! +h'' ( ( ( D    B C C C&+ - -H ,H4111$($:8$D!!$($4$4X$>$>!3D&x0/ - - &mmoo *4[*A J',0,<,< L) ,,,,  -  ' % & & &  , %.. ! ! !CC(CCCK +k** + + + <    : ; ; ;&+" =" =H !!&&"  = #8%I+V==G#&#1!# L))4<<!((666c78KLLL!$!:8I;N!OPW!XJJ!&J&,,D%-l%;tg%EN%K%K^%K%K%KN%l3H=F!@@@&.|&$>vr$J$J)")"-5\,B;,O(--61?(+(@(@(+%+Xl-CJ-O$P)")"+1*2<*@*J(2030G0G*,1,7 p ;    9 : : :  + ) * * *  ( #o&& ' ' 'r+c |tjkrY|dtjs4|dtjtjzsdSdt j|dvr |dsdSdt j|dvr |drdSd|||tt|dg}||j vr |j |St j |d}t j |d}t j|d}|t j|d d z }d |vrd |vrd |vr d|j |<nid |vrd|vr d|j |<nVd |vrd |vr d|j |<nCd |vrd|vr d|j |<n0d |vrd|vr d|j |<nd|vrd|vr d|j |<n d|j |<|j |S)a Verify if proaffixes (sytaxic affixes) are compatable with affixes ( conjugation) @param proclitic_nm: first level prefix. @type proclitic_nm: unicode. @param enclitic: first level suffix. @type enclitic: unicode. @param suffix: second level suffix. @type suffix: unicode. @return: compatible. @rtype: True/False. unvocalizedFجمع مذكر سالمr>masculin_plural تنوين mamnou3_sarf-rPrEu تعريفuمضافu إضافةu لايضافuجرu مجرورT)roFATHATANendswith TEH_MARBUTAALEFHAMZArrur|strboolr"rsrtr{) r(rBrencliticr=r]proclitic_tags enclitic_tags suffix_tagss r)__check_clitic_affixz NounStemmer.__check_clitic_affixasq" R[ } % . .r~ > > !-(11"'BH2DEE !5 '#*CF*KF*S S S01 T5 C5f=fE E E>* F5 8VSj6P1Q1Q-R-R S   D3 3 3259 92<@H1(;FC /7?  s08<مدرست+ي = > مدرسة +ي. Return a list of possible cases. @param stem: the input stem. @type stem: unicode. @param suffix_nm: suffix (no mark). @type suffix_nm: unicode. @return: list of stem variants. @rtype: list of unicode. N) rgrorTEHYEHraddNOONrWAW ALEF_MAKSURAr YEH_HAMZA)r^ suffix_nmpossible_noun_list possible_nounvalidated_lists r)rqzNounStemmer.get_stem_variantss"!     Gbf  FR^ # F FRW rv %    !2>1M  " "= 1 1 1 27*BFRW,<= = =$--PRPVBWBW = "I6M  " "= 1 1 1 2I"&27*:BFRW) findrorresubr is_harakarrKASRATANrrustrip_lastharaka)r4r=rrNr newsuffixsuffix_non_irab_marks r)get_suffix_variantszNounStemmer.get_suffix_variantss&  ;;r~ & &! + + +r~rvv>>II (f!5!5 (BCCyRVRW---  (K  34V|dkr|ddtjz}n|dkr|ddtjz}n|tj tjzr|d|vr&d|vr"|s |stj|dd}n|tjtj ztj tj zfvrtj|dd}n.|tj r|s|r | tjr|ddtjz}n| tj r|ddtjz}n|tjtj zs9|tjtjztj zr6| tjr|ddtjz}|S)u Get the word variant to be joined to the suffix. For example: word = مدرسة, suffix = ي. The word is converted to مدرست. @param word: word found in dictionary. @type word: unicode. @param proclitic: proclitic ( first level). @type proclitic: unicode. @param suffix: suffix ( first level). @type suffix: unicode. @param enclitic: enclitic( second level). @type enclitic: unicode. @return: variant of word. @rtype: unicode. Nrr الuلل)rostrip_tashkeelrrrrrrrrrrrrr WAW_HAMZArSUKUNr)r4 procliticr=r word_stemrrlong_suffix_nms r)get_word_variantzNounStemmer.get_word_variant@s  %f-- '11 "[0' 22    bn - -@ :!!"'BN":;; 4FFR^+FRW$rv-! !*#2# 7II"'BF"222 " 3IcrcN C CII#r)) )#2# 7I"& ' 26) / #2#??  2%%%crcNRV3    0 0$ :B%crcNRV3 ""%crcNRW4   26 1 2 2 : i'')++#,!,/ #2#?? rv/"'1ABBB/ #2#??    ) ) :y :K :  ** :%crcNR\9 ""28,, :%crcNR\9 ""26BH#455 :%%bfrx&7"(&BCC :##BK00 :&crcNR\9 r+cd||||g}||jvr%|j|||||j|<|j|S)a Join the noun and its affixes, and get the vocalized form @param noun: noun found in dictionary. @type noun: unicode. @param proclitic: first level prefix. @type proclitic: unicode. @param suffix: second level suffix. @type suffix: unicode. @param enclitic: first level suffix. @type enclitic: unicode. @return: vocalized word. @rtype: unicode. r)r|r$rryrrsrtrrorrrFATHAis_sunSHADDArrrrrrrrrrrAJUST_VOCAL_PATTERNS)r(r8rr=rkey proclitic_vocrencl_voc_non_inflectrrNrword_non_irab_markword_vocalized segmentedpatrnrepls r)ryzNounStemmer.vocalizesdhhi:;;d...,0N,C,Ci--D $S )',,r+cg}tj|vr||tjtjdz||tjtjtjz||tjtjtjz|S)zgenerate noun varaintes)ro ALEF_MADDArlrALEF_HAMZA_ABOVErr)r8rs r)rezNounStemmer.get_noun_variantss =D   T\\"-9Lq9PQQ R R R   T\\"-BG9KLL M M M   T\\"-9Lrw9VWW X X Xr+czg}|r|tjr+||ddtjznl|tjr*||ddtjzn"||tjz|S)zgenerate stem varaintesNr)rrorrlrrrr)r^rrs r)rkz#NounStemmer.get_input_stem_variants s  ,}}RW%% =  crcR_!<====rv&& =  crcR^!;<<<   TBF] + + +r+c$fd|DS)ad Verify possible affixes in the resulted segments according to the given affixes list. @param word: the input word. @type word: unicode. @param list_seg: list of word segments indexes (numbers). @type list_seg: list of pairs. @return: list of acceped segments. @rtype: list of pairs. c~g|]9}dd|d|ddgv7|:S)rNrr)r|)rGs affix_listr4s r)rIz,NounStemmer.verify_affix..+sS   388T&AaD&\4!<,H#I#IZ#W#WA#W#W#Wr+rE)r4list_segrs` `r)rizNounStemmer.verify_affixs4         r+c`|}d|vr |drdS|ddvrd|vrdSd|vrdSd|vrdS|d vr |d sdS|d vr |d sdSd |vr |dsdS|d r |dsdS|tjkr |dsdS|tjkr|ddkrdSdS)a~ Test if the given word from dictionary is compabilbe with affixes tags. @param noun_tuple: the input word attributes given from dictionary. @type noun_tuple: dict. @param affix_tags: a list of tags given by affixes. @type affix_tags:list. @param proclitic_nm: first level prefix vocalized. @type proclitic_nm: unicode. @param enclitic_nm: first level suffix vocalized. @type enclitic_nm: unicode. @param suffix_nm: first level suffix vocalized. @type suffix_nm: unicode. @return: if the tags are compatible. @rtype: Boolean. rrFrSrTuجمع مؤنث سالمruمثنى)uهمuهنuكماuكمuهما hm_suffix)uهuها ha_suffixrk_prefix kal_prefixw_suffixrUT)rrorr)rBr?rrrrs r)rwzNounStemmer.validate_tags/s*$!  : % %*^*D %5 h #B B B)Z77u)Z77uZ''u G G G{+ H5 . ( (K1H (5  " ":j+A "5   f % % j.F 5   z*'= 5  & &:h+?8+K+K5(tr+N)F)__name__ __module__ __qualname____doc__r*r/r1r6rrxrrr staticmethodrqrrrryrerkrirwrEr+r)r r s<)))   DDDLV6V6V6p*** +++ 11\1f&/&/&/\&/P55\5*ZZ\Zxb=b=b=H\\$   \  GG\GGGr+r c Pgd}t}|d|D]B}|j|t |jC|D]}||}|D]}|jD]d}t d |t|j| dg det t dS)z Test main)uيضربu الكتابuالاستخدامuفاستعمالهمuضربu لأكلهمT zunicode-escapeutf8N) r rrrhrmget_affix_listr__dict__keysr|reprdecodeencode)wordlist nounstemmerr4r5analyzedrs r)mainlyrzsBH--K$99 ((... k&55778888**400  H (--// ! !  $x'8'=">">"E"EFV"W"WX &.... GGG GGGG r+__main__)rrpyarabic.arabyarabyropyarabic.arabreprrtashaphyne.stemmingr alyahmor.aly_stem_noun_constaly_stem_noun_constralyahmor.noun_affixerrarramooz.arabicdictionaryr print_debugrr rrr rrrEr+r)rs ######******444444$$$$$$Z Z Z Z Z Z Z Z zF z FHHHHHr+