-i^ dZddlmZmZmZmZddlZddlZ en #e $re ZYnwxYwe dkrddl Z nddl m Z ddlZdZdZd Zd Zd Zd Zd ZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'dZ(d Z)d!Z*d"Z+d#Z,d$Z-d%Z.d&Z/d'Z0d(Z1d)Z2d*Z3d+Z4d,Z5d-Z6d.Z7d/Z8d0Z9d1Z:d2Z;d3Zd6Z?d7Z@d8ZAd9ZBd:ZCd;ZDdZGd?ZHd@ZIdAZJdBZKdCZLdDZMdEZNdFZOdGZPdHZQdIZRdJZSdKZTdLZUdMZVdNZWdOZXdPZYdQZZdRZ[dSZ\dTZ]dUZ^dVZ_dWZ`dXZadYZbdZZcd[Zdd\Zed]Zfd^Zgd_ZhdTZid`ZjdaZkdbZldcZmddZndeZodfZpdgZqdhZrdiZsdjtgeeeeeeee e!e"e#e$e%e&e'e(e)e*e+e,e.e/e0e1e2e3e4e5e6e7eeeeeeZue;ee?e@eAeBeCeDf ZveEeFeGeHeIeJeKeLeMeNf ZweOePeQeReSeTeUeVeWeXf ZxeaebecedeeefehegfZyeaebecedeeefehfZzedeeefehfZ{eaebecfZ|e-Z}elemeneofZ~eeee9e:eefZeeeee^e6eifZee5e7e6fZe7ee6ekfZe5eejfZeefZeiejekfZeeeeeeeee e+e,e.e/e0e2e4e5e7fZeee!e"e#e$e%e&e'e(e)e*e1e3fZiededkedledledmednedoe dpe!dqe"dre#dse$dte%due&dve'dwe(dxe)dyie*dze+d{e,d|e.d}e/d~e0de1de2de3de4de5de7dedededededediZiededededededede de!de"de#de$de%de&de'de(de)die*de+de,de.de/de0de1de2de3de4de5de7dede-dede6dededededeadebdecdeddeedefdegdehdi ZdjteZdjtezZdjteyZejddjtezzdzejZejddjtezddjte|dejZejddjte{zdzejZejddjteyzdzejZejddjtezdzejZejddjtezdzejZejddjte~zdzejZejdejZejdejZejdZejdelemeodejZejdejZejdejZdeddDZdZd„ZdÄZdĄZdńZdƄZdDŽZdȄZdɄZdʄZd˄Zd̄Zd̈́Zd΄ZdτZdЄZdфZd҄ZddӄZdԄZdՄZdքZdׄZd؄ZdلZdڄZdۄZd܄Zd݄ZdބZd߄ZdZdZdZdZdZdZdZÐddZdZdZƐddZdZdZɐddZdZdZdZdZdjggfdZdedefdZdZdZԐd dZe dkrededdede͐ddedeɐddede͐ddedeɐddededdededddZgdZeeצZe֐dee٦dSdS( a Arabic module Features: ========= - Arabic letters classification - Text tokenization - Strip Harakat (all, except Shadda, tatweel, last_haraka) - Sperate and join Letters and Harakat - Reduce tashkeel - Mesure tashkeel similarity (Harakats, fully or partially vocalized, similarity with a template) - Letters normalization (Ligatures and Hamza) @author: Taha Zerrouki @contact: taha dot zerrouki at gmail dot com @copyright: Arabtechies, Arabeyes, Taha Zerrouki @license: GPL @date:2010/03/01 @version: 0.1 )absolute_importprint_functionunicode_literalsdivisionN__main__)stacku،u؛u؟uءuآuأuؤuإuئuاuبuةuتuثuجuحuخuدuذuرuزuسuشuصuضuطuظuعuغuـuفuقuكuلuمuنuهuوuىuيuٓuٔuٕu٠u١u٢u٣u٤u٥u٦u٧u٨u٩0123456789u۰u۱u۲u۳u۴u۵u۶u۷u۸u۹u٪u٫u٬u٭uٰuٱu۔uuًuٌuٍuَuُuِuّuْuۥuۦuﻻuﻷuﻹuﻵuلاuلأuلإuلآ uألفuباءuتاءuتاء مربوطةuثاءuجيمuحاءuخاءuدالuذالuراءuزايuسينuشينuصادuضادuطاءuظاءuعينuغينuفاءuقافuكافuلامuميمuنونuهاءuواوuياءuهمزةu تطويلuألف ممدودةuألف مقصورةuهمزة على الألفuهمزة على الواوuهمزة تحت الألفuهمزة على الياءu فتحتانu ضمتانu كسرتانuفتحةuضمةuكسرةuشدةuسكون[]z]$|[u([^\wًٰ-ْ']+)u([\wًٰ-ْ']+)z | | | | ([^؀-ْz\s\d])u([^؀-ۿﭐ-﷿ﹰ-ݐ-ݿ])u(\s*([?؟!.,،:]+(?:\s+[?؟!.,،:]+)*)\s*cvg|]6}tjt|dk't|7S)Mn) unicodedatacategoryunichr).0xs L/var/www/html/speakWrite/venv/lib/python3.11/site-packages/pyarabic/araby.py r<"s; b b bA+:NvVWyy:Y:Y]a:a:afQii:a:a:aic|tkS)z)Checks if the given ``archar``Sukun Mark.)SUKUNarchars r;is_sukunrCZs U?r=c|tkS)z0Checks if the given ``archar`` is Shadda Mark. )SHADDArAs r; is_shaddarF_s V r=c|tkS)z7Checks if the given ``archar`` Tatweel letter modifier.)TATWEELrAs r; is_tatweelrIds W r=c|tvS)z,Checks if the given ``archar`` Tanwin Marks )TANWINrAs r; is_tanwinrLis V r=c|tvS)zChecks if the given ``archar`` Arabic Tashkeel Marks ( - FATHA, DAMMA, KASRA, SUKUN, - SHADDA, - FATHATAN, DAMMATAN, KASRATAn).)TASHKEELrAs r; is_tashkeelrOns X r=c|tvS)zYChecks if the given ``archar`` Arabic Harakat Marks (FATHA, DAMMA, KASRA, SUKUN, TANWIN).)HARAKATrAs r; is_harakarRvs W r=c|tvS)zQChecks if the given ``archar`` short Harakat Marks (FATHA, DAMMA, KASRA, SUKUN).) SHORTHARAKATrAs r;is_shortharakarU{s \ !!r=c|tvS)zChecks for Arabic Ligatures like LamAlef. (LAM_ALEF, LAM_ALEF_HAMZA_ABOVE, LAM_ALEF_HAMZA_BELOW, LAM_ALEF_MADDA_ABOVE) @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean ) LIGUATURESrAs r; is_ligaturerXs Z r=c|tvS)zChecks for Arabic Hamza forms. HAMZAT are (HAMZA, WAW_HAMZA, YEH_HAMZA, HAMZA_ABOVE, HAMZA_BELOW, ALEF_HAMZA_BELOW, ALEF_HAMZA_ABOVE) @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )HAMZATrAs r;is_hamzar[s V r=c|tvS)zChecks for Arabic Alef forms. ALEFAT = (ALEF, ALEF_MADDA, ALEF_HAMZA_ABOVE, ALEF_HAMZA_BELOW, ALEF_WASLA, ALEF_MAKSURA) @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )ALEFATrAs r;is_alefr^s V r=c|tvS)zChecks for Arabic Yeh forms. Yeh forms : YEH, YEH_HAMZA, SMALL_YEH, ALEF_MAKSURA @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )YEHLIKErAs r; is_yehlikera W r=c|tvS)zChecks for Arabic Waw like forms. Waw forms : WAW, WAW_HAMZA, SMALL_WAW @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )WAWLIKErAs r; is_wawlikererbr=c|tvS)zChecks for Arabic Teh forms. Teh forms : TEH, TEH_MARBUTA @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )TEHLIKErAs r;is_tehrhrbr=c|tvS)zChecks for Arabic Small letters. SMALL Letters : SMALL ALEF, SMALL WAW, SMALL YEH @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )SMALLrAs r;is_smallrks U?r=c|tvS)zChecks for Arabic Weak letters. Weak Letters : ALEF, WAW, YEH, ALEF_MAKSURA @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )WEAKrAs r;is_weakrn T>r=c|tvS)zChecks for Arabic Moon letters. Moon Letters : @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )MOONrAs r;is_moonrrror=c|tvS)zChecks for Arabic Sun letters. Moon Letters : @param archar: arabic unicode char @type archar: unicode @return: @rtype:Boolean )SUNrAs r;is_sunrus S=r=c8t|dS)zreturn Arabic letter order between 1 and 29. Alef order is 1, Yeh is 28, Hamza is 29. Teh Marbuta has the same ordre with Teh, 3. @param archar: arabic unicode char @type archar: unicode @return: arabic order. @rtype: integer r)ALPHABETIC_ORDERgetrAs r;orderrys    * **r=c8t||S)aFreturn Arabic letter name in arabic. Alef order is 1, Yeh is 28, Hamza is 29. Teh Marbuta has the same ordre with Teh, 3. @param archar: arabic unicode char @type archar: unicode @param default_name: arabic unicode char @type default_name: unicode @return: arabic name. @rtype: unicode )NAMESrx)rB default_names r;namer}s 99V\ * **r=cg}tddD]b} |t|&#t$r%|t |YTt $rY_wxYw|S)ureturn a list of arabic characteres . Return a list of characteres between ، to ْ @return: list of arabic characteres. @rtype: unicode r>iS)rangeappendr8 NameErrorchr ValueError)mylistis r; arabicranger s F 67 # #  MM&)) $ $ $ $ " " " MM#a&& ! ! ! ! !    D  Ms"9,A3' A32A3c>tjt|rdSdS)zChecks if the arabic word contains shadda. @param word: arabic unicode char @type word: unicode @return: if shadda exists @rtype:Boolean TF)researchrEwords r; has_shaddar s" yt 5r=cb|rdS|D]}t|rndSdS)zChecks if the arabic word is vocalized. the word musn't have any spaces and pounctuations. @param word: arabic unicode char @type word: unicode @return: if the word is vocalized @rtype:Boolean FT)isalpharO)rchars r; is_vocalizedr/sN ||~~u t    E u 4r=cPttjt|S)zChecks if the arabic text is vocalized. The text can contain many words and spaces @param text: arabic unicode char @type text: unicode @return: if the word is vocalized @rtype:Boolean )boolrrHARAKAT_PATTERNtexts r;is_vocalizedtextrAs  /400 1 11r=c>t|rdSdS)a; Checks for an Arabic standard Unicode block characters An arabic string can contain spaces, digits and pounctuation. but only arabic standard characters, not extended arabic @param text: input text @type text: unicode @return: True if all charaters are in Arabic block @rtype: Boolean FT) ARABIC_STRINGrrs r;is_arabicstringrLs$D!!u 4r=c>t|rdSdS)z Checks for an Arabic Unicode block characters @param text: input text @type text: unicode @return: True if all charaters are in Arabic block @rtype: Boolean FT) ARABIC_RANGErrs r;is_arabicrangerZs$4  u 4r=c t|dkrdStjdttt d|rdSt |ds|dttfvrdStj dtz|rdStj dtdtttd|rdStjttz|rdSd S) a% Checks for an valid Arabic word. An Arabic word not contains spaces, digits and pounctuation avoid some spelling error, TEH_MARBUTA must be at the end. @param word: input word @type word: unicode @return: True if all charaters are in Arabic block @rtype: Boolean rFr3])z^(.)*[%s](.)+$z^(.)*[z]([^z])(.)+$T)lenrrLAM_ALEFLAM_ALEF_HAMZA_ABOVELAM_ALEF_MADDA_ABOVErR WAW_HAMZA YEH_HAMZAmatch ALEF_MAKSURA TEH_MARBUTADAMMAKASRAFATHArErs r; is_arabicwordrfs 4yyA~~u h 4 46J6J6JLMQ S Su 47   tAw9i*@@@u #l2D 9 9u ;;uueee56: < <u 6F?D ) )utr=c|dS)z Return the first char @param word: given word @type word: unicode @return: the first char @rtype: unicode char rrs r; first_charrs  7Nr=c|ddS)z Return the second char @param word: given word @type word: unicode @return: the first char @rtype: unicode char rrrrs r; second_charrs !9r=c|ddS)z Return the last letter example: zerrouki; 'i' is the last. @param word: given word @type word: unicode @return: the last letter @rtype: unicode char Nrrs r; last_charrs 9r=c|ddS)z Return the second last letter example: zerrouki; 'k' is the second last. @param word: given word @type word: unicode @return: the second last letter @rtype: unicode char rrrs r;secondlast_charrs 2;r=cl|s|St|r tD]}||d}|S)uStrip Harakat from arabic word except Shadda. The striped marks are : - FATHA, DAMMA, KASRA - SUKUN - FATHATAN, DAMMATAN, KASRATAN, Example: >>> text = u"الْعَرَبِيّةُ" >>> strip_harakat(text) >>> العربيّة @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r)rrQreplacerrs r; strip_harakatrsJ( * d  * * *D<<b))DD Kr=c^|r*t|rtjtd|S|S)uStrip the last Haraka from arabic word except Shadda. The striped marks are : - FATHA, DAMMA, KASRA - SUKUN - FATHATAN, DAMMATAN, KASRATAN Example: >>> text = u"الْعَرَبِيّةُ" >>> strip_lastharaka(text) الْعَرَبِيّة @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r)rrsubLASTHARAKA_PATTERNrs r;strip_lastharakars6" 9    96,c488 8 Kr=cl|s|St|r tD]}||d}|S)uStrip vowels from a text, include Shadda. The striped marks are : - FATHA, DAMMA, KASRA - SUKUN - SHADDA - FATHATAN, DAMMATAN, KASRATAN,, , . Example: >>> text = u"الْعَرَبِيّةُ" >>> strip_tashkeel(text) العربية @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r)rrNrrs r;strip_tashkeelrsJ$ * d  * * *D<<b))DD Kr=cN|s|StD]}||d}|S)upStrip small_letters from a text The striped marks are : - Small Alef الألف الخنجرية, . -Small WAW -Small Yeh Example: >>> text = u"الرحمنٰ" >>> strip_small(text) الرحمن @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r)rjrrs r; strip_smallrs:  &&||D"%% Kr=c8|tdS)u# Strip tatweel from a text and return a result text. Example: >>> text = u"العـــــربية" >>> strip_tatweel(text) العربية @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r)rrHrs r; strip_tatweelrs << $ $$r=c8|tdS)u Strip Shadda from a text and return a result text. Example: >>> text = u"الشّمسيّة" >>> strip_shadda(text) الشمسية @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r)rrErs r; strip_shaddar*s << # ##r=cN|s|StD]}||d}|S)uStrip arabic diacritics from a text The striped marks are : - Small Alef الألف الخنجرية, . - Harakat + Shadda - Quranic marks - Extended arabic diacritics Example: >>> text = u"الرحمنٰ" >>> strip_small(text) الرحمن @param text: arabic text. @type text: unicode. @return: return a striped text. @rtype: unicode. r) DIACRITICSrrs r;strip_diacriticsr:s:"  &&||D"%% Kr=c\|r)ttt|S|S)udNormalize Lam Alef ligatures into two letters (LAM and ALEF), and Tand return a result text. Some systems present lamAlef ligature as a single letter, this function convert it into two letters, The converted letters into LAM and ALEF are : - LAM_ALEF, LAM_ALEF_HAMZA_ABOVE, LAM_ALEF_HAMZA_BELOW, LAM_ALEF_MADDA_ABOVE Example: >>> text = u"لانها لالء الاسلام" >>> normalize_ligature(text) لانها لالئ الاسلام @param text: arabic text. @type text: unicode. @return: return a converted text. @rtype: unicode. )LIGUATURES_PATTERNrLAMALEFrs r;normalize_ligaturerQs1$ C!%%dd&;TBBB Kr=uniformc|dks|dkr|tt}|tt}|tt}|t t}|t t}|tt}|tt}n| tr{t|dkrN|dtvr?|dtkst|dkrttz|ddz}nttz|ddz}|tttz}t t|}|S)uStandardize the Hamzat into one form of hamza, replace Madda by hamza and alef. Replace the LamAlefs by simplified letters. Example: >>> import pyarabic.araby as araby >>> text1 = u"جاء سؤال الأئمة عن الإسلام آجلا" >>> araby.normalize_hamza(text1) 'جاء سءال الءءمة عن الءسلام ءءجلا' >>> araby.normalize_hamza(text1, method="tasheel") 'جاء سوال الايمة عن الاسلام ا @param word: arabic text. @type word: unicode. @param method: how to convert hamzat (uniform, tasheel). @type method: unicode. @return: return a converted text. @rtype: unicode. tasheelu تسهيلrrrN)r ALEF_MADDArALEF_HAMZA_ABOVEALEF_HAMZA_BELOW HAMZA_ABOVE HAMZA_BELOWrWAWrYEH startswithrrQrEHAMZAHAMZAT_PATTERNr)rmethods r;normalize_hamzarhsO(f 55||J--||,d33||,d33||K..||K..||Is++||Is++ ??: & & 04yyA~~47'#9#9!W&&#d))q..t|d122h.u}tABBx/||J 66!!%.. Kr=cttjddtzdzt|S)u converts TEH_MARBUTA to HEH Example: >>> text = 'محبة' >>> normalize_teh(text) 'محبه' r1rr2)rrjoinrHEHrs r; normalize_tehrs5 6$+...5t  r=c|ttzt}|ttzt}tjt t |S)zP converts all alefs to ALEF_MAMDODA with the exception of Alef maksura )r SMALL_ALEFrrrALEFAT_PATTERNrrs r;normalize_alefrsH << </ > >D << Z/ > >D 6.$ - --r=Fc|tj|}|jtj}tj}t}|}||vr|}||v|dkr||vr*|||n|tkrc||t|t|tn/|t|||}|dk|rd |j}tj dtzt|}tj ttt|}t|}|d |j|fSd |jd |jfS)u* separate the letters from the vowels, in arabic word, if a letter hasn't a haraka, the not definited haraka is attributed. return (letters, vowels) Example: >>> araby.separate(text) (u'العربية', u'َََََُْ') >>> letters, marks =araby.separate(text) >>> print letters.encode('utf8') العربية >>> print marks.encode('utf8') >>> for m in marks: ... print araby.name(m) فتحة سكون فتحة فتحة فتحة فتحة ضمة @param word: the input word @type word: unicode @param extract_shadda: extract shadda as seperate text @type extract_shadda: Boolean @return: (letters, vowels) @rtype:couple of unicode Nrz[^%s])r StackitemsreverserQpoppushrEr@NOT_DEF_HARAKArrrrHr) rextract_shaddastack1lettersmarksvowelslast1 wordletters shaddaplacess r;separaters>[  F LkmmG KMME F JJLLE 6//  6// 4-- F?? IIKKK JJu     f__ IIKKK JJu    JJ~ & & & LL JJ~ & & & LL    # 4--$ @hhw}-- vh/+FF v&&96*,, #;// SXXek22LAA''%+)>)>??r=ct|t|krdStj|}|jtj|}|jtj}|}|}t }|dkr|dkr|tkrc|}||vr|||||tkr||n5|||tkr|||}|}|dkr|dk| r| sdSd |jS)u joint the letters with the marks the length ot letters and marks must be equal return word Example: >>> letters = u"العربية" >>> marks = u'َََََُْ' >>> word = araby.joint(letters, marks) >>> print word.encode('utf8') اَلْعَرَبَيَةُ @param letters: the word letters @type letters: unicode @param marks: the word marks @type marks: unicode @return: word @rtype: unicode rNF) rr rrrrrQrErris_emptyr) rr stack_letter stack_mark word_stack last_letter last_markrtops r;jointrs( 7||s5zz!!r;w''L   U##JJ""$$K  I F   )t"3"3 & ..""C&  $$$ OOK ( ( (N** *** OOK ( ( (N** ***"&&(( NN$$    )t"3"3  ! ! # #) (;(;(=(=)uwwz'(((r=c2t||dkrdSdS)u if the two words has the same letters and the same harakats, this fuction return True. The two words can be full vocalized, or partial vocalized Example: >>> word1 = u"ضَربٌ" >>> word2 = u"ضَرْبٌ" >>> araby.vocalizedlike(word1, word2) True @param word1: first word @type word1: unicode @param word2: second word @type word2: unicode @return: if two words have similar vocalization @rtype: Boolean rFT)vocalized_similarity)word1word2s r; vocalizedliker-s#$E5))A--utr=ctj|}tj|}tj}|}|}t}|dkr|dkr||kr?|tt t fvr)|}|}n||vrT|tt t fvr>|||}|}n;||vr||vr|}n||vr||vr|}nn |dkr|dk|j | r| sdS|dks|dkrdS|rd |jSdS)u}If the word1 is like a wazn (pattern), and can return root the letters must be equal, the wazn has FEH, AIN, LAM letters. this are as generic letters. The two words can be full vocalized, or partial vocalized Example: >>> word1 = u"ضارب" >>> wazn = u"فَاعِل" >>> araby.waznlike(word1, wazn) True @param word1: input word @type word1: unicode @param wazn: given word template وزن @type wazn: unicode @param extract_root: return root if True @type extract_root: boolean @return: if two words have similar vocalization @rtype: Boolean NFrT) r rrrQFEHAINrrrrrr) rwazn extract_rootrstack2rootrlast2rs r;waznliker Hs,[  F [  F ;==D JJLLE JJLLE F 4--ETMM E>>eCc?::JJLLEJJLLEE & UsCo%=%= IIe   JJLLEJJLLEE f__f!4!4JJLLEE & Uf__JJLLEE  4--ETMM  J OO   &//"3"3 u $%4--u  774:&& &4r=ct|sdSt|st|rdSt|}t|}tj|}tj|}|}|}|dkr|dkr~||kr)|}|}nC|t kr |t krn8|t kr |t kr|}nn |dkr|dk~|r|sdSdS)u8 If the two words has the same letters and the same harakats, this fuction return True. The first word is partially vocalized, the second is fully if the partially contians a shadda, it must be at the same place in the fully Example: >>> word1 = u"ردّ" >>> word2=u"ردَّ" >>> araby.shaddalike(word1, word2) True @param partial: the partially vocalized word @type partial: unicode @param fully: the fully vocalized word @type fully: unicode @return: if contains shadda @rtype: Boolean TFN)rrr rrrEr)partialfullypstackvstackplastvlasts r; shaddalikersB( g  t   :g#6#6uG$$G % E [ ! !F [  F JJLLE JJLLE 4--ETMM E>>JJLLEJJLLEE f__&  f__&JJLLEE  4--ETMM OO  &//"3"3utr=c Bdtdtdtdtd tdtdt dtdtdt ddtdtdtdtdt g}|}|D]}tj |d |}|S) urReduce the Tashkeel, by deleting evident cases. Exmaple: >>> word = u"يُتَسََلَّمْنَ" >>> reduced = araby.reduce_tashkeel(word) >>> print reduced.encode('utf8') يُتسلّمن @param text: the input text fully vocalized. @type text: unicode. @return : partially vocalized text. @rtype: unicode. z(?>> word1 = u"ضَربٌ" >>> word2 = u"ضَرْبٌ" >>> araby.vocalizedlike(word1, word2) True >>> word1 = u"ضَربٌ" >>> word2 = u"ضَرْبٍ" >>> araby.vocalized_similarity(word1, word2) -1 @param word1: first word @type word1: unicode @param word2: second word @type word2: unicode @return: return if words are similar, else return negative number of errors @rtype: Boolean / int rNrT)r rrrQrE)rrrr rr  err_countrs r;rrs@*[  F [  F JJLLE JJLLEI F 4--ETMM E>>JJLLEJJLLEE f__f!4!4JJLLEE & Uf__JJLLEE &   Q # 4--ETMM$1}}ztr=crtjdd|tj}tjd|}|S)u Tokenize text into sentences. Example: >>> text = u"العربية لغة جميلة. والبلاد بعيدة، والشوق زائد" >>> tokens = araby.sentence_tokenize(text) >>> print(tokens) ‎‎['العربية لغة جميلة.', 'والبلاد بعيدة،', 'والشوق زائد'] @param text: the input text. @type text: unicode. @return: list of sentences. @rtype: list. u([.,:;،؟? ])+([ ])+z \1z)rrUNICODEsplit)r sentencess r;sentence_tokenizer"0s3 61,bj Q QDD))I r=c|rtturgtturgt|}d|D}rfd|D}rfdfd|D}|SgS)uj Tokenize text into words. Example: >>> text = u"العربية لغة جميلة." >>> tokens = araby.tokenize(text) >>> print u"\n".join(tokens) ‎العربية ‎لغة ‎جميلة . Example 2 (To remove tashkeel and filter out non-Arabic words:): >>> text = u"ِاسمٌ الكلبِ في اللغةِ الإنجليزية Dog واسمُ الحمارِ Donky" >>> tokenize(text, conditions=is_arabicrange, morphs=strip_tashkeel) ['اسم', 'الكلب', 'في', 'اللغة', 'الإنجليزية', 'واسم', 'الحمار'] Example 3 (This structure will enable us to create functions on the fly and pass them:): >>> text = u"طلع البدر علينا من ثنيات الوداع" >>>tokenize(text, conditions=lambda x: x.startswith(u'ال')) ['البدر', 'الوداع'] @param text: the input text. @type text: unicode. @param conditions: a list of conditions to be applied on tokens, like avoiding non arabic letters. @type conditions: one or list of conditions . @param morphs: a list of morphological change functions to be applied on tokens, like striping tashkeel or normalizing tokens. @type morphs: one or list of morphological functions . @return: list of words. @rtype: list. czg|]8}td|td|9Sr) TOKEN_REPLACEr)r9toks r;r<ztokenize..js?]]]-BSBSTVX[B\B\]-##B,,]]]r=cLg|]tfdD S)c&g|] }|Srr)r9condr's r;r<z'tokenize...ms!3U3U3U$DDII3U3U3Ur=)all)r9r' conditionss @r;r<ztokenize..ms<WWWcs3U3U3U3U*3U3U3U/V/VWcWWWr=c(D] }||}|S)Nr)r'mmorphss r;morphztokenize..morphos&!!A!C&&CC r=c&g|] }|Srr)r9r'r0s r;r<ztokenize..ts!333SeeCjj333r=)typelist TOKEN_PATTERNr )rr,r/tokensr0s `` @r;tokenizer6Cs@    4 ' 'zl <>> text = "حدثنا ابن أبي عامر، قال: رايت مناما" >>> tokens = araby.tokenize_with_location(text) >>> print u"\n".join(tokens) [{'token': 'حدثنا', 'start': 0, 'end': 5}, {'token': 'ابن', 'start': 6, 'end': 9}, {'token': 'أبي', 'start': 10, 'end': 13}, {'token': 'عامر', 'start': 14, 'end': 18}, {'token': 'قال', 'start': 20, 'end': 23}, {'token': 'رايت', 'start': 25, 'end': 29}, {'token': 'مناما','start': 30, 'end': 35} ] @param text: the input text. @type text: unicode. @return: list of dict of (tokens, starts, ends). @rtype: list of dict. )tokenstartend)TOKEN_PATTERN_SPLITfinditerrr:r;)rr5rs r;tokenize_with_locationr>ys.F$--d33   %++--45[[]]99;;      Mr=cbtd|}|S)z czd|dddS)Nz{} r r)formatgroupr)r:s r;zfix_spaces..s, QWWQZZ5G5GR5P5P(Q(Qr=)FIX_SPACES_PATrstriprs r; fix_spacesrGs,   QQSW X XD ::<<r=c htjdtzd|tj}tjdtzd|tj}tjtt zt tz|tj}tjdtt tdtdd|tj}tjdtdtd d|tj}tjdtdtdd|tj}|S) z Correct most common errors on word like repetetion of harakats,or tanwin befor alef @param text: input text @type text: unicode @return: corrected text @rtype: unicode z(?<=[\s\d])([%s])+rz^([%s])+z(?<=[z])([z])+z([z])+(?=[r) rrTASHKEEL_STRINGrrFATHATANrrr@HARAKAT_STRINGrErs r; autocorrectrLs 6'9"TBJ O OD 6+/4"* E ED 6$x-D C CD 6644{{{EEERSUX\^`^h i iD 66nnnfff=brz R RD 66~~~~~~Fr$QSQ[ \ \D Kr=arcg}|dkr.|D]*}|tj||+n(|D]%}|t||&d|S)z. write the word in full letter' names unicodez, )rr6r}r)rlangnamescs r;spellitrSs E y 1 1A LL)!Q// 0 0 0 0 1 % %A LLa $ $ $ $ ::e  r=zlike: uمُتَوَهِّمًاuمتوهمًاzsim: uثمّu ثُمَّuثمu"العربية: لغة جميلة.)uالعربية:uلغةu جميلة.z use tokenizer%)r)F)rM)__doc__ __future__rrrrrr6r8rr__name__r rpyarabic.stackpyarabicCOMMA SEMICOLONQUESTIONrrrrrrrBEHrTEHTHEHJEEMHAHKHAHDALTHALREHZAINSEENSHEENSADDADTAHZAHrGHAINrHrQAFKAFrMEEMNOONrrrr MADDA_ABOVErrZEROONETWOTHREEFOURFIVESIXSEVENEIGHTNINEZERO_WONE_WTWO_WTHREE_WFOUR_WFIVE_WSIX_WSEVEN_WEIGHT_WNINE_WZERO_PONE_PTWO_PTHREE_PFOUR_PFIVE_PSIX_PSEVEN_PEIGHT_PNINE_PPERCENTDECIMAL THOUSANDSSTAR MINI_ALEF ALEF_WASLA FULL_STOPBYTE_ORDER_MARKrJDAMMATANKASRATANrrrrEr@r SMALL_WAW SMALL_YEHrrLAM_ALEF_HAMZA_BELOWrSIMPLE_LAM_ALEFSIMPLE_LAM_ALEF_HAMZA_ABOVESIMPLE_LAM_ALEF_HAMZA_BELOWSIMPLE_LAM_ALEF_MADDA_ABOVErLETTERS NUMBERS_EAST NUMBERS_WEST NUMBERS_PERSrNrQrTrKrrWrZr]rmr`rdrgrjrqrtrwr{ HAMZAT_STRINGrKrIcompilerrrSHORTHARAKAT_PATTERNTASHKEEL_PATTERNrrrr4r<r&rrrErrrCrFrIrLrOrRrUrXr[r^rarerhrkrnrrruryr}rrrrrrrrrrrrrrrrrrrrrrrrrr rrrr"r6strr3r>rGrLrSprinttext1wordlistwlrr=r;rsK(   FF FFF zLLLL                                               !--- ((PDP#PsPKPPtPSP$PPP"P$(P*.P05P7:P lJ 1 c3 %  < 3  9 %   Y * z+-=t T3c5#sCsCN D#tS$eS# #sD#! !!(!-116:A !!!1&)2/3R9=r 2BR"%b+.r47 2 B R "%b +.r 48   "  2  B  !$R  */  5?  b  $R  *:2 r. ). . . ' .   ) .   ) . .  ). .  ). .  ).  ).  9. .  !. "#. . $%. &'. ( 9). *+. ,-. ./. 01. 2 )3. 4 )5. 67. 89. :;. < ;=. > ]?. @&A. B(C. D3E. . F,3 , o m o ; 9 ; I ;[. . ^  '""((8$$"*TCHHW$5$55 KKD388F#3#33d:BJGGD388F#3#33d:BJGGRZsxx ';'; ;d BBJOO :BJGG  bj!?LL +,,  &h(<(<>R>R>RTUWU_aa rz@"*NN GTT b bvv!6!6 b b b p     """        + + + + + + +*   $222      >   8.2.%%%$$$$ ..))))X   ...I@I@I@I@X2)2)2)j67777t222j&&&R000`&B4444lB 6     z E(MM"=?PQQRRR E''' =AABBB E(MM)];;<<< E'''??@@@ E(MM'=99::: E'''(CEVWWXXX E'''(CEVWWXXX 0EHHHH %B E/ E"IIIII+s %%