-i4dZddlmZmZmZmZddlZddlZddlZ dZ ddl m Z GddZ dZedkr edSdS) zh Arabic Stop Word Dictionary Class from Arramooz Al Waseet. Used in multiporpus morpholigical treatment )absolute_importprint_functionunicode_literalsdivisionNzdata/stopwords.sqlitec>eZdZdZd dZdZdZdZdZdZ d Z d S) StopWordsDictionaryu Arabic dictionary Class Used to allow abstract acces to lexicon of arabic language, can get indexed and hashed entries from the basic lexicon add also, support to extract attributtes from entries Example: >>> mydict = StopWordsDictionary('classedstopwords') >>> wordlist = [u"بعضهما", u'في', u"منً", u"أن", u'عندما'] >>> tmp_list =[] >>> for word in wordlist: >>> print("-------") >>> print("word looked up ", mydict.is_stopword(word)) >>> idlist = mydict.lookup(word) >>> for word_tuple in idlist: >>> tmp_list.append(dict(word_tuple)) >>> print(tmp_list) [{'definition': 0, 'qasam': 1, 'object_type': u'اسم', 'vocalized': u'فِي', 'conjonction': 1, 'pronoun': 1, 'defined': 0, 'interrog': 1, 'is_inflected': 0, 'word_type': u'حرف', 'preposition': 0, 'need': u'', 'conjugation': 0, 'word_class': u'حرف جر', 'action': u'جار', 'WORD': u'في', 'ID': 203, 'tanwin': 0}, {'definition': 0, 'qasam': 0, 'object_type': u'اسم', 'vocalized': u'فِي', 'conjonction': 1, 'pronoun': 1, 'defined': 0, 'interrog': 0, 'is_inflected': 0, 'word_type': u'اسم', 'preposition': 1, 'need': u'', 'conjugation': 0, 'word_class': u'الأسماء الخمسة', 'action': u'جار', 'WORD': u'في', 'ID': 304, 'tanwin': 0}, {'definition': 0, 'qasam': 1, 'object_type': u'فعل', 'vocalized': u'أَنْ', 'conjonction': 1, 'pronoun': 0, 'defined': 0, 'interrog': 0, 'is_inflected': 0, 'word_type': u'حرف', 'preposition': 1, 'need': u'', 'conjugation': 0, 'word_class': u'حرف نصب', 'action': u'ناصب', 'WORD': u'أن', 'ID': 168, 'tanwin': 0}, {'definition': 0, 'qasam': 0, 'object_type': u'اسم', 'vocalized': u'أَنَّ', 'conjonction': 1, 'pronoun': 1, 'defined': 0, 'interrog': 0, 'is_inflected': 0, 'word_type': u'حرف', 'preposition': 1, 'need': u'', 'conjugation': 0, 'word_class': u'إن و أخواتها', 'action': u'ناصب', 'WORD': u'أن', 'ID': 481, 'tanwin': 0}] unvocalizedc0i|_||_||_ttdr tj}nAt jt j t}t j |t}t j |rd|zdz} tj|dd|_tj|j_|j|_nh#tj$rt+d|YnGwxYwt+d d |d t jgd d }||d S)zw initialisation of dictionary from a data dictionary, create indexes to speed up the access. frozenzfile:z?mode=roFT)check_same_threaduriz(Fatal Error Can't find the database file zInexistant Filez current dir utf8r N) dictionary key_attribute table_namehasattrsysprefixospathdirnamerealpath__file__join FILE_DB_FREQexistssqliteconnect db_connectRow row_factorycursorOperationalErrorprintcurdirencodecreate_table_index)selfrrbase file_pathfile_uri index_fields _/var/www/html/speakWrite/venv/lib/python3.11/site-packages/arramooz/stopwordsdictionaryclass.py__init__zStopWordsDictionary.__init__6st+ % 3 ! ! AZDDW__RW%5%5h%?%?@@DGLL|44 7>>) $ $ )y(3H 9$*N8eY]$^$^$^ 17 + $ 6 6 8 8 * M M M@)LLLLL M 499/O Y ) ) )$   ,,,,,sDD98D9cJ|jr|jdSdS)zH Delete instance and close database connection N)r close)r)s r.__del__zStopWordsDictionary.__del__`s2 ? $ O ! ! # # # # # $ $cd|jd|d} |j||jrdSdS#tj$rYdSwxYw)z create the database index if not exists @param index_field: the given to be indexed field @type index_field: text @return: void @rtype: void z&create index if not exists myindex on z ()TFNrr#executerr$)r)r-sqls r.r(z&StopWordsDictionary.create_table_indexjsw &  K   $ $ ${ t  &   55 s!5AAcd|jd|d} |j||jr?|jD]7}i}|jD](}|j|}||r ||||<#d||<)|cSn#tj$rYdSwxYwdS)z Get dictionary entry by id from the dictionary @param idf: word identifier @type idf: integer @return: all attributes @rtype: dict select * FROM WHERE id = ''F)rr#r7attrib_num_indexrr$)r)idfr8row entry_dictnum_keytext_keys r.get_entry_by_idz#StopWordsDictionary.get_entry_by_id|s8<L  K   $ $ ${ &;&&C!#J#'#8;;%)%:7%Cw<;58\Jx007:j22%%%%&   55 usAA11BBcd|jd|d} |j||jr|jD] }||cSn#tj$rYdSwxYwdS)a Get attribute value by id from the dictionary @param idf : word identifier @type idf: integer @param attribute:the attribute name @type attribute: unicode @return: The attribute value @rtype: mix. r:r;r<Fr6)r)r? attributer8r@s r.get_attrib_by_idz$StopWordsDictionary.get_attrib_by_idsP8<L  K   $ $ ${ +;++C N***&   55 us3AAAcg}d|jd|d} |j|n5#tj$r#}t d|jd|zYd}~nd}~wwxYw|jr|jD]}|||S)uU look up for all word forms in the dictionary Example: >>> mydict = StopWordsDictionary('classedstopwords') >>> wordlist = [u"بعضهما", u'في', u"منً", u"أن", u'عندما'] >>> tmp_list =[] >>> for word in wordlist: >>> print("-------") >>> print("word looked up ", mydict.is_stopword(word)) >>> idlist = mydict.lookup(word) >>> for word_tuple in idlist: >>> tmp_list.append(dict(word_tuple)) >>> print(tmp_list) [{'definition': 0, 'qasam': 1, 'object_type': u'اسم', 'vocalized': u'فِي', 'conjonction': 1, 'pronoun': 1, 'defined': 0, 'interrog': 1, 'is_inflected': 0, 'word_type': u'حرف', 'preposition': 0, 'need': u'', 'conjugation': 0, 'word_class': u'حرف جر', 'action': u'جار', 'WORD': u'في', 'ID': 203, 'tanwin': 0}, {'definition': 0, 'qasam': 0, 'object_type': u'اسم', 'vocalized': u'فِي', 'conjonction': 1, 'pronoun': 1, 'defined': 0, 'interrog': 0, 'is_inflected': 0, 'word_type': u'اسم', 'preposition': 1, 'need': u'', 'conjugation': 0, 'word_class': u'الأسماء الخمسة', 'action': u'جار', 'WORD': u'في', 'ID': 304, 'tanwin': 0}, {'definition': 0, 'qasam': 1, 'object_type': u'فعل', 'vocalized': u'أَنْ', 'conjonction': 1, 'pronoun': 0, 'defined': 0, 'interrog': 0, 'is_inflected': 0, 'word_type': u'حرف', 'preposition': 1, 'need': u'', 'conjugation': 0, 'word_class': u'حرف نصب', 'action': u'ناصب', 'WORD': u'أن', 'ID': 168, 'tanwin': 0}, {'definition': 0, 'qasam': 0, 'object_type': u'اسم', 'vocalized': u'أَنَّ', 'conjonction': 1, 'pronoun': 1, 'defined': 0, 'interrog': 0, 'is_inflected': 0, 'word_type': u'حرف', 'preposition': 1, 'need': u'', 'conjugation': 0, 'word_class': u'إن و أخواتها', 'action': u'ناصب', 'WORD': u'أن', 'ID': 481, 'tanwin': 0}] @param text:vocalized word. @type text: unicode. @return: list of dictionary entries IDs. @rtype: list. r:z WHERE word = 'r<zer:z%Fatal error in query: stopwords on %sN)rr#r7rErrorr%messageappend)r)textidlistr8err@s r.lookupzStopWordsDictionary.lookups2 ///444! R K   $ $ $ $| R R R %%LS%P Q Q Q Q Q Q Q Q R ; #{ # # c"""" s-AAAc<g}||}|rdSdS)z return the word frequency from the in the dictionary @param text:vocalized word. @type text: unicode. @return: word freq. @rtype: integer. TF)rO)r)rLrMs r. is_stopwordzStopWordsDictionary.is_stopwords,T""  45r3N)r ) __name__ __module__ __qualname____doc__r/r2r(rDrGrOrQr3r.rrs0(-(-(-(-T$$$$>111f(((Tr3rctd}gd}g}|D]p}tdtd||||}|D]$}|t |%qtt |ddddS) z Test main classedstopwords)u بعضهماuفيuمنًuأنu عندماz-------zword looked up z},z}, zunicode-escapeN) rr%rQrOrKdictreprreplacedecode)mydictwordlisttmp_listwordrM word_tuples r.mainlyrb s!!3 4 4FLLLHH.. i !3!3D!9!9:::t$$  . .J OOD,, - - - - . $x.. f - - 4 45E F FGGGGGr3__main__)rU __future__rrrrrrsqlite3rrpyarabic.arabyarabyrrbrRrVr3r.rhs ' jjjjjjjjZ H H H  FHHHHHr3