#!/usr/bin/python # -*- coding=utf-8 -*- #----------------------------------------------------------------------- # Name: collocations # Purpose: Arabic automatic vocalization. # # Author: Taha Zerrouki (taha.zerrouki[at]gmail.com) # # Created: 31-10-2011 # Copyright: (c) Taha Zerrouki 2011 # Licence: GPL #----------------------------------------------------------------------- """ Arabic _collocations Class """ import maskouk.collocationdictionary as collocationdictionary import maskouk.collocation_const as cconst import pyarabic.araby as araby import re class CollocationClass: """ Arabic _collocations Class """ def __init__(self, show_delimiter = False): self.mini = 2 self.maxi = 5 # used to quote the collocations self.delimiter = u"'" # used to mention the unvocalized collocation # this feature is temporary to get the user feed back about #unvocalized collocations self.unknown_delimiter = u"~" self.show_delimiter = show_delimiter COLLOCATION_DICTIONARY_INDEX = { u'id':0, u'vocalized':1, u'unvocalized':2, u'rule':3, u'category':4, u'note':5, } #cache of collocations to speedup the process self.collo_cache = {} # enable and disable _cache use self.cache_enabled = True #from dictionaries.verb_dictionary import * self.collo_dict = collocationdictionary.collocationDictionary( 'collocations', COLLOCATION_DICTIONARY_INDEX) for key in cconst.GENERAL_COLLOCATIONS.keys(): self.collo_cache[key] = \ cconst.GENERAL_COLLOCATIONS[key]['v'] def set_min(self, min0): """ set minimum """ # to avoid errers, verify the min self.mini = min(min0, self.mini) self.maxi = max(min0, self.maxi) def set_max(self, max0): """ set maximum """ # to avoid errers, verify the max self.mini = min(max0, self.mini) self.maxi = max(max0, self.maxi) def set_delimiter(self, delimiter): """ set the delimiter for collocations output @param delimiter : the given delimiter. @type delimiter : one unicode char. """ self.delimiter = delimiter def set_unknown_delimiter(self, delimiter): """ set the delimiter for unvocalized collocations output @param delimiter: the given delimiter. @type delimiter: one unicode char. """ self.unknown_delimiter = delimiter def enable_show_delimiter(self): """ Enable the option to show collocation delimiters """ self.show_delimiter = True def disable_show_delimiter(self): """ Enable the option to show collocation delimiters """ self.show_delimiter = False def enable_cache(self): """ Enable the option to enable cache """ self.cache_enabled = True def disable_cache(self): """ disable the option to show collocation delimiters """ self.cache_enabled = False def is_collocated(self, wordlist): """ Return The vocalized text if the word list is collocated. Example: >>> wlist = [u"كرة", u"القدم"] >>> # test if collocation exists >>> results = mydict.is_collocated(wlist) >>> print("inuput:", wlist) >>> print("output:",results) inuput: ['كرة', 'القدم'] output: كرة القدم >>> wlist = [u"شمس", u"النهار"] >>> results = mydict.is_collocated(wlist) >>> print("inuput:", wlist) >>> print("output:",results) inuput: ['شمس', 'النهار'] output: False @param wordlist: word of list, 2 or more words. @type wordlist: list of unicode. @return : The collocation as a key if exists. else False. @rtype: dict/None. """ # The fisrt case is the two words collocation # get two element from the list start key = u' '.join(wordlist) if len(wordlist)<2: return False elif len(wordlist)>2: # the more than 2 words collocations are a small list # mentioned in Gcconst.GENRAL_COLLOCATIONS # key = u' '.join(wordlist) if self.cache_enabled and key in self.collo_cache: return key elif not cconst.token_pat.search(key) : # invalid words return False else: # get two element from the list start key = u' '.join(wordlist) # if the key existss in the cache. if self.cache_enabled and key in self.collo_cache: return key #print "ok",key.encode('utf8') idlist = self.collo_dict.lookup(key) # if the wordlist as key existes in collocation database, # insert its vocalization in a collocation cache dict if len(idlist) >= 1 : if self.cache_enabled and not key in self.collo_cache: first_entry = idlist[0] self.collo_cache[key] = first_entry['vocalized'] return key else: #before return false we can strip samm prefix from the clause # for example : الحمد لله # is a collocation, , but بالحمد لله is not found if key[0] in (araby.FEH, araby.WAW, araby.BEH, araby.LAM, araby.KAF): first_letter = key[0] new_key = key[1:] #print key.encode('utf8') #look up for the new key idlist = self.collo_dict.lookup(new_key) # if the wordlist as key existes in collocation data base, # insert its vocalization in a collocation cache dict if len(idlist) >= 1 : if self.cache_enabled and not \ key in self.collo_cache: vocalized_collocation = idlist[0]['vocalized'] # some fields in database are not vocalized if vocalized_collocation != "": # save the variant of collocation also, by key self.collo_cache[key] = first_letter + \ vocalized_collocation else: # if vocalized is empty, save a empty #string in cache # if the returned string is empty the program #suggest to vocalized the collocation self.collo_cache[key] = vocalized_collocation # save the found collocation in cache by new_key if not new_key in self.collo_cache: self.collo_cache[new_key] = \ vocalized_collocation # return the given key return key else: return False return False return False def ngramfinder(self, mini, wordlist): """ Lookup for ngram (min number of words), in the word list. return a list of single words and collocations. Example: >>> # detect collocations in phrase >>> text = u"لعبنا مباراة كرة القدم في بيت المقدس" >>> wordlist = araby.tokenize(text) >>> results = mydict.ngramfinder(2, wordlist) >>> print("inuput:", text) >>> print("output:",results) inuput: لعبنا مباراة كرة القدم في بيت المقدس output: ['لعبنا', 'مباراة', 'كرة القدم', 'في', 'بيت المقدس'] @param wordlist: word of list, 2 or more words. @type wordlist: list of unicode. @param mini: minimum number of words in the collocation @type mini: integer. @return : list of words and collocations, else False. @rtype: list /None. """ liste = wordlist newlist = [] while len(liste) >= mini: sublist = [] for i in range(mini): current = liste.pop() # if araby.isArabicword(current): sublist.insert(0, current) # else: # sublist = [] # break if sublist and self.is_possible_collocation(sublist): #for x in sublist: # if not araby.is_arabicword(x): # result = False; # break; #else: # if self.is_possible_collocation(sublist): # result = self.is_collocated(sublist) # else: # result = False result = self.is_collocated(sublist) if result: newlist.append(result) else: newlist.append(sublist.pop()) liste.extend(sublist) # rest element liste.reverse() newlist.extend(liste) newlist.reverse() return newlist def lookup4long_collocations(self, inputtext): """ Lookup for long collocations in a text. return a vocalized words collocations. Example: >>> # get Long collocations >>> text = u" قلت لهم السلام عليكم ورحمة الله تعالى وبركاته ثم رجعت" >>> results = mydict.lookup4long_collocations(text) >>> print("inuput:", text) inuput: قلت لهم السلام عليكم ورحمة الله تعالى وبركاته ثم رجعت >>> print("output:",results) output: قلت لهم السّلامُ عَلَيكُمْ وَرَحْمَةُ اللهِ تَعَالَى وبركاته ثم رجعت @param inputtext: given text @type inputtext: unicode. @return : text. @rtype: unicode. """ for k in cconst.GENERAL_COLLOCATIONS.keys(): inputtext = re.sub( k, cconst.GENERAL_COLLOCATIONS[k].get('v', ''), inputtext ) return inputtext def lookup(self, wordlist): """ Lookup for all ngrams , in the word list. return a list of vocalized words collocations Example: >>> # detect collocations in phrase >>> text = u"لعبنا مباراة كرة القدم في بيت المقدس" >>> wordlist = araby.tokenize(text) >>> results = mydict.lookup(wordlist) >>> print("inuput:", text) >>> print("output:",results) inuput: لعبنا مباراة كرة القدم في بيت المقدس output: (['لعبنا', 'مباراة', 'كُرَة', 'الْقَدَمِ', 'في', 'بَيْت', 'الْمَقْدِسِ'], ['CO', 'CO', 'CB', 'CI', 'CO', 'CB', 'CI']) @param wordlist: word of list, 2 or more words. @type wordlist: list of unicode. @return : dict of words attributes like dict {'vocalized': vocalizedword list, 'category': categoryOf_collocation}. else False. @rtype: dict of dict /None. """ # first lookup for all collocations of each word # if we have text like "I am an new world" # we lookup for evrey word, all collocations put it in a cache system previous = False taglist =[] vocalized_list=[] for word in wordlist: if not word in self.collo_cache: # get all collocations starting with word result = self.is_collocated_word(word) # result is False or a list of collocations self.collo_cache[word] = result #word_collocation(result) if previous: if word in self.collo_cache[previous].keys(): # remove the previous tag taglist.pop() taglist.append("CB") # B means Begin taglist.append("CI") # I means Intern vocalized_tuple = self.collo_cache[previous][word] vocalized_list.pop() vocalized_list.extend(vocalized_tuple.split(" ")) else: taglist.append("CO") vocalized_list.append(word) else: taglist.append("CO") vocalized_list.append(word) previous = word #return zip(taglist, vocalized_list) #print taglist #print u" ".join(vocalized_list).encode('utf8') return vocalized_list, taglist def is_collocated_word(self, word): """ Return The list of collocations started by given word, else False. Example: >>> # get all collocations for a specific word >>> word1 = u"كرة" >>> results = mydict.is_collocated_word(word1) >>> print("inuput:", word1) >>> print("output:",results) inuput: كرة output: {'القدم': 'كُرَة الْقَدَمِ'} >>> >>> word = u"بيت" >>> # get all collocations for a specific word >>> results = mydict.is_collocated_word(word) >>> print("inuput:", word) >>> print("output:",results) inuput: بيت output: {'العدة': 'بَيْت الْعِدَّةِ', 'المستأجر': 'بَيْت الْمُسْتَأْجِرِ', 'المشتري': 'بَيْتِ الْمُشْتَرِي', 'الرجل': 'بَيْت الرَّجُلِ', 'البناء': 'بَيْت الْبِنَاءِ', 'الزوج': 'بَيْت الزَّوْجِ', 'المال': 'بيت المال', 'المقدس': 'بَيْت الْمَقْدِسِ', 'البائع': 'بَيْت الْبَائِعِ', 'الخلاء': 'بَيْت الْخَلَاءِ', 'الأب': 'بَيْت الْأَبِ', 'الله': 'بَيْت اللّهِ'} @param word: input word. @type word: unicode. @return : dict of collocations and vocalized collocations if exists, the keys are second words in collocations. else False. @rtype: dict/Boolean. """ if re.search("[::pounctuation::]", word): return {} result = {} # lookup the word in dict collocations idlist = self.collo_dict.lookup(word, singleword = True) # if the wordlist as key existes in collocation database, # insert its vocalization in a collocation cache dict if len(idlist) >= 1 : for item in idlist: # the key is the second word in collocation key = item["unvocalized"].split(" ")[1] if item["vocalized"]: # some collocations have not a vocalized case result[key] = item["vocalized"] else: #before return false we can strip some prefixes from the clause # for example : الحمد لله # is a collocation, , but بالحمد لله is not found if word[0] in (araby.FEH, araby.WAW, araby.BEH, araby.LAM, araby.KAF): first_letter = word[0] new_word = word[1:] #look up for the new key idlist = self.collo_dict.lookup(new_word, singleword = True) # if the wordlist as key existes in collocation data base, # insert its vocalization in a collocation cache dict if len(idlist) >= 1 : for item in idlist: # the key is the second word in collocation key = item["unvocalized"].split(" ")[1] result[key] = first_letter + item["vocalized"] return result def is_possible_collocation(self, wordlist, context = "", length = 2): """ Guess if the given list is a possible collocation This is used to collect unkown collocations, from user input return True oor false Example: >>> text = u"ظهر رئيس الوزراء السيد عبد الملك بن عامر ومعه أمير دولة غرناطة ونهر النيل انطلاق السباق" >>> wordlist = araby.tokenize(text) >>> previous = "__" >>> for wrd in wordlist: >>> wlist = [previous, wrd] >>> results = mydict.is_possible_collocation(wlist, length = 2) >>> print("inuput:", wlist) >>> print("output:", results) >>> previous = wrd inuput: ['__', 'ظهر'] output: 100 inuput: ['ظهر', 'رئيس'] output: 100 inuput: ['رئيس', 'الوزراء'] output: 100 inuput: ['الوزراء', 'السيد'] output: 20 inuput: ['السيد', 'عبد'] output: 100 inuput: ['عبد', 'الملك'] output: 15 inuput: ['الملك', 'بن'] output: 100 inuput: ['بن', 'عامر'] output: 15 inuput: ['عامر', 'ومعه'] output: 100 inuput: ['ومعه', 'أمير'] output: 100 inuput: ['أمير', 'دولة'] output: 100 inuput: ['دولة', 'غرناطة'] output: 10 inuput: ['غرناطة', 'ونهر'] output: 100 inuput: ['ونهر', 'النيل'] output: 100 inuput: ['النيل', 'انطلاق'] output: 100 inuput: ['انطلاق', 'السباق'] output: 100 @param wordlist: word of list, 2 or more words. @type wordlist: list of unicode. @param length: minimum number of words in the collocation @type length: integer. @return : the rule of found collocation, 100 default. @rtype: interger. """ list2 = wordlist if len(list2)