# -*- coding: utf-8 -*-
"""
Analogy analyzer units
----------------------
This module provides analyzer units that analyzes unknown words by looking
at how similar known words are analyzed.
"""
from __future__ import absolute_import, unicode_literals, division
import operator
from pymorphy2.units.base import AnalogyAnalizerUnit
from pymorphy2.units.by_lookup import DictionaryAnalyzer
from pymorphy2.units.utils import (add_parse_if_not_seen, add_tag_if_not_seen,
without_fixed_prefix, with_prefix)
from pymorphy2.utils import word_splits
class _PrefixAnalyzer(AnalogyAnalizerUnit):
def normalizer(self, form, this_method):
prefix = this_method[1]
normal_form = yield without_fixed_prefix(form, len(prefix))
yield with_prefix(normal_form, prefix)
def lexemizer(self, form, this_method):
prefix = this_method[1]
lexeme = yield without_fixed_prefix(form, len(prefix))
yield [with_prefix(f, prefix) for f in lexeme]
[docs]class KnownPrefixAnalyzer(_PrefixAnalyzer):
"""
Parse the word by checking if it starts with a known prefix
and parsing the reminder.
Example: псевдокошка -> (псевдо) + кошка.
"""
terminal = True
ESTIMATE_DECAY = 0.75
MIN_REMINDER_LENGTH = 3
def parse(self, word, word_lower, seen_parses):
result = []
for prefix, unprefixed_word in self.possible_splits(word_lower):
method = (self, prefix)
parses = self.morph.parse(unprefixed_word)
for fixed_word, tag, normal_form, estimate, methods_stack in parses:
if not tag.is_productive():
continue
parse = (
prefix + fixed_word,
tag,
prefix + normal_form,
estimate * self.ESTIMATE_DECAY,
methods_stack + (method,)
)
add_parse_if_not_seen(parse, result, seen_parses)
return result
def tag(self, word, word_lower, seen_tags):
result = []
for prefix, unprefixed_word in self.possible_splits(word_lower):
for tag in self.morph.tag(unprefixed_word):
if not tag.is_productive():
continue
add_tag_if_not_seen(tag, result, seen_tags)
return result
def possible_splits(self, word):
word_prefixes = self.dict.prediction_prefixes.prefixes(word)
word_prefixes.sort(key=len, reverse=True)
for prefix in word_prefixes:
unprefixed_word = word[len(prefix):]
if len(unprefixed_word) < self.MIN_REMINDER_LENGTH:
continue
yield prefix, unprefixed_word
[docs]class UnknownPrefixAnalyzer(_PrefixAnalyzer):
"""
Parse the word by parsing only the word suffix
(with restrictions on prefix & suffix lengths).
Example: байткод -> (байт) + код
"""
terminal = False
ESTIMATE_DECAY = 0.5
def __init__(self, morph):
super(AnalogyAnalizerUnit, self).__init__(morph)
self.dict_analyzer = DictionaryAnalyzer(morph)
def parse(self, word, word_lower, seen_parses):
result = []
for prefix, unprefixed_word in word_splits(word_lower):
method = (self, prefix)
parses = self.dict_analyzer.parse(unprefixed_word, unprefixed_word, seen_parses)
for fixed_word, tag, normal_form, estimate, methods_stack in parses:
if not tag.is_productive():
continue
parse = (
prefix+fixed_word,
tag,
prefix+normal_form,
estimate*self.ESTIMATE_DECAY,
methods_stack+(method,)
)
add_parse_if_not_seen(parse, result, seen_parses)
return result
def tag(self, word, word_lower, seen_tags):
result = []
for _, unprefixed_word in word_splits(word_lower):
tags = self.dict_analyzer.tag(unprefixed_word, unprefixed_word, seen_tags)
for tag in tags:
if not tag.is_productive():
continue
add_tag_if_not_seen(tag, result, seen_tags)
return result
[docs]class KnownSuffixAnalyzer(AnalogyAnalizerUnit):
"""
Parse the word by checking how the words with similar suffixes
are parsed.
Example: бутявкать -> ...вкать
"""
terminal = False
ESTIMATE_DECAY = 0.5
[docs] class FakeDictionary(DictionaryAnalyzer):
""" This is just a DictionaryAnalyzer with different __repr__ """
pass
def __init__(self, morph):
super(KnownSuffixAnalyzer, self).__init__(morph)
self._paradigm_prefixes = list(reversed(list(enumerate(self.dict.paradigm_prefixes))))
max_suffix_length = self.dict.meta['prediction_options']['max_suffix_length']
self._prediction_splits = list(reversed(range(1, max_suffix_length+1)))
self.fake_dict = self.FakeDictionary(morph)
def parse(self, word, word_lower, seen_parses):
result = []
# smoothing; XXX: isn't max_cnt better?
# or maybe use a proper discounting?
total_counts = [1] * len(self._paradigm_prefixes)
for prefix_id, prefix, suffixes_dawg in self._possible_prefixes(word_lower):
for i in self._prediction_splits:
# XXX: this should be counted once, not for each prefix
word_start, word_end = word_lower[:-i], word_lower[-i:]
para_data = suffixes_dawg.similar_items(word_end, self.dict.ee)
for fixed_suffix, parses in para_data:
fixed_word = word_start + fixed_suffix
for cnt, para_id, idx in parses:
tag = self.dict.build_tag_info(para_id, idx)
# skip non-productive tags
if not tag.is_productive():
continue
total_counts[prefix_id] += cnt
# avoid duplicate parses
reduced_parse = fixed_word, tag, para_id
if reduced_parse in seen_parses:
continue
seen_parses.add(reduced_parse)
# ok, build the result
normal_form = self.dict.build_normal_form(para_id, idx, fixed_word)
methods = (
(self.fake_dict, fixed_word, para_id, idx),
(self, fixed_suffix),
)
parse = (cnt, fixed_word, tag, normal_form, prefix_id, methods)
result.append(parse)
if total_counts[prefix_id] > 1:
break
result = [
(fixed_word, tag, normal_form, cnt/total_counts[prefix_id] * self.ESTIMATE_DECAY, methods_stack)
for (cnt, fixed_word, tag, normal_form, prefix_id, methods_stack) in result
]
result.sort(key=operator.itemgetter(3), reverse=True)
return result
def tag(self, word, word_lower, seen_tags):
# XXX: the result order may be different from
# ``self.parse(...)``.
result = []
for prefix_id, prefix, suffixes_dawg in self._possible_prefixes(word_lower):
for i in self._prediction_splits:
# XXX: end should be counted once, not for each prefix
end = word_lower[-i:]
para_data = suffixes_dawg.similar_items(end, self.dict.ee)
found = False
for fixed_suffix, parses in para_data:
for cnt, para_id, idx in parses:
tag = self.dict.build_tag_info(para_id, idx)
if not tag.is_productive():
continue
found = True
if tag in seen_tags:
continue
seen_tags.add(tag)
result.append((cnt, tag))
if found:
break
result.sort(reverse=True)
return [tag for cnt, tag in result]
def _possible_prefixes(self, word):
for prefix_id, prefix in self._paradigm_prefixes:
if not word.startswith(prefix):
continue
suffixes_dawg = self.dict.prediction_suffixes_dawgs[prefix_id]
yield prefix_id, prefix, suffixes_dawg