Code666 (代码贴、代码片段)

[Python] 一个非常高效的提取内容关键词的python代码 →→→→→进入此内容的聊天室

来自 , 2019-09-23, 写在 Python, 查看 138 次.
URL http://www.code666.cn/view/e3670ce0
  1. # coding=UTF-8
  2. import nltk
  3. from nltk.corpus import brown
  4.  
  5. # This is a fast and simple noun phrase extractor (based on NLTK)
  6. # Feel free to use it, just keep a link back to this post
  7. # http://thetokenizer.com/2013/05/09/efficient-way-to-extract-the-main-topics-of-a-sentence/
  8. # http://www.sharejs.com/codes/
  9. # Create by Shlomi Babluki
  10. # May, 2013
  11.  
  12.  
  13. # This is our fast Part of Speech tagger
  14. #############################################################################
  15. brown_train = brown.tagged_sents(categories='news')
  16. regexp_tagger = nltk.RegexpTagger(
  17.     [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
  18.      (r'(-|:|;)$', ':'),
  19.      (r'\'*$', 'MD'),
  20.      (r'(The|the|A|a|An|an)$', 'AT'),
  21.      (r'.*able$', 'JJ'),
  22.      (r'^[A-Z].*$', 'NNP'),
  23.      (r'.*ness$', 'NN'),
  24.      (r'.*ly$', 'RB'),
  25.      (r'.*s$', 'NNS'),
  26.      (r'.*ing$', 'VBG'),
  27.      (r'.*ed$', 'VBD'),
  28.      (r'.*', 'NN')
  29. ])
  30. unigram_tagger = nltk.UnigramTagger(brown_train, backoff=regexp_tagger)
  31. bigram_tagger = nltk.BigramTagger(brown_train, backoff=unigram_tagger)
  32. #############################################################################
  33.  
  34.  
  35. # This is our semi-CFG; Extend it according to your own needs
  36. #############################################################################
  37. cfg = {}
  38. cfg["NNP+NNP"] = "NNP"
  39. cfg["NN+NN"] = "NNI"
  40. cfg["NNI+NN"] = "NNI"
  41. cfg["JJ+JJ"] = "JJ"
  42. cfg["JJ+NN"] = "NNI"
  43. #############################################################################
  44.  
  45.  
  46. class NPExtractor(object):
  47.  
  48.     def __init__(self, sentence):
  49.         self.sentence = sentence
  50.  
  51.     # Split the sentence into singlw words/tokens
  52.     def tokenize_sentence(self, sentence):
  53.         tokens = nltk.word_tokenize(sentence)
  54.         return tokens
  55.  
  56.     # Normalize brown corpus' tags ("NN", "NN-PL", "NNS" > "NN")
  57.     def normalize_tags(self, tagged):
  58.         n_tagged = []
  59.         for t in tagged:
  60.             if t[1] == "NP-TL" or t[1] == "NP":
  61.                 n_tagged.append((t[0], "NNP"))
  62.                 continue
  63.             if t[1].endswith("-TL"):
  64.                 n_tagged.append((t[0], t[1][:-3]))
  65.                 continue
  66.             if t[1].endswith("S"):
  67.                 n_tagged.append((t[0], t[1][:-1]))
  68.                 continue
  69.             n_tagged.append((t[0], t[1]))
  70.         return n_tagged
  71.  
  72.     # Extract the main topics from the sentence
  73.     def extract(self):
  74.  
  75.         tokens = self.tokenize_sentence(self.sentence)
  76.         tags = self.normalize_tags(bigram_tagger.tag(tokens))
  77.  
  78.         merge = True
  79.         while merge:
  80.             merge = False
  81.             for x in range(0, len(tags) - 1):
  82.                 t1 = tags[x]
  83.                 t2 = tags[x + 1]
  84.                 key = "%s+%s" % (t1[1], t2[1])
  85.                 value = cfg.get(key, '')
  86.                 if value:
  87.                     merge = True
  88.                     tags.pop(x)
  89.                     tags.pop(x)
  90.                     match = "%s %s" % (t1[0], t2[0])
  91.                     pos = value
  92.                     tags.insert(x, (match, pos))
  93.                     break
  94.  
  95.         matches = []
  96.         for t in tags:
  97.             if t[1] == "NNP" or t[1] == "NNI":
  98.             #if t[1] == "NNP" or t[1] == "NNI" or t[1] == "NN":
  99.                 matches.append(t[0])
  100.         return matches
  101.  
  102.  
  103. # Main method, just run "python np_extractor.py"
  104. def main():
  105.  
  106.     sentence = "Swayy is a beautiful new dashboard for discovering and curating online content."
  107.     np_extractor = NPExtractor(sentence)
  108.     result = np_extractor.extract()
  109.     print "This sentence is about: %s" % ", ".join(result)
  110.  
  111. if __name__ == '__main__':
  112.     main()
  113. #//python/8752

回复 "一个非常高效的提取内容关键词的python代码"

这儿你可以回复上面这条便签

captcha

贴代码片段 - Code666