پیش پردازش متن در پایتون — راهنمای جامع

1input_str = ”The 5 biggest countries by population in 2017 are China, India, United States, Indonesia, and Brazil.”
2input_str = input_str.lower()
3print(input_str)

1import re
2input_str = ’Box A contains 3 red and 5 white balls, while Box B contains 4 red and 2 blue balls.’
3result = re.sub(r’\d+’, ‘’, input_str)
4print(result)

1import string
2input_str = 'This &is [an] example? {of} string. with.? punctuation!!!!' # Sample string
3result = input_str.translate(string.maketrans('',''), string.punctuation)
4print(result)

1This is an example of string with punctuation

1input_str = “ \t a string example\t “
2input_str = input_str.strip()
3input_str

1‘a string example’

1# import the existing word and sentence tokenizing  
2# libraries 
3from nltk.tokenize import sent_tokenize, word_tokenize 
4  
5text = "Natural language processing (NLP) is a field " + \ 
6       "of computer science, artificial intelligence " + \ 
7       "and computational linguistics concerned with " + \ 
8       "the interactions between computers and human " + \ 
9       "(natural) languages, and, in particular, " + \ 
10       "concerned with programming computers to " + \ 
11       "fruitfully process large natural language " + \ 
12       "corpora. Challenges in natural language " + \ 
13       "processing frequently involve natural " + \ 
14       "language understanding, natural language" + \ 
15       "generation frequently from formal, machine" + \ 
16       "-readable logical forms), connecting language " + \ 
17       "and machine perception, managing human-" + \ 
18       "computer dialog systems, or some combination " + \ 
19       "thereof."
20  
21print(sent_tokenize(text)) 
22print(word_tokenize(text))

1[‘Natural language processing (NLP) is a field of computer science, artificial intelligence and computational linguistics concerned with the interactions between computers and human (natural) languages, and, in particular, concerned with programming computers to fruitfully process large natural language corpora.’, ‘Challenges in natural language processing frequently involve natural language understanding, natural language generation (frequently from formal, machine-readable logical forms), connecting language and machine perception, managing human-computer dialog systems, or some combination thereof.’]
2[‘Natural’, ‘language’, ‘processing’, ‘(‘, ‘NLP’, ‘)’, ‘is’, ‘a’, ‘field’, ‘of’, ‘computer’, ‘science’, ‘,’, ‘artificial’, ‘intelligence’, ‘and’, ‘computational’, ‘linguistics’, ‘concerned’, ‘with’, ‘the’, ‘interactions’, ‘between’, ‘computers’, ‘and’, ‘human’, ‘(‘, ‘natural’, ‘)’, ‘languages’, ‘,’, ‘and’, ‘,’, ‘in’, ‘particular’, ‘,’, ‘concerned’, ‘with’, ‘programming’, ‘computers’, ‘to’, ‘fruitfully’, ‘process’, ‘large’, ‘natural’, ‘language’, ‘corpora’, ‘.’, ‘Challenges’, ‘in’, ‘natural’, ‘language’, ‘processing’, ‘frequently’, ‘involve’, ‘natural’, ‘language’, ‘understanding’, ‘,’, ‘natural’, ‘language’, ‘generation’, ‘(‘, ‘frequently’, ‘from’, ‘formal’, ‘,’, ‘machine-readable’, ‘logical’, ‘forms’, ‘)’, ‘,’, ‘connecting’, ‘language’, ‘and’, ‘machine’, ‘perception’, ‘,’, ‘managing’, ‘human-computer’, ‘dialog’, ‘systems’, ‘,’, ‘or’, ‘some’, ‘combination’, ‘thereof’, ‘.’]

1input_str = “NLTK is a leading platform for building Python programs to work with human language data.”
2stop_words = set(stopwords.words(‘english’))
3from nltk.tokenize import word_tokenize
4tokens = word_tokenize(input_str)
5result = [i for i in tokens if not i in stop_words]

1[‘NLTK’, ‘leading’, ‘platform’, ‘building’, ‘Python’, ‘programs’, ‘work’, ‘human’, ‘language’, ‘data’, ‘.’]

1from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS

1from spacy.lang.en.stop_words import STOP_WORDS

1from nltk.stem import PorterStemmer
2from nltk.tokenize import word_tokenize
3stemmer= PorterStemmer()
4input_str=”There are several types of stemming algorithms.”
5input_str=word_tokenize(input_str)
6for word in input_str:
7    print(stemmer.stem(word))

1There are sever type of stem algorithm.

1# import these modules 
2from nltk.stem import PorterStemmer 
3from nltk.tokenize import word_tokenize 
4   
5ps = PorterStemmer() 
6  
7# choose some words to be stemmed 
8words = ["program", "programs", "programer", "programing", "programers"] 
9  
10for w in words: 
11    print(w, " : ", ps.stem(w))

1program  :  program
2programs  :  program
3programer  :  program
4programing  :  program
5programers  :  program

1# importing modules 
2from nltk.stem import PorterStemmer 
3from nltk.tokenize import word_tokenize 
4   
5ps = PorterStemmer() 
6   
7sentence = "Programers program with programing languages"
8words = word_tokenize(sentence) 
9   
10for w in words: 
11    print(w, " : ", ps.stem(w))

1Programers  :  program
2program  :  program
3with  :  with
4programing  :  program
5languages  :  languag

1from nltk.stem import WordNetLemmatizer
2from nltk.tokenize import word_tokenize
3lemmatizer=WordNetLemmatizer()
4input_str=”been had done languages cities mice”
5input_str=word_tokenize(input_str)
6for word in input_str:
7    print(lemmatizer.lemmatize(word))

1be have do language city mouse

1# import these modules 
2from nltk.stem import WordNetLemmatizer 
3  
4lemmatizer = WordNetLemmatizer() 
5  
6print("rocks :", lemmatizer.lemmatize("rocks")) 
7print("corpora :", lemmatizer.lemmatize("corpora")) 
8  
9# a denotes adjective in "pos" 
10print("better :", lemmatizer.lemmatize("better", pos ="a"))

1rocks : rock
2corpora : corpus
3better : good

1# from textblob lib import Word method 
2from textblob import Word 
3  
4# create a Word object. 
5u = Word("rocks") 
6  
7# apply lemmatization. 
8print("rocks :", u.lemmatize()) 
9  
10# create a Word object. 
11v = Word("corpora") 
12  
13# apply lemmatization. 
14print("corpora :", v.lemmatize()) 
15  
16# create a Word object. 
17w = Word("better") 
18   
19# apply lemmatization with  
20# parameter "a", "a" denotes adjective. 
21print("better :", w.lemmatize("a"))

1rocks : rock
2corpora : corpus
3better : good

1input_str=”Parts of speech examples: an article, to write, interesting, easily, and, of”
2from textblob import TextBlob
3result = TextBlob(input_str)
4print(result.tags)

1[(‘Parts’, u’NNS’), (‘of’, u’IN’), (‘speech’, u’NN’), (‘examples’, u’NNS’), (‘an’, u’DT’), (‘article’, u’NN’), (‘to’, u’TO’), (‘write’, u’VB’), (‘interesting’, u’VBG’), (‘easily’, u’RB’), (‘and’, u’CC’), (‘of’, u’IN’)]

1import nltk 
2from nltk.corpus import stopwords 
3from nltk.tokenize import word_tokenize, sent_tokenize 
4stop_words = set(stopwords.words('english')) 
5  
6// Dummy text 
7txt = "Sukanya, Rajib and Naba are my good friends. " \ 
8    "Sukanya is getting married next year. " \ 
9    "Marriage is a big step in one’s life." \ 
10    "It is both exciting and frightening. " \ 
11    "But friendship is a sacred bond between people." \ 
12    "It is a special kind of love between us. " \ 
13    "Many of you must have tried searching for a friend "\ 
14    "but never found the right one."
15  
16# sent_tokenize is one of instances of  
17# PunktSentenceTokenizer from the nltk.tokenize.punkt module 
18  
19tokenized = sent_tokenize(txt) 
20for i in tokenized: 
21      
22    # Word tokenizers is used to find the words  
23    # and punctuation in a string 
24    wordsList = nltk.word_tokenize(i) 
25  
26    # removing stop words from wordList 
27    wordsList = [w for w in wordsList if not w in stop_words]  
28  
29    #  Using a Tagger. Which is part-of-speech  
30    # tagger or POS-tagger.  
31    tagged = nltk.pos_tag(wordsList) 
32  
33    print(tagged)

1[('Sukanya', 'NNP'), ('Rajib', 'NNP'), ('Naba', 'NNP'), ('good', 'JJ'), ('friends', 'NNS')]
2[('Sukanya', 'NNP'), ('getting', 'VBG'), ('married', 'VBN'), ('next', 'JJ'), ('year', 'NN')]
3[('Marriage', 'NN'), ('big', 'JJ'), ('step', 'NN'), ('one', 'CD'), ('’', 'NN'), ('life', 'NN')]
4[('It', 'PRP'), ('exciting', 'VBG'), ('frightening', 'VBG')]
5[('But', 'CC'), ('friendship', 'NN'), ('sacred', 'VBD'), ('bond', 'NN'), ('people', 'NNS')]
6[('It', 'PRP'), ('special', 'JJ'), ('kind', 'NN'), ('love', 'VB'), ('us', 'PRP')]
7[('Many', 'JJ'), ('must', 'MD'), ('tried', 'VB'), ('searching', 'VBG'), ('friend', 'NN'), 
8('never', 'RB'), ('found', 'VBD'), ('right', 'RB'), ('one', 'CD')]

1# from textblob lib import TextBlob method 
2from textblob import TextBlob 
3  
4text = ("Sukanya, Rajib and Naba are my good friends. " +
5    "Sukanya is getting married next year. " +
6    "Marriage is a big step in one’s life." +
7    "It is both exciting and frightening. " + 
8    "But friendship is a sacred bond between people." +
9    "It is a special kind of love between us. " +
10    "Many of you must have tried searching for a friend "+ 
11    "but never found the right one.") 
12  
13# create a textblob object 
14blob_object = TextBlob(text) 
15  
16# Part-of-speech tags can be accessed  
17# through the tags property of blob object.' 
18  
19# print word with pos tag. 
20print(blob_object.tags)

1[('Sukanya', 'NNP'),
2 ('Rajib', 'NNP'),
3 ('and', 'CC'),
4 ('Naba', 'NNP'),
5 ('are', 'VBP'),
6 ('my', 'PRP$'),
7 ('good', 'JJ'),
8 ('friends', 'NNS'),
9 ('Sukanya', 'NNP'),
10 ('is', 'VBZ'),
11 ('getting', 'VBG'),
12 ('married', 'VBN'),
13 ('next', 'JJ'),
14 ('year', 'NN'),
15 ('Marriage', 'NN'),
16 ('is', 'VBZ'),
17 ('a', 'DT'),
18 ('big', 'JJ'),
19 ('step', 'NN'),
20 ('in', 'IN'),
21 ('one', 'CD'),
22 ('’', 'NN'),
23 ('s', 'NN'),
24 ('life.It', 'NN'),
25 ('is', 'VBZ'),
26 ('both', 'DT'),
27 ('exciting', 'VBG'),
28 ('and', 'CC'),
29 ('frightening', 'NN'),
30 ('But', 'CC'),
31 ('friendship', 'NN'),
32 ('is', 'VBZ'),
33 ('a', 'DT'),
34 ('sacred', 'JJ'),
35 ('bond', 'NN'),
36 ('between', 'IN'),
37 ('people.It', 'NN'),
38 ('is', 'VBZ'),
39 ('a', 'DT'),
40 ('special', 'JJ'),
41 ('kind', 'NN'),
42 ('of', 'IN'),
43 ('love', 'NN'),
44 ('between', 'IN'),
45 ('us', 'PRP'),
46 ('Many', 'JJ'),
47 ('of', 'IN'),
48 ('you', 'PRP'),
49 ('must', 'MD'),
50 ('have', 'VB'),
51 ('tried', 'VBN'),
52 ('searching', 'VBG'),
53 ('for', 'IN'),
54 ('a', 'DT'),
55 ('friend', 'NN'),
56 ('but', 'CC'),
57 ('never', 'RB'),
58 ('found', 'VBD'),
59 ('the', 'DT'),
60 ('right', 'JJ'),
61 ('one', 'NN')]

1import spacy 
2  
3# Load English tokenizer, tagger,  
4# parser, NER and word vectors 
5nlp = spacy.load("en_core_web_sm") 
6  
7# Process whole documents 
8text = ("""My name is Shaurya Uppal.  
9I enjoy writing articles on GeeksforGeeks checkout 
10my other article by going to my profile section.""") 
11  
12doc = nlp(text) 
13  
14# Token and Tag 
15for token in doc: 
16  print(token, token.pos_) 
17  
18# You want list of Verb tokens 
19print("Verbs:", [token.text for token in doc if token.pos_ == "VERB"]) 

1My DET
2name NOUN
3is VERB
4Shaurya PROPN
5Uppal PROPN
6. PUNCT
7I PRON
8enjoy VERB
9writing VERB
10articles NOUN
11on ADP
12GeeksforGeeks PROPN
13checkout VERB
14my DET
15other ADJ
16article NOUN
17by ADP
18going VERB
19to ADP
20my DET
21profile NOUN
22section NOUN
23. PUNCT
24
25# Verb based Tagged Reviews:-
26Verbs: ['is', 'enjoy', 'writing', 'checkout', 'going']

1input_str=”A black television and a white stove were bought for the new apartment of John.”
2from textblob import TextBlob
3result = TextBlob(input_str)
4print(result.tags)

1[(‘A’, u’DT’), (‘black’, u’JJ’), (‘television’, u’NN’), (‘and’, u’CC’), (‘a’, u’DT’), (‘white’, u’JJ’), (‘stove’, u’NN’), (‘were’, u’VBD’), (‘bought’, u’VBN’), (‘for’, u’IN’), (‘the’, u’DT’), (‘new’, u’JJ’), (‘apartment’, u’NN’), (‘of’, u’IN’), (‘John’, u’NNP’)]

1reg_exp = “NP: {<DT>?<JJ>*<NN>}”
2rp = nltk.RegexpParser(reg_exp)
3result = rp.parse(result.tags)
4print(result)

1(S (NP A/DT black/JJ television/NN) and/CC (NP a/DT white/JJ stove/NN) were/VBD bought/VBN for/IN (NP the/DT new/JJ apartment/NN)
2of/IN John/NNP)

1result.draw()

1from nltk.tokenize import word_tokenize  
2from nltk import pos_tag 
3  
4# define chunking function with text and regular 
5# expression representing grammar as parameter 
6def chunking(text, grammar): 
7    word_tokens = word_tokenize(text) 
8  
9    # label words with part of speech 
10    word_pos = pos_tag(word_tokens) 
11  
12    # create a chunk parser using grammar 
13    chunkParser = nltk.RegexpParser(grammar) 
14  
15    # test it on the list of word tokens with tagged pos 
16    tree = chunkParser.parse(word_pos) 
17      
18    for subtree in tree.subtrees(): 
19        print(subtree) 
20    tree.draw() 
21      
22sentence = 'the little yellow bird is flying in the sky'
23grammar = "NP: {<DT>?<JJ>*<NN>}"
24chunking(sentence, grammar)

1(S
2(NP the/DT little/JJ yellow/JJ bird/NN)
3is/VBZ
4flying/VBG
5in/IN
6(NP the/DT sky/NN))
7(NP the/DT little/JJ yellow/JJ bird/NN)
8(NP the/DT sky/NN)

1from nltk import word_tokenize, pos_tag, ne_chunk
2input_str = “Bill works for Apple so he went to Boston for a conference.”
3print ne_chunk(pos_tag(word_tokenize(input_str)))

1(S (PERSON Bill/NNP) works/VBZ for/IN Apple/NNP so/IN he/PRP went/VBD to/TO (GPE Boston/NNP) for/IN a/DT conference/NN ./.)

1pip install spacy
2python -m spacy download en_core_web_sm

1import spacy 
2  
3nlp = spacy.load('en_core_web_sm') 
4  
5sentence = "Apple is looking at buying U.K. startup for $1 billion"
6  
7doc = nlp(sentence) 
8  
9for ent in doc.ents: 
10    print(ent.text, ent.start_char, ent.end_char, ent.label_) 

1Apple 0 5 ORG
2U.K. 27 31 GPE
3$1 billion 44 54 MONEY

1import nltk.tag 
2from nltk.chunk.util import conlltags2tree 
3from nltk.corpus import ieer 
4  
5def ieertree2conlltags(tree, tag = nltk.tag.pos_tag): 
6    words, ents = zip(*tree.pos()) 
7    iobs = [] 
8    prev = None
9    for ent in ents: 
10        if ent == tree.label(): 
11            iobs.append('O') 
12            prev = None
13        elif prev == ent: 
14            iobs.append('I-% s' % ent) 
15        else: 
16            iobs.append('B-% s' % ent) 
17            prev = ent 
18      
19    words, tags = zip(*tag(words)) 
20      
21    return zip(words, tags, iobs)

1import nltk.tag 
2from nltk.chunk.util import conlltags2tree 
3from nltk.corpus import ieer 
4  
5def ieer_chunked_sents(tag = nltk.tag.pos_tag): 
6    for doc in ieer.parsed_docs(): 
7        tagged = ieertree2conlltags(doc.text, tag) 
8        yield conlltags2tree(tagged) 

1from nltk.corpus import ieer 
2from chunkers import ieer_chunked_sents, ClassifierChunker 
3from nltk.corpus import treebank_chunk 
4  
5ieer_chunks = list(ieer_chunked_sents()) 
6  
7print ("Length of ieer_chunks : ", len(ieer_chunks)) 
8  
9# initializing chunker 
10chunker = ClassifierChunker(ieer_chunks[:80]) 
11print("\nparsing : \n", chunker.parse( 
12        treebank_chunk.tagged_sents()[0])) 
13  
14# evaluating 
15score = chunker.evaluate(ieer_chunks[80:]) 
16  
17a = score.accuracy() 
18p = score.precision() 
19r = score.recall() 
20  
21print ("\nAccuracy : ", a) 
22print ("\nPrecision : ", p) 
23print ("\nRecall : ", r)

1Length of ieer_chunks : 94
2
3parsing : 
4Tree('S', [Tree('LOCATION', [('Pierre', 'NNP'), ('Vinken', 'NNP')]),
5(', ', ', '), Tree('DURATION', [('61', 'CD'), ('years', 'NNS')]),
6Tree('MEASURE', [('old', 'JJ')]), (', ', ', '), ('will', 'MD'), ('join', 'VB'), 
7('the', 'DT'), ('board', 'NN'), ('as', 'IN'), ('a', 'DT'), ('nonexecutive', 'JJ'),
8('director', 'NN'), Tree('DATE', [('Nov.', 'NNP'), ('29', 'CD')]), ('.', '.')])
9
10Accuracy : 0.8829018388070625
11
12Precision : 0.4088717454194793
13
14Recall : 0.5053635280095352

1from xrenner import Xrenner
2
3xrenner = Xrenner()
4# Get a parse in basic Stanford Dependencies (not UD)
5my_conllx_result = some_parser.parse("Mrs. Hills said that the U.S. is still concerned about disturbing developments in Turkey and continuing slow progress in Malaysia . She did n't elaborate.")
6
7sgml_result = xrenner.analyze(my_conllx_result,"sgml")
8print(sgml_result)

1<referent id="referent_197" entity="person" group="34" antecedent="referent_142" type="coref">
2Mrs.
3Hills
4</referent>
5said
6that
7<referent id="referent_198" entity="place" group="2" antecedent="referent_157" type="coref">
8the
9U.S.
10</referent>
11is
12still
13concerned
14about
15``
16disturbing
17developments
18in
19<referent id="referent_201" entity="place" group="20" antecedent="referent_193" type="coref">
20Turkey
21</referent>
22and
23continuing
24slow
25progress
26in
27<referent id="referent_203" entity="place" group="20" antecedent="referent_201" type="coref">
28Malaysia
29</referent>
30.
31''
32<referent id="referent_204" entity="person" group="34" antecedent="referent_197" type="ana">
33She
34</referent>
35did
36n't
37elaborate

1input=[“he and Chazz duel with all keys on the line.”]
2from ICE import CollocationExtractor
3extractor = CollocationExtractor.with_collocation_pipeline(“T1” , bing_key = “Temp”,pos_check = False)
4print(extractor.get_collocations_of_length(input, length = 3))

1[“on the line”]