محاسبه آمار کلمات یکتا در زبان های برنامه نویسی – راهنمای کاربردی

1# Python code to find frequency of each word
2def freq(str):
3    # break the string into list of words
4    str = str.split()
5    str2 = []
6
7    # loop till string values present in list str
8    for i in str:
9
10        # checking for the duplicacy
11        if i not in str2:
12            # insert value in str2
13            str2.append(i)
14
15    for i in range(0, len(str2)):
16        # count the frequency of each word(present
17        # in str2) in str and print
18        print('Frequency of (', str2[i], ') is :', str.count(str2[i]))
19
20
21def main():
22    wordstring = 'it was the best of times it was the worst of times '
23    wordstring += 'it was the age of wisdom it was the age of foolishness'
24    freq(wordstring)
25
26
27if __name__ == "__main__":
28    main()  # call main function

1# Python3 code to find frequency of each word
2# function for calculating the frequency
3def freq(str):
4    # break the string into list of words
5    str_list = str.split()
6
7    # gives set of unique words
8    unique_words = set(str_list)
9
10    for words in unique_words:
11        print('Frequency of (', words, ') is :', str_list.count(words))
12
13    # driver code
14
15
16if __name__ == "__main__":
17    wordstring = 'it was the best of times it was the worst of times '
18    wordstring += 'it was the age of wisdom it was the age of foolishness'
19
20    # calling the freq function
21    freq(wordstring)

1s = 'hello world'
2print(s[0])
3-> h
4
5print(s[1])
6-> e
7
8m = ['hello', 'world']
9print(m[0])
10-> hello
11
12print(m[1])
13-> world
14
15print(m[0][1])
16-> e
17
18print(m[1][0])
19-> w

1d = {'world': 1, 'hello': 0}
2print(d['hello'])
3-> 0
4
5print(d['world'])
6-> 1
7
8print(d.keys())
9-> ['world', 'hello']

1# Dictionary_Unique_Word_Freq.py
2
3def stripTags(pageContents):
4    startLoc = pageContents.find("<p>")
5    endLoc = pageContents.rfind("<br/>")
6
7    pageContents = pageContents[startLoc:endLoc]
8
9    inside = 0
10    text = ''
11
12    for char in pageContents:
13        if char == '<':
14            inside = 1
15        elif (inside == 1 and char == '>'):
16            inside = 0
17        elif inside == 1:
18            continue
19        else:
20            text += char
21
22    return text
23
24
25# Given a text string, remove all non-alphanumeric
26# characters (using Unicode definition of alphanumeric).
27
28def stripNonAlphaNum(text):
29    import re
30    return re.compile(r'\W+', re.UNICODE).split(text)
31
32# Given a list of words, return a dictionary of
33# word-frequency pairs.
34def sortFreqDict(freqdict):
35    aux = [(freqdict[key], key) for key in freqdict]
36    aux.sort()
37    aux.reverse()
38    return aux
39
40def wordListToFreqDict(wordlist):
41    wordfreq = [wordlist.count(p) for p in wordlist]
42    return dict(zip(wordlist,wordfreq))
43

1import urllib.request
2import Dictionary_Unique_Word_Freq
3
4wordstring = 'it was the best of times it was the worst of times '
5wordstring += 'it was the age of wisdom it was the age of foolishness'
6
7wordlist = Dictionary_Unique_Word_Freq.stripNonAlphaNum(wordstring)
8dictionary = Dictionary_Unique_Word_Freq.wordListToFreqDict(wordlist)
9sorteddict = Dictionary_Unique_Word_Freq.sortFreqDict(dictionary)
10
11for s in sorteddict: print(str(s))

1# Dictionary_Unique_Word_Freq.py
2stopwords = ['a', 'about', 'above', 'across', 'after', 'afterwards']
3stopwords += ['again', 'against', 'all', 'almost', 'alone', 'along']
4stopwords += ['already', 'also', 'although', 'always', 'am', 'among']
5stopwords += ['amongst', 'amoungst', 'amount', 'an', 'and', 'another']
6stopwords += ['any', 'anyhow', 'anyone', 'anything', 'anyway', 'anywhere']
7stopwords += ['are', 'around', 'as', 'at', 'back', 'be', 'became']
8stopwords += ['because', 'become', 'becomes', 'becoming', 'been']
9stopwords += ['before', 'beforehand', 'behind', 'being', 'below']
10stopwords += ['beside', 'besides', 'between', 'beyond', 'bill', 'both']
11stopwords += ['bottom', 'but', 'by', 'call', 'can', 'cannot', 'cant']
12stopwords += ['co', 'computer', 'con', 'could', 'couldnt', 'cry', 'de']
13stopwords += ['describe', 'detail', 'did', 'do', 'done', 'down', 'due']
14stopwords += ['during', 'each', 'eg', 'eight', 'either', 'eleven', 'else']
15stopwords += ['elsewhere', 'empty', 'enough', 'etc', 'even', 'ever']
16stopwords += ['every', 'everyone', 'everything', 'everywhere', 'except']
17stopwords += ['few', 'fifteen', 'fifty', 'fill', 'find', 'fire', 'first']
18stopwords += ['five', 'for', 'former', 'formerly', 'forty', 'found']
19stopwords += ['four', 'from', 'front', 'full', 'further', 'get', 'give']
20stopwords += ['go', 'had', 'has', 'hasnt', 'have', 'he', 'hence', 'her']
21stopwords += ['here', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers']
22stopwords += ['herself', 'him', 'himself', 'his', 'how', 'however']
23stopwords += ['hundred', 'i', 'ie', 'if', 'in', 'inc', 'indeed']
24stopwords += ['interest', 'into', 'is', 'it', 'its', 'itself', 'keep']
25stopwords += ['last', 'latter', 'latterly', 'least', 'less', 'ltd', 'made']
26stopwords += ['many', 'may', 'me', 'meanwhile', 'might', 'mill', 'mine']
27stopwords += ['more', 'moreover', 'most', 'mostly', 'move', 'much']
28stopwords += ['must', 'my', 'myself', 'name', 'namely', 'neither', 'never']
29stopwords += ['nevertheless', 'next', 'nine', 'no', 'nobody', 'none']
30stopwords += ['noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of']
31stopwords += ['off', 'often', 'on','once', 'one', 'only', 'onto', 'or']
32stopwords += ['other', 'others', 'otherwise', 'our', 'ours', 'ourselves']
33stopwords += ['out', 'over', 'own', 'part', 'per', 'perhaps', 'please']
34stopwords += ['put', 'rather', 're', 's', 'same', 'see', 'seem', 'seemed']
35stopwords += ['seeming', 'seems', 'serious', 'several', 'she', 'should']
36stopwords += ['show', 'side', 'since', 'sincere', 'six', 'sixty', 'so']
37stopwords += ['some', 'somehow', 'someone', 'something', 'sometime']
38stopwords += ['sometimes', 'somewhere', 'still', 'such', 'system', 'take']
39stopwords += ['ten', 'than', 'that', 'the', 'their', 'them', 'themselves']
40stopwords += ['then', 'thence', 'there', 'thereafter', 'thereby']
41stopwords += ['therefore', 'therein', 'thereupon', 'these', 'they']
42stopwords += ['thick', 'thin', 'third', 'this', 'those', 'though', 'three']
43stopwords += ['three', 'through', 'throughout', 'thru', 'thus', 'to']
44stopwords += ['together', 'too', 'top', 'toward', 'towards', 'twelve']
45stopwords += ['twenty', 'two', 'un', 'under', 'until', 'up', 'upon']
46stopwords += ['us', 'very', 'via', 'was', 'we', 'well', 'were', 'what']
47stopwords += ['whatever', 'when', 'whence', 'whenever', 'where']
48stopwords += ['whereafter', 'whereas', 'whereby', 'wherein', 'whereupon']
49stopwords += ['wherever', 'whether', 'which', 'while', 'whither', 'who']
50stopwords += ['whoever', 'whole', 'whom', 'whose', 'why', 'will', 'with']
51stopwords += ['within', 'without', 'would', 'yet', 'you', 'your']
52stopwords += ['yours', 'yourself', 'yourselves']
53
54def stripTags(pageContents):
55    startLoc = pageContents.find("<p>")
56    endLoc = pageContents.rfind("<br/>")
57
58    pageContents = pageContents[startLoc:endLoc]
59
60    inside = 0
61    text = ''
62
63    for char in pageContents:
64        if char == '<':
65            inside = 1
66        elif (inside == 1 and char == '>'):
67            inside = 0
68        elif inside == 1:
69            continue
70        else:
71            text += char
72
73    return text
74
75
76# Given a text string, remove all non-alphanumeric
77# characters (using Unicode definition of alphanumeric).
78
79def stripNonAlphaNum(text):
80    import re
81    return re.compile(r'\W+', re.UNICODE).split(text)
82
83# Given a list of words, return a dictionary of
84# word-frequency pairs.
85def sortFreqDict(freqdict):
86    aux = [(freqdict[key], key) for key in freqdict]
87    aux.sort()
88    aux.reverse()
89    return aux
90
91def wordListToFreqDict(wordlist):
92    wordfreq = [wordlist.count(p) for p in wordlist]
93    return dict(zip(wordlist,wordfreq))
94
95# Given a list of words, remove any that are
96# in a list of stop words.
97
98def removeStopwords(wordlist, stopwords):
99    return [w for w in wordlist if w not in stopwords]
100

1import urllib.request
2import Dictionary_Unique_Word_Freq
3
4wordstring = 'it was the best of times it was the worst of times '
5wordstring += 'it was the age of wisdom it was the age of foolishness'
6
7wordlist = Dictionary_Unique_Word_Freq.stripNonAlphaNum(wordstring)
8wordlist = Dictionary_Unique_Word_Freq.removeStopwords(wordlist, Dictionary_Unique_Word_Freq.stopwords)
9dictionary = Dictionary_Unique_Word_Freq.wordListToFreqDict(wordlist)
10sorteddict = Dictionary_Unique_Word_Freq.sortFreqDict(dictionary)
11
12for s in sorteddict: print(str(s))

1using System;
2using System.Text.RegularExpressions;
3using System.Net;
4using System.Collections.Generic;
5using System.Linq;
6
7
8public class Program
9{
10	//Removes Html Tags from input provided
11	public static string RemoveHtmlTags(string html)
12    {
13		string htmlRemoved = Regex.Replace(html, @"<script[^>]*>[\s\S]*?</script>|<[^>]+>| ", " ").Trim();
14		string normalised = Regex.Replace(htmlRemoved, @"\s{2,}", " ");
15		return normalised;
16    }
17	
18	public static void Main()
19	{
20		using (WebClient client = new WebClient())
21		{ 
22			//get the page source
23			//string html = client.DownloadString("http://stackoverflow.com/").ToLower();
24			//define string
25			string strdata= "it was the best of times it was the worst of times it was the age of wisdom it was the age of foolishness";
26			
27			//remove html elements
28			//html = RemoveHtmlTags(html);
29			
30			//split list into keywords by space characters
31			//List<string> list = html.Split(' ').ToList();
32			List<string> list = strdata.Split(' ').ToList();
33			
34			//remove any non alphabet characters
35			var onlyAlphabetRegEx = new Regex(@"^[A-z]+$");
36			list = list.Where(f => onlyAlphabetRegEx.IsMatch(f)).ToList();
37
38			//further blacklist words (greater than 2 characters, not important, etc..)
39            //string[] blacklist = { "a", "an", "on", "of", "or", "as", "i", "in", "is", "to", "the", "and", "for", "with", "not", "by" }; //add your own
40            //list = list.Where(x => x.Length > 2).Where(x => !blacklist.Contains(x)).ToList();
41			
42			//distict keywords by key and count, and then order by count.
43			var keywords = list.GroupBy(x => x).OrderByDescending(x => x.Count());
44
45			//print each keyword to console.
46			foreach (var word in keywords)
47			{
48				Console.WriteLine("{0} {1}", word.Key, word.Count());
49			}
50		} 
51	}
52}

1import java.util.Map; 
2import java.util.HashMap; 
3
4
5class Main {
6  public static void main(String[] args) {
7    String word= "it was the best of times it was the worst of times it was the age of wisdom it was the age of foolishness";
8    String words[] =word.split(" ");
9    Map<String, Integer> map = new HashMap<>();
10    for (String w : words) {
11        Integer n = map.get(w);
12        n = (n == null) ? 1 : ++n;
13        map.put(w, n);
14    }
15for (Map.Entry<String,Integer> entry : map.entrySet())  
16            System.out.println("Word = " + entry.getKey () + ", Frequency = "+entry.getValue()); 
17    } 
18  }