ContentCoding/AnalyzeText.cs at main · BUTTER-Tools/ContentCoding · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
using PluginContracts;
using System;
using System.Linq;
using System.Text;
using System.Collections.Generic;
using System.Text.RegularExpressions;

namespace ContentCoding
{


    internal class TextAnalyzer
    {


        public Dictionary<string, int> AnalyzeText(DictionaryData DictData, string[] Words)
        {

            int NumberOfMatches = 0;
            int TotalStringLength = Words.Length;

            Dictionary<string, int> DictionaryResults = new Dictionary<string, int>();
            for (int i = 0; i < DictData.NumCats; i++) DictionaryResults.Add(DictData.CatValues[i], 0);

            for (int i = 0; i < TotalStringLength; i++)
            {


                //iterate over n-grams, starting with the largest possible n-gram (derived from the user's dictionary file)
                for (int NumberOfWords = DictData.MaxWords; NumberOfWords > 0; NumberOfWords--)
                {


                    //make sure that we don't overextend past the array
                    if (i + NumberOfWords - 1 >= TotalStringLength) continue;

                    //make the target string

                    string TargetString;

                    if (NumberOfWords > 1)
                    {
                        TargetString = String.Join(" ", Words.Skip(i).Take(NumberOfWords).ToArray());
                    }
                    else
                    {
                        TargetString = Words[i];
                    }


                    //look for an exact match

                    if (DictData.FullDictionary["Standards"].ContainsKey(NumberOfWords))
                    {
                        if (DictData.FullDictionary["Standards"][NumberOfWords].ContainsKey(TargetString))
                        {

                            NumberOfMatches += NumberOfWords;
                            //add in the number of words found
                            for (int j = 0; j < DictData.FullDictionary["Standards"][NumberOfWords][TargetString].Length; j++)
                            {

                                if (DictionaryResults.ContainsKey(DictData.FullDictionary["Standards"][NumberOfWords][TargetString][j])) DictionaryResults[DictData.FullDictionary["Standards"][NumberOfWords][TargetString][j]] += NumberOfWords;

                            }
                            //manually increment the for loop so that we're not testing on words that have already been picked up
                            i += NumberOfWords - 1;
                            //break out of the lower level for loop back to moving on to new words altogether
                            break;
                        }
                    }
                    //if there isn't an exact match, we have to go through the wildcards
                    if (DictData.WildCardArrays.ContainsKey(NumberOfWords))
                    {
                        for (int j = 0; j < DictData.WildCardArrays[NumberOfWords].Length; j++)
                        {
                            if (DictData.PrecompiledWildcards[DictData.WildCardArrays[NumberOfWords][j]].Matches(TargetString).Count > 0)
                            {

                                NumberOfMatches += NumberOfWords;

                                for (int k = 0; k < DictData.FullDictionary["Wildcards"][NumberOfWords][DictData.WildCardArrays[NumberOfWords][j]].Length; k++)
                                {

                                    if (DictionaryResults.ContainsKey(DictData.FullDictionary["Wildcards"][NumberOfWords][DictData.WildCardArrays[NumberOfWords][j]][k])) DictionaryResults[DictData.FullDictionary["Wildcards"][NumberOfWords][DictData.WildCardArrays[NumberOfWords][j]][k]] += NumberOfWords;

                                }
                                //manually increment the for loop so that we're not testing on words that have already been picked up
                                i += NumberOfWords - 1;
                                //break out of the lower level for loop back to moving on to new words altogether
                                break;

                            }
                        }
                    }


                }


            }


            return DictionaryResults;


        }


    }


}