Tesseract  3.02
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
char_bigrams.cpp
Go to the documentation of this file.
1 /**********************************************************************
2  * File: char_bigrams.cpp
3  * Description: Implementation of a Character Bigrams Class
4  * Author: Ahmad Abdulkader
5  * Created: 2007
6  *
7  * (C) Copyright 2008, Google Inc.
8  ** Licensed under the Apache License, Version 2.0 (the "License");
9  ** you may not use this file except in compliance with the License.
10  ** You may obtain a copy of the License at
11  ** http://www.apache.org/licenses/LICENSE-2.0
12  ** Unless required by applicable law or agreed to in writing, software
13  ** distributed under the License is distributed on an "AS IS" BASIS,
14  ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  ** See the License for the specific language governing permissions and
16  ** limitations under the License.
17  *
18  **********************************************************************/
19 
20 #include <algorithm>
21 #include <math.h>
22 #include <string>
23 #include <vector>
24 
25 #include "char_bigrams.h"
26 #include "cube_utils.h"
27 #include "ndminx.h"
28 #include "cube_const.h"
29 
30 namespace tesseract {
31 
33  memset(&bigram_table_, 0, sizeof(bigram_table_));
34 }
35 
37  if (bigram_table_.char_bigram != NULL) {
38  for (int ch1 = 0; ch1 <= bigram_table_.max_char; ch1++) {
39  CharBigram *char_bigram = bigram_table_.char_bigram + ch1;
40 
41  if (char_bigram->bigram != NULL) {
42  delete []char_bigram->bigram;
43  }
44  }
45  delete []bigram_table_.char_bigram;
46  }
47 }
48 
49 CharBigrams *CharBigrams::Create(const string &data_file_path,
50  const string &lang) {
51  string file_name;
52  string str;
53 
54  file_name = data_file_path + lang;
55  file_name += ".cube.bigrams";
56 
57  // load the string into memory
58  if (!CubeUtils::ReadFileToString(file_name, &str)) {
59  return NULL;
60  }
61 
62  // construct a new object
63  CharBigrams *char_bigrams_obj = new CharBigrams();
64  if (char_bigrams_obj == NULL) {
65  fprintf(stderr, "Cube ERROR (CharBigrams::Create): could not create "
66  "character bigrams object.\n");
67  return NULL;
68  }
69  CharBigramTable *table = &char_bigrams_obj->bigram_table_;
70 
71  table->total_cnt = 0;
72  table->max_char = -1;
73  table->char_bigram = NULL;
74 
75  // split into lines
76  vector<string> str_vec;
77  CubeUtils::SplitStringUsing(str, "\r\n", &str_vec);
78 
79  for (int big = 0; big < str_vec.size(); big++) {
80  char_32 ch1;
81  char_32 ch2;
82  int cnt;
83  if (sscanf(str_vec[big].c_str(), "%d %x %x", &cnt, &ch1, &ch2) != 3) {
84  fprintf(stderr, "Cube ERROR (CharBigrams::Create): invalid format "
85  "reading line: %s\n", str_vec[big].c_str());
86  return NULL;
87  }
88 
89  // expand the bigram table
90  if (ch1 > table->max_char) {
91  CharBigram *char_bigram = new CharBigram[ch1 + 1];
92  if (char_bigram == NULL) {
93  fprintf(stderr, "Cube ERROR (CharBigrams::Create): error allocating "
94  "additional memory for character bigram table.\n");
95  return NULL;
96  }
97 
98  if (table->char_bigram != NULL && table->max_char >= 0) {
99  memcpy(char_bigram, table->char_bigram,
100  (table->max_char + 1) * sizeof(*char_bigram));
101 
102  delete []table->char_bigram;
103  }
104  table->char_bigram = char_bigram;
105 
106  // init
107  for (int new_big = table->max_char + 1; new_big <= ch1; new_big++) {
108  table->char_bigram[new_big].total_cnt = 0;
109  table->char_bigram[new_big].max_char = -1;
110  table->char_bigram[new_big].bigram = NULL;
111  }
112  table->max_char = ch1;
113  }
114 
115  if (ch2 > table->char_bigram[ch1].max_char) {
116  Bigram *bigram = new Bigram[ch2 + 1];
117  if (bigram == NULL) {
118  fprintf(stderr, "Cube ERROR (CharBigrams::Create): error allocating "
119  "memory for bigram.\n");
120  return NULL;
121  }
122 
123  if (table->char_bigram[ch1].bigram != NULL &&
124  table->char_bigram[ch1].max_char >= 0) {
125  memcpy(bigram, table->char_bigram[ch1].bigram,
126  (table->char_bigram[ch1].max_char + 1) * sizeof(*bigram));
127  delete []table->char_bigram[ch1].bigram;
128  }
129  table->char_bigram[ch1].bigram = bigram;
130 
131  // init
132  for (int new_big = table->char_bigram[ch1].max_char + 1;
133  new_big <= ch2; new_big++) {
134  table->char_bigram[ch1].bigram[new_big].cnt = 0;
135  }
136  table->char_bigram[ch1].max_char = ch2;
137  }
138 
139  table->char_bigram[ch1].bigram[ch2].cnt = cnt;
140  table->char_bigram[ch1].total_cnt += cnt;
141  table->total_cnt += cnt;
142  }
143 
144  // compute costs (-log probs)
145  table->worst_cost = static_cast<int>(
146  -PROB2COST_SCALE * log(0.5 / table->total_cnt));
147  for (char_32 ch1 = 0; ch1 <= table->max_char; ch1++) {
148  for (char_32 ch2 = 0; ch2 <= table->char_bigram[ch1].max_char; ch2++) {
149  int cnt = table->char_bigram[ch1].bigram[ch2].cnt;
150  table->char_bigram[ch1].bigram[ch2].cost =
151  static_cast<int>(-PROB2COST_SCALE *
152  log(MAX(0.5, static_cast<double>(cnt)) /
153  table->total_cnt));
154  }
155  }
156  return char_bigrams_obj;
157 }
158 
159 int CharBigrams::PairCost(char_32 ch1, char_32 ch2) const {
160  if (ch1 > bigram_table_.max_char) {
161  return bigram_table_.worst_cost;
162  }
163  if (ch2 > bigram_table_.char_bigram[ch1].max_char) {
164  return bigram_table_.worst_cost;
165  }
166  return bigram_table_.char_bigram[ch1].bigram[ch2].cost;
167 }
168 
169 int CharBigrams::Cost(const char_32 *char_32_ptr, CharSet *char_set) const {
170  if (!char_32_ptr || char_32_ptr[0] == 0) {
171  return bigram_table_.worst_cost;
172  }
173  int cost = MeanCostWithSpaces(char_32_ptr);
174  if (CubeUtils::StrLen(char_32_ptr) >= kMinLengthCaseInvariant &&
175  CubeUtils::IsCaseInvariant(char_32_ptr, char_set)) {
176  char_32 *lower_32 = CubeUtils::ToLower(char_32_ptr, char_set);
177  if (lower_32 && lower_32[0] != 0) {
178  int cost_lower = MeanCostWithSpaces(lower_32);
179  cost = MIN(cost, cost_lower);
180  delete [] lower_32;
181  }
182  char_32 *upper_32 = CubeUtils::ToUpper(char_32_ptr, char_set);
183  if (upper_32 && upper_32[0] != 0) {
184  int cost_upper = MeanCostWithSpaces(upper_32);
185  cost = MIN(cost, cost_upper);
186  delete [] upper_32;
187  }
188  }
189  return cost;
190 }
191 
192 int CharBigrams::MeanCostWithSpaces(const char_32 *char_32_ptr) const {
193  if (!char_32_ptr)
194  return bigram_table_.worst_cost;
195  int len = CubeUtils::StrLen(char_32_ptr);
196  int cost = 0;
197  int c = 0;
198  cost = PairCost(' ', char_32_ptr[0]);
199  for (c = 1; c < len; c++) {
200  cost += PairCost(char_32_ptr[c - 1], char_32_ptr[c]);
201  }
202  cost += PairCost(char_32_ptr[len - 1], ' ');
203  return static_cast<int>(cost / static_cast<double>(len + 1));
204 }
205 } // namespace tesseract