1 : /* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
2 :
3 : This program is free software; you can redistribute it and/or modify
4 : it under the terms of the GNU General Public License as published by
5 : the Free Software Foundation; version 2 of the License.
6 :
7 : This program is distributed in the hope that it will be useful,
8 : but WITHOUT ANY WARRANTY; without even the implied warranty of
9 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 : GNU General Public License for more details.
11 :
12 : You should have received a copy of the GNU General Public License
13 : along with this program; if not, write to the Free Software
14 : Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
15 :
16 : /* Written by Sergei A. Golubchik, who has a shared copyright to this code */
17 :
18 : #include "ma_ftdefs.h"
19 :
20 : typedef struct st_maria_ft_docstat {
21 : FT_WORD *list;
22 : uint uniq;
23 : double sum;
24 : } FT_DOCSTAT;
25 :
26 :
27 : typedef struct st_my_maria_ft_parser_param
28 : {
29 : TREE *wtree;
30 : MEM_ROOT *mem_root;
31 : } MY_FT_PARSER_PARAM;
32 :
33 :
34 : static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
35 0 : {
36 0 : return ha_compare_text(cs, (uchar*) w1->pos, w1->len,
37 : (uchar*) w2->pos, w2->len, 0, 0);
38 : }
39 :
40 : static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
41 0 : {
42 0 : word->weight=LWS_IN_USE;
43 0 : docstat->sum+=word->weight;
44 0 : memcpy_fixed((docstat->list)++,word,sizeof(FT_WORD));
45 0 : return 0;
46 : }
47 :
48 : /* transforms tree of words into the array, applying normalization */
49 :
50 : FT_WORD * maria_ft_linearize(TREE *wtree, MEM_ROOT *mem_root)
51 0 : {
52 : FT_WORD *wlist,*p;
53 : FT_DOCSTAT docstat;
54 0 : DBUG_ENTER("maria_ft_linearize");
55 :
56 0 : if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*
57 : (1+wtree->elements_in_tree))))
58 : {
59 0 : docstat.list=wlist;
60 0 : docstat.uniq=wtree->elements_in_tree;
61 0 : docstat.sum=0;
62 0 : tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right);
63 : }
64 0 : delete_tree(wtree);
65 0 : if (!wlist)
66 0 : DBUG_RETURN(NULL);
67 :
68 0 : docstat.list->pos=NULL;
69 :
70 0 : for (p=wlist;p->pos;p++)
71 : {
72 0 : p->weight=PRENORM_IN_USE;
73 : }
74 :
75 0 : for (p=wlist;p->pos;p++)
76 : {
77 0 : p->weight/=NORM_IN_USE;
78 : }
79 :
80 0 : DBUG_RETURN(wlist);
81 : }
82 :
83 : my_bool maria_ft_boolean_check_syntax_string(const uchar *str)
84 0 : {
85 : uint i, j;
86 :
87 0 : if (!str ||
88 : (strlen((const char *) str) + 1 != sizeof(ft_boolean_syntax)) ||
89 : (str[0] != ' ' && str[1] != ' '))
90 0 : return 1;
91 0 : for (i=0; i<sizeof(ft_boolean_syntax); i++)
92 : {
93 : /* limiting to 7-bit ascii only */
94 0 : if ((unsigned char)(str[i]) > 127 ||
95 : my_isalnum(default_charset_info, str[i]))
96 0 : return 1;
97 0 : for (j=0; j<i; j++)
98 0 : if (str[i] == str[j] && (i != 11 || j != 10))
99 0 : return 1;
100 : }
101 0 : return 0;
102 : }
103 :
104 : /*
105 : RETURN VALUE
106 : 0 - eof
107 : 1 - word found
108 : 2 - left bracket
109 : 3 - right bracket
110 : 4 - stopword found
111 : */
112 : uchar maria_ft_get_word(CHARSET_INFO *cs, uchar **start, uchar *end,
113 : FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
114 0 : {
115 0 : uchar *doc=*start;
116 : int ctype;
117 : uint mwc, length;
118 : int mbl;
119 :
120 0 : param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
121 0 : param->weight_adjust= param->wasign= 0;
122 0 : param->type= FT_TOKEN_EOF;
123 :
124 0 : while (doc<end)
125 : {
126 0 : for (; doc < end; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
127 : {
128 0 : mbl= cs->cset->ctype(cs, &ctype, doc, end);
129 0 : if (true_word_char(ctype, *doc))
130 : break;
131 0 : if (*doc == FTB_RQUOT && param->quot)
132 : {
133 0 : param->quot= (char *) doc;
134 0 : *start=doc+1;
135 0 : param->type= FT_TOKEN_RIGHT_PAREN;
136 0 : goto ret;
137 : }
138 0 : if (!param->quot)
139 : {
140 0 : if (*doc == FTB_LBR || *doc == FTB_RBR || *doc == FTB_LQUOT)
141 : {
142 : /* param->prev=' '; */
143 0 : *start=doc+1;
144 0 : if (*doc == FTB_LQUOT)
145 0 : param->quot= (char *) *start;
146 0 : param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN);
147 0 : goto ret;
148 : }
149 0 : if (param->prev == ' ')
150 : {
151 0 : if (*doc == FTB_YES ) { param->yesno=+1; continue; } else
152 0 : if (*doc == FTB_EGAL) { param->yesno= 0; continue; } else
153 0 : if (*doc == FTB_NO ) { param->yesno=-1; continue; } else
154 0 : if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else
155 0 : if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else
156 0 : if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; }
157 : }
158 : }
159 0 : param->prev=*doc;
160 0 : param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
161 0 : param->weight_adjust= param->wasign= 0;
162 : }
163 :
164 0 : mwc=length=0;
165 0 : for (word->pos= doc; doc < end; length++,
166 0 : doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
167 : {
168 0 : mbl= cs->cset->ctype(cs, &ctype, doc, end);
169 0 : if (true_word_char(ctype, *doc))
170 0 : mwc=0;
171 : else if (!misc_word_char(*doc) || mwc)
172 : break;
173 : else
174 : mwc++;
175 : }
176 0 : param->prev='A'; /* be sure *prev is true_word_char */
177 0 : word->len= (uint)(doc-word->pos) - mwc;
178 0 : if ((param->trunc=(doc<end && *doc == FTB_TRUNC)))
179 0 : doc++;
180 :
181 0 : if (((length >= ft_min_word_len && !is_stopword((char *) word->pos,
182 : word->len))
183 : || param->trunc) && length < ft_max_word_len)
184 : {
185 0 : *start=doc;
186 0 : param->type= FT_TOKEN_WORD;
187 0 : goto ret;
188 : }
189 0 : else if (length) /* make sure length > 0 (if start contains spaces only) */
190 : {
191 0 : *start= doc;
192 0 : param->type= FT_TOKEN_STOPWORD;
193 0 : goto ret;
194 : }
195 : }
196 0 : if (param->quot)
197 : {
198 0 : param->quot= (char *)(*start= doc);
199 0 : param->type= 3; /* FT_RBR */
200 : goto ret;
201 : }
202 0 : ret:
203 0 : return param->type;
204 : }
205 :
206 : uchar maria_ft_simple_get_word(CHARSET_INFO *cs, uchar **start,
207 : const uchar *end, FT_WORD *word,
208 : my_bool skip_stopwords)
209 0 : {
210 0 : uchar *doc= *start;
211 : uint mwc, length;
212 : int ctype, mbl;
213 0 : DBUG_ENTER("maria_ft_simple_get_word");
214 :
215 : do
216 : {
217 0 : for (;; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
218 : {
219 0 : if (doc >= end)
220 0 : DBUG_RETURN(0);
221 0 : mbl= cs->cset->ctype(cs, &ctype, doc, end);
222 0 : if (true_word_char(ctype, *doc))
223 : break;
224 0 : }
225 :
226 0 : mwc= length= 0;
227 0 : for (word->pos= doc; doc < end; length++,
228 0 : doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
229 : {
230 0 : mbl= cs->cset->ctype(cs, &ctype, doc, end);
231 0 : if (true_word_char(ctype, *doc))
232 0 : mwc= 0;
233 : else if (!misc_word_char(*doc) || mwc)
234 : break;
235 : else
236 : mwc++;
237 : }
238 :
239 0 : word->len= (uint)(doc-word->pos) - mwc;
240 :
241 0 : if (skip_stopwords == FALSE ||
242 : (length >= ft_min_word_len && length < ft_max_word_len &&
243 : !is_stopword((char *) word->pos, word->len)))
244 : {
245 0 : *start= doc;
246 0 : DBUG_RETURN(1);
247 : }
248 0 : } while (doc < end);
249 0 : DBUG_RETURN(0);
250 : }
251 :
252 : void maria_ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
253 0 : {
254 0 : DBUG_ENTER("maria_ft_parse_init");
255 0 : if (!is_tree_inited(wtree))
256 0 : init_tree(wtree,0,0,sizeof(FT_WORD),(qsort_cmp2)&FT_WORD_cmp,0,NULL, cs);
257 0 : DBUG_VOID_RETURN;
258 : }
259 :
260 :
261 : static int maria_ft_add_word(MYSQL_FTPARSER_PARAM *param,
262 : char *word, int word_len,
263 : MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
264 0 : {
265 : TREE *wtree;
266 : FT_WORD w;
267 0 : MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
268 0 : DBUG_ENTER("maria_ft_add_word");
269 0 : wtree= ft_param->wtree;
270 0 : if (param->flags & MYSQL_FTFLAGS_NEED_COPY)
271 : {
272 : uchar *ptr;
273 0 : DBUG_ASSERT(wtree->with_delete == 0);
274 0 : ptr= (uchar *)alloc_root(ft_param->mem_root, word_len);
275 0 : memcpy(ptr, word, word_len);
276 0 : w.pos= ptr;
277 : }
278 : else
279 0 : w.pos= (uchar *) word;
280 0 : w.len= word_len;
281 0 : if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
282 : {
283 0 : delete_tree(wtree);
284 0 : DBUG_RETURN(1);
285 : }
286 0 : DBUG_RETURN(0);
287 : }
288 :
289 :
290 : static int maria_ft_parse_internal(MYSQL_FTPARSER_PARAM *param,
291 : char *doc_arg, int doc_len)
292 0 : {
293 0 : uchar *doc= (uchar*) doc_arg;
294 0 : uchar *end= doc + doc_len;
295 0 : MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
296 0 : TREE *wtree= ft_param->wtree;
297 : FT_WORD w;
298 0 : DBUG_ENTER("maria_ft_parse_internal");
299 :
300 0 : while (maria_ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
301 0 : if (param->mysql_add_word(param, (char *) w.pos, w.len, 0))
302 0 : DBUG_RETURN(1);
303 0 : DBUG_RETURN(0);
304 : }
305 :
306 :
307 : int maria_ft_parse(TREE *wtree, uchar *doc, int doclen,
308 : struct st_mysql_ftparser *parser,
309 : MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
310 0 : {
311 : MY_FT_PARSER_PARAM my_param;
312 0 : DBUG_ENTER("maria_ft_parse");
313 0 : DBUG_ASSERT(parser);
314 0 : my_param.wtree= wtree;
315 0 : my_param.mem_root= mem_root;
316 :
317 0 : param->mysql_parse= maria_ft_parse_internal;
318 0 : param->mysql_add_word= maria_ft_add_word;
319 0 : param->mysql_ftparam= &my_param;
320 0 : param->cs= wtree->custom_arg;
321 0 : param->doc= (char *) doc;
322 0 : param->length= doclen;
323 0 : param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
324 0 : DBUG_RETURN(parser->parse(param));
325 : }
326 :
327 :
328 : #define MAX_PARAM_NR 2
329 :
330 : MYSQL_FTPARSER_PARAM* maria_ftparser_alloc_param(MARIA_HA *info)
331 10 : {
332 10 : if (!info->ftparser_param)
333 : {
334 : /*
335 : . info->ftparser_param can not be zero after the initialization,
336 : because it always includes built-in fulltext parser. And built-in
337 : parser can be called even if the table has no fulltext indexes and
338 : no varchar/text fields.
339 :
340 : ftb_find_relevance... parser (ftb_find_relevance_parse,
341 : ftb_find_relevance_add_word) calls ftb_check_phrase... parser
342 : (ftb_check_phrase_internal, ftb_phrase_add_word). Thus MAX_PARAM_NR=2.
343 : */
344 10 : info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
345 : my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *
346 : info->s->ftkeys, MYF(MY_WME | MY_ZEROFILL));
347 10 : init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
348 : }
349 10 : return info->ftparser_param;
350 : }
351 :
352 :
353 : MYSQL_FTPARSER_PARAM *maria_ftparser_call_initializer(MARIA_HA *info,
354 : uint keynr, uint paramnr)
355 0 : {
356 : uint32 ftparser_nr;
357 : struct st_mysql_ftparser *parser;
358 :
359 0 : if (!maria_ftparser_alloc_param(info))
360 0 : return 0;
361 :
362 0 : if (keynr == NO_SUCH_KEY)
363 : {
364 0 : ftparser_nr= 0;
365 0 : parser= &ft_default_parser;
366 : }
367 : else
368 : {
369 0 : ftparser_nr= info->s->keyinfo[keynr].ftkey_nr;
370 0 : parser= info->s->keyinfo[keynr].parser;
371 : }
372 0 : DBUG_ASSERT(paramnr < MAX_PARAM_NR);
373 0 : ftparser_nr= ftparser_nr*MAX_PARAM_NR + paramnr;
374 0 : if (! info->ftparser_param[ftparser_nr].mysql_add_word)
375 : {
376 : /* Note, that mysql_add_word is used here as a flag:
377 : mysql_add_word == 0 - parser is not initialized
378 : mysql_add_word != 0 - parser is initialized, or no
379 : initialization needed. */
380 0 : info->ftparser_param[ftparser_nr].mysql_add_word=
381 : (int (*)(struct st_mysql_ftparser_param *, char *, int,
382 : MYSQL_FTPARSER_BOOLEAN_INFO *)) 1;
383 0 : if (parser->init && parser->init(&info->ftparser_param[ftparser_nr]))
384 0 : return 0;
385 : }
386 0 : return &info->ftparser_param[ftparser_nr];
387 : }
388 :
389 :
390 : void maria_ftparser_call_deinitializer(MARIA_HA *info)
391 1244 : {
392 1244 : uint i, j, keys= info->s->state.header.keys;
393 1244 : free_root(&info->ft_memroot, MYF(0));
394 1244 : if (! info->ftparser_param)
395 10 : return;
396 70 : for (i= 0; i < keys; i++)
397 : {
398 60 : MARIA_KEYDEF *keyinfo= &info->s->keyinfo[i];
399 60 : for (j=0; j < MAX_PARAM_NR; j++)
400 : {
401 : MYSQL_FTPARSER_PARAM *ftparser_param=
402 60 : &info->ftparser_param[keyinfo->ftkey_nr*MAX_PARAM_NR + j];
403 60 : if (keyinfo->flag & HA_FULLTEXT && ftparser_param->mysql_add_word)
404 : {
405 0 : if (keyinfo->parser->deinit)
406 0 : keyinfo->parser->deinit(ftparser_param);
407 0 : ftparser_param->mysql_add_word= 0;
408 : }
409 : else
410 : break;
411 : }
412 : }
413 : }
|