LTP GCOV extension - code coverage report
Current view: directory - storage/maria - ma_ft_parser.c
Test: maria-mtr.html
Date: 2009-03-04 Instrumented lines: 184
Code covered: 2.2 % Executed lines: 4

       1                 : /* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
       2                 : 
       3                 :    This program is free software; you can redistribute it and/or modify
       4                 :    it under the terms of the GNU General Public License as published by
       5                 :    the Free Software Foundation; version 2 of the License.
       6                 : 
       7                 :    This program is distributed in the hope that it will be useful,
       8                 :    but WITHOUT ANY WARRANTY; without even the implied warranty of
       9                 :    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      10                 :    GNU General Public License for more details.
      11                 : 
      12                 :    You should have received a copy of the GNU General Public License
      13                 :    along with this program; if not, write to the Free Software
      14                 :    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
      15                 : 
      16                 : /* Written by Sergei A. Golubchik, who has a shared copyright to this code */
      17                 : 
      18                 : #include "ma_ftdefs.h"
      19                 : 
      20                 : typedef struct st_maria_ft_docstat {
      21                 :   FT_WORD *list;
      22                 :   uint uniq;
      23                 :   double sum;
      24                 : } FT_DOCSTAT;
      25                 : 
      26                 : 
      27                 : typedef struct st_my_maria_ft_parser_param
      28                 : {
      29                 :   TREE *wtree;
      30                 :   MEM_ROOT *mem_root;
      31                 : } MY_FT_PARSER_PARAM;
      32                 : 
      33                 : 
      34                 : static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
      35               0 : {
      36               0 :   return ha_compare_text(cs, (uchar*) w1->pos, w1->len,
      37                 :                          (uchar*) w2->pos, w2->len, 0, 0);
      38                 : }
      39                 : 
      40                 : static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
      41               0 : {
      42               0 :     word->weight=LWS_IN_USE;
      43               0 :     docstat->sum+=word->weight;
      44               0 :     memcpy_fixed((docstat->list)++,word,sizeof(FT_WORD));
      45               0 :     return 0;
      46                 : }
      47                 : 
      48                 : /* transforms tree of words into the array, applying normalization */
      49                 : 
      50                 : FT_WORD * maria_ft_linearize(TREE *wtree, MEM_ROOT *mem_root)
      51               0 : {
      52                 :   FT_WORD *wlist,*p;
      53                 :   FT_DOCSTAT docstat;
      54               0 :   DBUG_ENTER("maria_ft_linearize");
      55                 : 
      56               0 :   if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*
      57                 :                                     (1+wtree->elements_in_tree))))
      58                 :   {
      59               0 :     docstat.list=wlist;
      60               0 :     docstat.uniq=wtree->elements_in_tree;
      61               0 :     docstat.sum=0;
      62               0 :     tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right);
      63                 :   }
      64               0 :   delete_tree(wtree);
      65               0 :   if (!wlist)
      66               0 :     DBUG_RETURN(NULL);
      67                 : 
      68               0 :   docstat.list->pos=NULL;
      69                 : 
      70               0 :   for (p=wlist;p->pos;p++)
      71                 :   {
      72               0 :     p->weight=PRENORM_IN_USE;
      73                 :   }
      74                 : 
      75               0 :   for (p=wlist;p->pos;p++)
      76                 :   {
      77               0 :     p->weight/=NORM_IN_USE;
      78                 :   }
      79                 : 
      80               0 :   DBUG_RETURN(wlist);
      81                 : }
      82                 : 
      83                 : my_bool maria_ft_boolean_check_syntax_string(const uchar *str)
      84               0 : {
      85                 :   uint i, j;
      86                 : 
      87               0 :   if (!str ||
      88                 :       (strlen((const char *) str) + 1 != sizeof(ft_boolean_syntax)) ||
      89                 :       (str[0] != ' ' && str[1] != ' '))
      90               0 :     return 1;
      91               0 :   for (i=0; i<sizeof(ft_boolean_syntax); i++)
      92                 :   {
      93                 :     /* limiting to 7-bit ascii only */
      94               0 :     if ((unsigned char)(str[i]) > 127 ||
      95                 :         my_isalnum(default_charset_info, str[i]))
      96               0 :       return 1;
      97               0 :     for (j=0; j<i; j++)
      98               0 :       if (str[i] == str[j] && (i != 11 || j != 10))
      99               0 :         return 1;
     100                 :   }
     101               0 :   return 0;
     102                 : }
     103                 : 
     104                 : /*
     105                 :   RETURN VALUE
     106                 :   0 - eof
     107                 :   1 - word found
     108                 :   2 - left bracket
     109                 :   3 - right bracket
     110                 :   4 - stopword found
     111                 : */
     112                 : uchar maria_ft_get_word(CHARSET_INFO *cs, uchar **start, uchar *end,
     113                 :                         FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
     114               0 : {
     115               0 :   uchar *doc=*start;
     116                 :   int ctype;
     117                 :   uint mwc, length;
     118                 :   int mbl;
     119                 : 
     120               0 :   param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
     121               0 :   param->weight_adjust= param->wasign= 0;
     122               0 :   param->type= FT_TOKEN_EOF;
     123                 : 
     124               0 :   while (doc<end)
     125                 :   {
     126               0 :     for (; doc < end; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     127                 :     {
     128               0 :       mbl= cs->cset->ctype(cs, &ctype, doc, end);
     129               0 :       if (true_word_char(ctype, *doc))
     130                 :         break;
     131               0 :       if (*doc == FTB_RQUOT && param->quot)
     132                 :       {
     133               0 :         param->quot= (char *) doc;
     134               0 :         *start=doc+1;
     135               0 :         param->type= FT_TOKEN_RIGHT_PAREN;
     136               0 :         goto ret;
     137                 :       }
     138               0 :       if (!param->quot)
     139                 :       {
     140               0 :         if (*doc == FTB_LBR || *doc == FTB_RBR || *doc == FTB_LQUOT)
     141                 :         {
     142                 :           /* param->prev=' '; */
     143               0 :           *start=doc+1;
     144               0 :           if (*doc == FTB_LQUOT)
     145               0 :             param->quot= (char *) *start;
     146               0 :           param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN);
     147               0 :           goto ret;
     148                 :         }
     149               0 :         if (param->prev == ' ')
     150                 :         {
     151               0 :           if (*doc == FTB_YES ) { param->yesno=+1;    continue; } else
     152               0 :           if (*doc == FTB_EGAL) { param->yesno= 0;    continue; } else
     153               0 :           if (*doc == FTB_NO  ) { param->yesno=-1;    continue; } else
     154               0 :           if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else
     155               0 :           if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else
     156               0 :           if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; }
     157                 :         }
     158                 :       }
     159               0 :       param->prev=*doc;
     160               0 :       param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
     161               0 :       param->weight_adjust= param->wasign= 0;
     162                 :     }
     163                 : 
     164               0 :     mwc=length=0;
     165               0 :     for (word->pos= doc; doc < end; length++,
     166               0 :          doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     167                 :     {
     168               0 :       mbl= cs->cset->ctype(cs, &ctype, doc, end);
     169               0 :       if (true_word_char(ctype, *doc))
     170               0 :         mwc=0;
     171                 :       else if (!misc_word_char(*doc) || mwc)
     172                 :         break;
     173                 :       else
     174                 :         mwc++;
     175                 :     }
     176               0 :     param->prev='A'; /* be sure *prev is true_word_char */
     177               0 :     word->len= (uint)(doc-word->pos) - mwc;
     178               0 :     if ((param->trunc=(doc<end && *doc == FTB_TRUNC)))
     179               0 :       doc++;
     180                 : 
     181               0 :     if (((length >= ft_min_word_len && !is_stopword((char *) word->pos,
     182                 :                                                     word->len))
     183                 :          || param->trunc) && length < ft_max_word_len)
     184                 :     {
     185               0 :       *start=doc;
     186               0 :       param->type= FT_TOKEN_WORD;
     187               0 :       goto ret;
     188                 :     }
     189               0 :     else if (length) /* make sure length > 0 (if start contains spaces only) */
     190                 :     {
     191               0 :       *start= doc;
     192               0 :       param->type= FT_TOKEN_STOPWORD;
     193               0 :       goto ret;
     194                 :     }
     195                 :   }
     196               0 :   if (param->quot)
     197                 :   {
     198               0 :     param->quot= (char *)(*start= doc);
     199               0 :     param->type= 3; /* FT_RBR */
     200                 :     goto ret;
     201                 :   }
     202               0 : ret:
     203               0 :   return param->type;
     204                 : }
     205                 : 
     206                 : uchar maria_ft_simple_get_word(CHARSET_INFO *cs, uchar **start,
     207                 :                                const uchar *end, FT_WORD *word,
     208                 :                                my_bool skip_stopwords)
     209               0 : {
     210               0 :   uchar *doc= *start;
     211                 :   uint mwc, length;
     212                 :   int ctype, mbl;
     213               0 :   DBUG_ENTER("maria_ft_simple_get_word");
     214                 : 
     215                 :   do
     216                 :   {
     217               0 :     for (;; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     218                 :     {
     219               0 :       if (doc >= end)
     220               0 :         DBUG_RETURN(0);
     221               0 :       mbl= cs->cset->ctype(cs, &ctype, doc, end);
     222               0 :       if (true_word_char(ctype, *doc))
     223                 :         break;
     224               0 :     }
     225                 : 
     226               0 :     mwc= length= 0;
     227               0 :     for (word->pos= doc; doc < end; length++,
     228               0 :          doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     229                 :     {
     230               0 :       mbl= cs->cset->ctype(cs, &ctype, doc, end);
     231               0 :       if (true_word_char(ctype, *doc))
     232               0 :         mwc= 0;
     233                 :       else if (!misc_word_char(*doc) || mwc)
     234                 :         break;
     235                 :       else
     236                 :         mwc++;
     237                 :     }
     238                 : 
     239               0 :     word->len= (uint)(doc-word->pos) - mwc;
     240                 : 
     241               0 :     if (skip_stopwords == FALSE ||
     242                 :         (length >= ft_min_word_len && length < ft_max_word_len &&
     243                 :          !is_stopword((char *) word->pos, word->len)))
     244                 :     {
     245               0 :       *start= doc;
     246               0 :       DBUG_RETURN(1);
     247                 :     }
     248               0 :   } while (doc < end);
     249               0 :   DBUG_RETURN(0);
     250                 : }
     251                 : 
     252                 : void maria_ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
     253               0 : {
     254               0 :   DBUG_ENTER("maria_ft_parse_init");
     255               0 :   if (!is_tree_inited(wtree))
     256               0 :     init_tree(wtree,0,0,sizeof(FT_WORD),(qsort_cmp2)&FT_WORD_cmp,0,NULL, cs);
     257               0 :   DBUG_VOID_RETURN;
     258                 : }
     259                 : 
     260                 : 
     261                 : static int maria_ft_add_word(MYSQL_FTPARSER_PARAM *param,
     262                 :                        char *word, int word_len,
     263                 :              MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
     264               0 : {
     265                 :   TREE *wtree;
     266                 :   FT_WORD w;
     267               0 :   MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
     268               0 :   DBUG_ENTER("maria_ft_add_word");
     269               0 :   wtree= ft_param->wtree;
     270               0 :   if (param->flags & MYSQL_FTFLAGS_NEED_COPY)
     271                 :   {
     272                 :     uchar *ptr;
     273               0 :     DBUG_ASSERT(wtree->with_delete == 0);
     274               0 :     ptr= (uchar *)alloc_root(ft_param->mem_root, word_len);
     275               0 :     memcpy(ptr, word, word_len);
     276               0 :     w.pos= ptr;
     277                 :   }
     278                 :   else
     279               0 :     w.pos= (uchar *) word;
     280               0 :   w.len= word_len;
     281               0 :   if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
     282                 :   {
     283               0 :     delete_tree(wtree);
     284               0 :     DBUG_RETURN(1);
     285                 :   }
     286               0 :   DBUG_RETURN(0);
     287                 : }
     288                 : 
     289                 : 
     290                 : static int maria_ft_parse_internal(MYSQL_FTPARSER_PARAM *param,
     291                 :                                    char *doc_arg, int doc_len)
     292               0 : {
     293               0 :   uchar *doc= (uchar*) doc_arg;
     294               0 :   uchar *end= doc + doc_len;
     295               0 :   MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
     296               0 :   TREE *wtree= ft_param->wtree;
     297                 :   FT_WORD w;
     298               0 :   DBUG_ENTER("maria_ft_parse_internal");
     299                 : 
     300               0 :   while (maria_ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
     301               0 :     if (param->mysql_add_word(param, (char *) w.pos, w.len, 0))
     302               0 :       DBUG_RETURN(1);
     303               0 :   DBUG_RETURN(0);
     304                 : }
     305                 : 
     306                 : 
     307                 : int maria_ft_parse(TREE *wtree, uchar *doc, int doclen,
     308                 :                     struct st_mysql_ftparser *parser,
     309                 :                    MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
     310               0 : {
     311                 :   MY_FT_PARSER_PARAM my_param;
     312               0 :   DBUG_ENTER("maria_ft_parse");
     313               0 :   DBUG_ASSERT(parser);
     314               0 :   my_param.wtree= wtree;
     315               0 :   my_param.mem_root= mem_root;
     316                 : 
     317               0 :   param->mysql_parse= maria_ft_parse_internal;
     318               0 :   param->mysql_add_word= maria_ft_add_word;
     319               0 :   param->mysql_ftparam= &my_param;
     320               0 :   param->cs= wtree->custom_arg;
     321               0 :   param->doc= (char *) doc;
     322               0 :   param->length= doclen;
     323               0 :   param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
     324               0 :   DBUG_RETURN(parser->parse(param));
     325                 : }
     326                 : 
     327                 : 
     328                 : #define MAX_PARAM_NR 2
     329                 : 
     330                 : MYSQL_FTPARSER_PARAM* maria_ftparser_alloc_param(MARIA_HA *info)
     331               0 : {
     332               0 :   if (!info->ftparser_param)
     333                 :   {
     334                 :     /* 
     335                 : .     info->ftparser_param can not be zero after the initialization,
     336                 :       because it always includes built-in fulltext parser. And built-in
     337                 :       parser can be called even if the table has no fulltext indexes and
     338                 :       no varchar/text fields.
     339                 : 
     340                 :       ftb_find_relevance... parser (ftb_find_relevance_parse,
     341                 :       ftb_find_relevance_add_word) calls ftb_check_phrase... parser
     342                 :       (ftb_check_phrase_internal, ftb_phrase_add_word). Thus MAX_PARAM_NR=2.
     343                 :     */
     344               0 :     info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
     345                 :       my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *
     346                 :                 info->s->ftkeys, MYF(MY_WME | MY_ZEROFILL));
     347               0 :     init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
     348                 :   }
     349               0 :   return info->ftparser_param;
     350                 : }
     351                 : 
     352                 : 
     353                 : MYSQL_FTPARSER_PARAM *maria_ftparser_call_initializer(MARIA_HA *info,
     354                 :                                                       uint keynr, uint paramnr)
     355               0 : {
     356                 :   uint32 ftparser_nr;
     357                 :   struct st_mysql_ftparser *parser;
     358                 :   
     359               0 :   if (!maria_ftparser_alloc_param(info))
     360               0 :     return 0;
     361                 : 
     362               0 :   if (keynr == NO_SUCH_KEY)
     363                 :   {
     364               0 :     ftparser_nr= 0;
     365               0 :     parser= &ft_default_parser;
     366                 :   }
     367                 :   else
     368                 :   {
     369               0 :     ftparser_nr= info->s->keyinfo[keynr].ftkey_nr;
     370               0 :     parser= info->s->keyinfo[keynr].parser;
     371                 :   }
     372               0 :   DBUG_ASSERT(paramnr < MAX_PARAM_NR);
     373               0 :   ftparser_nr= ftparser_nr*MAX_PARAM_NR + paramnr;
     374               0 :   if (! info->ftparser_param[ftparser_nr].mysql_add_word)
     375                 :   {
     376                 :     /* Note, that mysql_add_word is used here as a flag:
     377                 :        mysql_add_word == 0 - parser is not initialized
     378                 :        mysql_add_word != 0 - parser is initialized, or no
     379                 :                              initialization needed. */
     380               0 :     info->ftparser_param[ftparser_nr].mysql_add_word=
     381                 :       (int (*)(struct st_mysql_ftparser_param *, char *, int,
     382                 :                MYSQL_FTPARSER_BOOLEAN_INFO *)) 1;
     383               0 :     if (parser->init && parser->init(&info->ftparser_param[ftparser_nr]))
     384               0 :       return 0;
     385                 :   }
     386               0 :   return &info->ftparser_param[ftparser_nr];
     387                 : }
     388                 : 
     389                 : 
     390                 : void maria_ftparser_call_deinitializer(MARIA_HA *info)
     391               9 : {
     392               9 :   uint i, j, keys= info->s->state.header.keys;
     393               9 :   free_root(&info->ft_memroot, MYF(0));
     394               9 :   if (! info->ftparser_param)
     395               0 :     return;
     396               0 :   for (i= 0; i < keys; i++)
     397                 :   {
     398               0 :     MARIA_KEYDEF *keyinfo= &info->s->keyinfo[i];
     399               0 :     for (j=0; j < MAX_PARAM_NR; j++)
     400                 :     {
     401                 :       MYSQL_FTPARSER_PARAM *ftparser_param=
     402               0 :         &info->ftparser_param[keyinfo->ftkey_nr*MAX_PARAM_NR + j];
     403               0 :       if (keyinfo->flag & HA_FULLTEXT && ftparser_param->mysql_add_word)
     404                 :       {
     405               0 :         if (keyinfo->parser->deinit)
     406               0 :           keyinfo->parser->deinit(ftparser_param);
     407               0 :         ftparser_param->mysql_add_word= 0;
     408                 :       }
     409                 :       else
     410                 :         break;
     411                 :     }
     412                 :   }
     413                 : }

Generated by: LTP GCOV extension version 1.4