YAZ  5.23.1
stemmer.c
Go to the documentation of this file.
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 
11 #if HAVE_CONFIG_H
12 #include <config.h>
13 #endif
14 
15 #if YAZ_HAVE_ICU
16 
17 #include <yaz/stemmer.h>
18 
19 #include <yaz/xmalloc.h>
20 
21 #include <libstemmer.h>
22 
23 #include <unicode/ustring.h> /* some more string fcns*/
24 #include <unicode/uchar.h> /* char names */
25 
26 enum stemmer_implementation {
27  yaz_no_operation,
28  yaz_snowball
29 };
30 struct yaz_stemmer_t
31 {
32  int implementation;
33  // Required for cloning.
34  char *locale;
35  char *rule;
36  struct sb_stemmer *sb_stemmer;
37 };
38 
39 const char* yaz_stemmer_lookup_charenc(const char *charenc, const char *rule) {
40  return "UTF_8";
41 }
42 
43 const char* yaz_stemmer_lookup_algorithm(const char *locale, const char *rule) {
44  return locale;
45 }
46 
47 yaz_stemmer_p yaz_stemmer_snowball_create(const char *locale, const char *rule, UErrorCode *status) {
48  const char *charenc = yaz_stemmer_lookup_charenc(locale, rule);
49  const char *algorithm = yaz_stemmer_lookup_algorithm(locale,rule);
50  struct sb_stemmer *stemmer = sb_stemmer_new(algorithm, charenc);
51  yaz_stemmer_p yaz_stemmer;
52  if (stemmer == 0) {
53  *status = U_ILLEGAL_ARGUMENT_ERROR;
54  yaz_log(YLOG_FATAL, "yaz_stemmer: Failed to create snowball stemmer from locale %srule %s. Showball: charenc %s algorithm %s ",
55  locale, rule, charenc, algorithm);
56  return 0;
57  }
58  yaz_log(YLOG_DEBUG, "created snowball stemmer: algorithm %s charenc %s ", algorithm, charenc);
59  yaz_stemmer = xmalloc(sizeof(*yaz_stemmer));
60  yaz_stemmer->implementation = yaz_snowball;
61 
62  yaz_stemmer->locale = xstrdup(locale);
63  yaz_stemmer->rule = xstrdup(rule);
64  yaz_stemmer->sb_stemmer = stemmer;
65  yaz_log(YLOG_DEBUG, "created snowball stemmer: algorithm %s charenc %s ", algorithm, charenc);
66  return yaz_stemmer;
67 }
68 
69 yaz_stemmer_p yaz_stemmer_create(const char *locale, const char *rule, UErrorCode *status) {
70  *status = U_ZERO_ERROR;
71  // dispatch logic required if more algorithms is implemented.
72  yaz_log(YLOG_DEBUG, "create stemmer: locale %s rule %s ", locale, rule);
73  return yaz_stemmer_snowball_create(locale, rule, status);
74 }
75 
77  UErrorCode error = U_ZERO_ERROR;
78  if (stemmer == 0)
79  return 0;
80  return yaz_stemmer_create(stemmer->locale, stemmer->rule, &error);
81 }
82 
83 void yaz_stemmer_stem(yaz_stemmer_p stemmer, struct icu_buf_utf16 *dst, struct icu_buf_utf16* src, UErrorCode *status)
84 {
85  switch(stemmer->implementation) {
86  case yaz_snowball: {
87  struct icu_buf_utf8 *utf8_buf = icu_buf_utf8_create(0);
88  icu_utf16_to_utf8(utf8_buf, src, status);
89  if (*status == U_ZERO_ERROR) {
90  const sb_symbol *cstr = (const sb_symbol*) icu_buf_utf8_to_cstr(utf8_buf);
91  const sb_symbol *sb_symbol = sb_stemmer_stem(stemmer->sb_stemmer, cstr, utf8_buf->utf8_len);
92  if (sb_symbol == 0) {
93  icu_buf_utf16_copy(dst, src);
94  }
95  else {
96 
97  const char *cstr2 = (const char *) sb_symbol;
98  icu_utf16_from_utf8_cstr(dst, cstr2 , status);
99 #if 0
100  yaz_log(YLOG_DEBUG, "stemming %s to %s ", cstr, cstr2);
101 #endif
102  }
103  }
104  icu_buf_utf8_destroy(utf8_buf);
105  return ;
106  break;
107  }
108  case yaz_no_operation:
109  yaz_log(YLOG_DEBUG, "Stemmer (No operation) called");
110  default: {
111  // Default return the same as given.
112  icu_buf_utf16_copy(dst, src);
113  }
114  }
115 }
116 
118 {
119  /* Handle no stemmer correctly */
120  if (stemmer == 0)
121  return ;
122 
123  switch (stemmer->implementation) {
124  case yaz_snowball:
125  sb_stemmer_delete(stemmer->sb_stemmer);
126  break;
127  }
128  xfree(stemmer->locale);
129  xfree(stemmer->rule);
130  xfree(stemmer);
131 }
132 
133 #endif /* YAZ_HAVE_ICU */
Header for the stemming API.
int32_t utf8_len
Definition: icu_I18N.h:81
#define YLOG_DEBUG
log level: debugging
Definition: log.h:44
#define xstrdup(s)
utility macro which calls xstrdup_f
Definition: xmalloc.h:55
#define YLOG_FATAL
log level: fatal
Definition: log.h:42
const char * icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 *dest8, const struct icu_buf_utf16 *src16, UErrorCode *status)
#define xfree(x)
utility macro which calls xfree_f
Definition: xmalloc.h:53
yaz_stemmer_p yaz_stemmer_clone(yaz_stemmer_p stemmer)
void yaz_stemmer_stem(yaz_stemmer_p stemmer, struct icu_buf_utf16 *dst, struct icu_buf_utf16 *src, UErrorCode *status)
#define xmalloc(x)
utility macro which calls malloc_f
Definition: xmalloc.h:49
Header for memory handling functions.
void yaz_log(int level, const char *fmt,...)
Writes log message.
Definition: log.c:485
void yaz_stemmer_destroy(yaz_stemmer_p stemmer)
yaz_stemmer_p yaz_stemmer_create(const char *locale, const char *rule, UErrorCode *status)
struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 *dest16, const struct icu_buf_utf16 *src16)
void icu_buf_utf8_destroy(struct icu_buf_utf8 *buf8)
UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 *dest16, const char *src8cstr, UErrorCode *status)
struct yaz_stemmer_t * yaz_stemmer_p
Definition: stemmer.h:49
struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)