IDZEBRA  2.1.2
retrieve.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #include <stdio.h>
24 #include <assert.h>
25 
26 #include <fcntl.h>
27 #ifdef WIN32
28 #include <io.h>
29 #include <process.h>
30 #endif
31 #if HAVE_UNISTD_H
32 #include <unistd.h>
33 #endif
34 
35 #include "index.h"
36 #include <yaz/diagbib1.h>
37 #include <yaz/snprintf.h>
38 #include <direntz.h>
39 #include <yaz/oid_db.h>
40 #include <zebra_strmap.h>
41 
42 #define MAX_SYSNOS_PER_RECORD 40
43 
44 #define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
45 
48  const char *setname;
50  int score;
51  NMEM nmem;
52 };
53 
55  Record *rec,
56  struct ZebraRecStream *stream)
57 {
58  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
59 
60  if ((*rec)->size[recInfo_storeData] > 0
61  || (*rec)->info[recInfo_filename] == 0)
62  zebra_create_stream_mem(stream, (*rec)->info[recInfo_storeData],
63  (*rec)->size[recInfo_storeData]);
64  else
65  {
66  char full_rep[1024];
67  int fd;
68 
69  if (zh->path_reg && !yaz_is_abspath((*rec)->info[recInfo_filename])){
70  strcpy(full_rep, zh->path_reg);
71  strcat(full_rep, "/");
72  strcat(full_rep, (*rec)->info[recInfo_filename]);
73  }
74  else
75  strcpy(full_rep, (*rec)->info[recInfo_filename]);
76 
77  if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1){
78  yaz_log(YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
79  full_rep);
80  rec_free(rec);
81  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
82  }
83  zebra_create_stream_fd(stream, fd, recordAttr->recordOffset);
84  }
85  return 0;
86 }
87 
88 
89 struct index_spec {
90  const char *index_name;
91  const char *index_type;
92  const char *extra;
93  struct index_spec *next;
94 };
95 
96 
97 struct index_spec *parse_index_spec(const char *elem, NMEM nmem,
98  int *error)
99 {
100  struct index_spec *first = 0;
101  struct index_spec **last = &first;
102  const char *cp = elem;
103 
104  *error = 0;
105  if (cp[0] == ':' && cp[1] == ':')
106  {
107 
108  cp++; /* skip first ':' */
109 
110  for (;;)
111  {
112  const char *cp0;
113  struct index_spec *spec = nmem_malloc(nmem, sizeof(*spec));
114  spec->index_type = 0;
115  spec->next = 0;
116  spec->extra = 0;
117 
118  if (!first)
119  first = spec;
120  *last = spec;
121  last = &spec->next;
122 
123  cp++; /* skip ',' or second ':' */
124  cp0 = cp;
125  while (*cp != ':' && *cp != '\0' && *cp != ',')
126  cp++;
127  spec->index_name = nmem_strdupn(nmem, cp0, cp - cp0);
128  if (*cp == ':') /* type as well */
129  {
130  cp++;
131  cp0 = cp;
132 
133  while (*cp != '\0' && *cp != ',' && *cp != ':')
134  cp++;
135  spec->index_type = nmem_strdupn(nmem, cp0, cp - cp0);
136  }
137  if (*cp == ':') /* extra arguments */
138  {
139  cp++;
140  cp0 = cp;
141 
142  while (*cp != '\0' && *cp != ',' && *cp != ':')
143  cp++;
144  spec->extra = nmem_strdupn(nmem, cp0, cp - cp0);
145  }
146  if (*cp != ',')
147  break;
148  }
149  }
150  if (*cp != '\0')
151  *error = 1;
152  return first;
153 }
154 
155 static int sort_fetch(
156  struct special_fetch_s *fi, const char *elemsetname,
157  const Odr_oid *input_format,
158  const Odr_oid **output_format,
159  WRBUF result, WRBUF addinfo)
160 {
161  int ord;
162  ZebraHandle zh = fi->zh;
163  int error;
164  struct index_spec *spec;
165 
166  spec = parse_index_spec(elemsetname, fi->nmem, &error);
167  if (error)
168  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
169 
170  /* for sort fetches.. We allow only one index and type must be given */
171  if (!spec || spec->next || !spec->index_type)
172  return -1;
175  spec->index_type,
176  spec->index_name);
177  if (ord == -1)
178  return -1; /* is not a sort index */
179  else
180  {
181  WRBUF wrbuf_str = wrbuf_alloc();
182  const char *index_type;
183  const char *db = 0;
184  const char *string_index = 0;
185  WRBUF wrbuf_result = result;
186  int off = 0;
187 
188  zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
189  &string_index);
190  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
191  {
192  *output_format = yaz_oid_recsyn_xml;
193  wrbuf_printf(wrbuf_result, ZEBRA_XML_HEADER_STR
194  " sysno=\"" ZINT_FORMAT "\""
195  " set=\"zebra::index%s\">\n",
196  fi->sysno, elemsetname);
197  }
198  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
199  {
200  *output_format = yaz_oid_recsyn_sutrs;
201  }
202  else
203  {
204  yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
205  elemsetname);
206  *output_format = 0;
207  wrbuf_destroy(wrbuf_str);
208  return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
209  }
210  zebra_sort_type(zh->reg->sort_index, ord);
212  zebra_sort_read(zh->reg->sort_index, 0, wrbuf_str);
213 
214  while (off != wrbuf_len(wrbuf_str))
215  {
216  char dst_buf[IT_MAX_WORD];
217  assert(off < wrbuf_len(wrbuf_str));
218  zebra_term_untrans(zh, index_type, dst_buf,
219  wrbuf_buf(wrbuf_str)+off);
220 
221  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
222  {
223  wrbuf_printf(wrbuf_result, " <index name=\"%s\"",
224  string_index);
225  wrbuf_printf(wrbuf_result, " type=\"%s\">", index_type);
226  wrbuf_xmlputs(wrbuf_result, dst_buf);
227  wrbuf_printf(wrbuf_result, "</index>\n");
228  }
229  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
230  {
231  wrbuf_printf(wrbuf_result, "%s %s %s\n", string_index, index_type,
232  dst_buf);
233  }
234  off += strlen(wrbuf_buf(wrbuf_str)+off) + 1;
235  }
236  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
237  {
238  wrbuf_printf(wrbuf_result, "</record>\n");
239  }
240  wrbuf_destroy(wrbuf_str);
241  return 0;
242  }
243 }
244 
246  struct special_fetch_s *fi, const char *elemsetname,
247  const Odr_oid *input_format,
248  const Odr_oid **output_format,
249  WRBUF result, WRBUF addinfo,
250  Record rec)
251 {
252  zebra_rec_keys_t keys;
253  int ret_code = 0;
254  ZebraHandle zh = fi->zh;
255  struct index_spec *spec, *spec_list;
256  int error;
257 
258  /* set output variables before processing possible error states */
259  /* *rec_lenp = 0; */
260 
261  /* only accept XML and SUTRS requests */
262  if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
263  && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
264  {
265  yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
266  elemsetname);
267  *output_format = 0;
268  return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
269  }
270 
271  spec_list = parse_index_spec(elemsetname, fi->nmem, &error);
272  if (error)
273  {
274  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
275  }
276 
277  for (spec = spec_list; spec; spec = spec->next)
278  {
281  spec->index_type,
282  spec->index_name) == -1)
283  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
284  }
285 
286  keys = zebra_rec_keys_open();
288  rec->size[recInfo_delKeys], 0);
289 
290  if (!zebra_rec_keys_rewind(keys))
291  {
292  ret_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
293  }
294  else
295  {
296  size_t slen;
297  const char *str;
298  struct it_key key_in;
299  WRBUF wrbuf = result;
300 
301  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
302  {
303  *output_format = input_format;
304  wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
305  " sysno=\"" ZINT_FORMAT "\""
306  " set=\"zebra::index%s\">\n",
307  fi->sysno, elemsetname);
308  }
309  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
310  *output_format = input_format;
311 
312  while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
313  {
314  int i;
315  int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
316  const char *index_type;
317  const char *db = 0;
318  const char *string_index = 0;
319  char dst_buf[IT_MAX_WORD];
320  int match = 0;
321 
322  zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
323  &string_index);
324  if (!spec_list)
325  match = 1; /* match all if no specs were given */
326  else
327  {
328  for (spec = spec_list; spec; spec = spec->next)
329  {
330  if ((!spec->index_type ||
331  !yaz_matchstr(spec->index_type, index_type))
332  &&
333  !yaz_matchstr(spec->index_name, string_index))
334  match = 1;
335  }
336  }
337  if (match)
338  {
339  if (zebra_term_untrans(zh, index_type, dst_buf, str))
340  *dst_buf = '\0'; /* untrans failed */
341 
342  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
343  {
344  wrbuf_printf(wrbuf, " <index name=\"%s\"",
345  string_index);
346 
347  wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
348 
349  wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">",
350  key_in.mem[key_in.len -1]);
351  wrbuf_xmlputs(wrbuf, dst_buf);
352  wrbuf_printf(wrbuf, "</index>\n");
353  }
354  else
355  {
356  wrbuf_printf(wrbuf, "%s ", string_index);
357 
358  wrbuf_printf(wrbuf, "%s", index_type);
359 
360  for (i = 1; i < key_in.len; i++)
361  wrbuf_printf(wrbuf, " " ZINT_FORMAT,
362  key_in.mem[i]);
363 
364  wrbuf_printf(wrbuf, " %s", dst_buf);
365 
366  wrbuf_printf(wrbuf, "\n");
367  }
368  }
369  }
370  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
371  wrbuf_printf(wrbuf, "</record>\n");
372  }
373  zebra_rec_keys_close(keys);
374  return ret_code;
375 }
376 
377 
378 static void retrieve_puts_attr(WRBUF wrbuf, const char *name,
379  const char *value)
380 {
381  if (value)
382  {
383  wrbuf_printf(wrbuf, " %s=\"", name);
384  wrbuf_xmlputs(wrbuf, value);
385  wrbuf_printf(wrbuf, "\"");
386  }
387 }
388 
389 static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name,
390  const int value)
391 {
392  wrbuf_printf(wrbuf, " %s=\"%i\"", name, value);
393 }
394 
395 static void retrieve_puts_str(WRBUF wrbuf, const char *name,
396  const char *value)
397 {
398  if (value)
399  wrbuf_printf(wrbuf, "%s %s\n", name, value);
400 }
401 
402 static void retrieve_puts_int(WRBUF wrbuf, const char *name,
403  const int value)
404 {
405  wrbuf_printf(wrbuf, "%s %i\n", name, value);
406 }
407 
408 
409 static void snippet_check_fields(ZebraHandle zh, WRBUF wrbuf,
410  zebra_snippets *doc,
411  const zebra_snippet_word *doc_w,
412  const char *w_index_type)
413 {
414  /* beginning of snippet. See which fields the snippet also
415  occur */
416  const zebra_snippet_word *w;
417  int no = 0;
418  for (w = zebra_snippets_constlist(doc); w; w = w->next)
419  {
420  /* same sequence but other field? */
421  if (w->seqno == doc_w->seqno && w->ord != doc_w->ord)
422  {
423  const char *index_type;
424  const char *db = 0;
425  const char *string_index = 0;
426 
428  &index_type, &db, &string_index);
429  /* only report for same index type */
430  if (!strcmp(w_index_type, index_type))
431  {
432  if (no == 0)
433  wrbuf_printf(wrbuf, " fields=\"%s", string_index);
434  else
435  wrbuf_printf(wrbuf, " %s", string_index);
436  no++;
437  }
438  }
439  }
440  if (no)
441  wrbuf_printf(wrbuf, "\"");
442 }
443 
444 static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
445 {
446  const zebra_snippet_word *doc_w;
447  int mark_state = 0;
448 
449  wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
450  for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
451  {
452  if (doc_w->mark)
453  {
454  const char *index_type;
455  const char *db = 0;
456  const char *string_index = 0;
457 
458  zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord,
459  &index_type, &db, &string_index);
460 
461  if (mark_state == 0)
462  {
463 
464  wrbuf_printf(wrbuf, " <snippet name=\"%s\"", string_index);
465  wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
466  snippet_check_fields(zh, wrbuf, doc, doc_w, index_type);
467  wrbuf_printf(wrbuf, ">");
468  }
469  if (doc_w->match)
470  wrbuf_puts(wrbuf, "<s>");
471  /* not printing leading ws */
472  if (mark_state || !doc_w->ws || doc_w->match)
473  wrbuf_xmlputs(wrbuf, doc_w->term);
474  if (doc_w->match)
475  wrbuf_puts(wrbuf, "</s>");
476  }
477  else if (mark_state == 1)
478  {
479  wrbuf_puts(wrbuf, "</snippet>\n");
480  }
481  mark_state = doc_w->mark;
482  }
483  if (mark_state == 1)
484  {
485  wrbuf_puts(wrbuf, "</snippet>\n");
486  }
487  wrbuf_printf(wrbuf, "</record>");
488 }
489 
491  zebra_snippets *snippets)
492 {
493  int return_code = 0;
494  Record rec = rec_get(zh->reg->records, sysno);
495  if (!rec)
496  {
497  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
498  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
499  }
500  else
501  {
502  const char *file_type = rec->info[recInfo_fileType];
503  void *recTypeClientData;
504  RecType rt = recType_byName(zh->reg->recTypes, zh->res,
505  file_type, &recTypeClientData);
506 
507  if (!rt)
508  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
509  else
510  {
511  struct ZebraRecStream stream;
512  return_code = zebra_create_record_stream(zh, &rec, &stream);
513  if (return_code == 0)
514  {
515  extract_snippet(zh, snippets, &stream,
516  rt, recTypeClientData);
517 
518  stream.destroy(&stream);
519  }
520  }
521  rec_free(&rec);
522  }
523  return return_code;
524 }
525 
526 static int snippet_fetch(
527  struct special_fetch_s *fi, const char *elemsetname,
528  const Odr_oid *input_format,
529  const Odr_oid **output_format,
530  WRBUF result, WRBUF addinfo)
531 {
532  ZebraHandle zh = fi->zh;
533  zebra_snippets *rec_snippets = zebra_snippets_create();
534  int return_code = zebra_get_rec_snippets(zh, fi->sysno, rec_snippets);
535 
536  if (!return_code)
537  {
538  WRBUF wrbuf = result;
539  zebra_snippets *hit_snippet = zebra_snippets_create();
540 
541  zebra_snippets_hit_vector(zh, fi->setname, fi->sysno, hit_snippet);
542 
543 #if 0
544  /* for debugging purposes */
545  yaz_log(YLOG_LOG, "---------------------------");
546  yaz_log(YLOG_LOG, "REC SNIPPET:");
547  zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
548  yaz_log(YLOG_LOG, "---------------------------");
549  yaz_log(YLOG_LOG, "HIT SNIPPET:");
550  zebra_snippets_log(hit_snippet, YLOG_LOG, 1);
551 #endif
552 
553  zebra_snippets_ring(rec_snippets, hit_snippet, 5, 5);
554 
555 #if 0
556  yaz_log(YLOG_LOG, "---------------------------");
557  yaz_log(YLOG_LOG, "RING SNIPPET:");
558  zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
559 #endif
560  snippet_xml_record(zh, wrbuf, rec_snippets);
561 
562  *output_format = yaz_oid_recsyn_xml;
563 
564  zebra_snippets_destroy(hit_snippet);
565  }
566  zebra_snippets_destroy(rec_snippets);
567  return return_code;
568 }
569 
570 struct term_collect {
571  const char *term;
572  int oc;
574 };
575 
576 static zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set)
577 {
579  char ord_buf[IT_MAX_WORD];
580  int ord_len = key_SU_encode(ord, ord_buf);
581  char *info;
582  zint hits = 0;
583  NMEM nmem = nmem_create();
584 
585  strcpy(ord_buf + ord_len, term);
586 
587  info = dict_lookup(zh->reg->dict, ord_buf);
588  if (info)
589  {
590  ISAM_P isam_p;
591  RSET rsets[2], rset;
592  memcpy(&isam_p, info+1, sizeof(ISAM_P));
593 
594  rsets[0] = zebra_create_rset_isam(zh, nmem, kc, kc->scope, isam_p, 0);
595  rsets[1] = rset_dup(rset_set);
596 
597  rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
598 
599  zebra_count_set(zh, rset, &hits, zh->approx_limit);
600 
601  rset_delete(rsets[0]);
602  rset_delete(rset);
603  }
604  (*kc->dec)(kc);
605  nmem_destroy(nmem);
606  return hits;
607 }
608 
609 static int term_qsort_handle(const void *a, const void *b)
610 {
611  const struct term_collect *l = a;
612  const struct term_collect *r = b;
613  if (l->set_occur < r->set_occur)
614  return 1;
615  else if (l->set_occur > r->set_occur)
616  return -1;
617  else
618  {
619  const char *lterm = l->term ? l->term : "";
620  const char *rterm = r->term ? r->term : "";
621  return strcmp(lterm, rterm);
622  }
623 }
624 
626  struct term_collect *col, int no_terms_collect,
627  int ord, RSET rset, double scale_factor)
628 {
629  int i;
630  for (i = 0; i < no_terms_collect; i++)
631  {
632  if (col[i].term)
633  {
634  if (scale_factor < 0.0)
635  {
636  col[i].set_occur = freq_term(zh, ord, col[i].term, rset);
637  }
638  else
639  col[i].set_occur = scale_factor * col[i].oc;
640  }
641  }
642  qsort(col, no_terms_collect, sizeof(*col), term_qsort_handle);
643 }
644 
646  int no_terms_collect,
647  NMEM nmem)
648 {
649  const char *term;
650  void *data_buf;
651  size_t data_len;
652  zebra_strmap_it it;
653  struct term_collect *col = nmem_malloc(nmem,
654  sizeof *col *no_terms_collect);
655  int i;
656  for (i = 0; i < no_terms_collect; i++)
657  {
658  col[i].term = 0;
659  col[i].oc = 0;
660  col[i].set_occur = 0;
661  }
662  /* iterate over terms and collect the most frequent ones */
663  it = zebra_strmap_it_create(sm);
664  while ((term = zebra_strmap_it_next(it, &data_buf, &data_len)))
665  {
666  /* invariant:
667  col[0] has lowest oc . col[no_terms_collect-1] has highest oc */
668  int oc = *(int*) data_buf;
669  int j = 0;
670  /* insertion may be slow but terms terms will be "infrequent" and
671  thus number of iterations should be small below
672  */
673  while (j < no_terms_collect && oc > col[j].oc)
674  j++;
675  if (j)
676  { /* oc <= col[j] and oc > col[j-1] */
677  --j;
678  memmove(col, col+1, sizeof(*col) * j);
679  col[j].term = term;
680  col[j].oc = oc;
681  }
682  }
684  return col;
685 }
686 
687 static int perform_facet_sort(ZebraHandle zh, int no_ord, int *ord_array,
688  zebra_strmap_t *map_array,
689  int num_recs, ZebraMetaRecord *poset)
690 {
691  int rec_i;
692  WRBUF w = wrbuf_alloc();
693  int ord_i;
694 
695  for (ord_i = 0; ord_i < no_ord; ord_i++)
696  {
697  for (rec_i = 0; rec_i < num_recs; rec_i++)
698  {
699  if (!poset[rec_i].sysno)
700  continue;
701 
702  zebra_sort_sysno(zh->reg->sort_index, poset[rec_i].sysno);
703  zebra_sort_type(zh->reg->sort_index, ord_array[ord_i]);
704 
705  wrbuf_rewind(w);
706  if (zebra_sort_read(zh->reg->sort_index, 0, w))
707  {
708  zebra_strmap_t sm = map_array[ord_i];
709  int off = 0;
710  while (off != wrbuf_len(w))
711  {
712  const char *str = wrbuf_buf(w) + off;
713  int *freq = zebra_strmap_lookup(sm, str, 0, 0);
714  if (freq)
715  (*freq)++;
716  else
717  {
718  int v = 1;
719  zebra_strmap_add(sm, str, &v, sizeof v);
720  }
721  off += strlen(str)+1;
722  }
723  }
724  }
725  }
726  wrbuf_destroy(w);
727  return 0;
728 }
729 
730 
732  struct special_fetch_s *fi,
733  int no_ord, int *ord_array,
734  zebra_strmap_t *map_array,
735  int num_recs, ZebraMetaRecord *poset,
736  struct index_spec *spec_list)
737 {
738  int max_chunks = 2;
739  int rec_i;
740  res_get_int(zh->res, "facetMaxChunks", &max_chunks);
741 
742  for (rec_i = 0; rec_i < num_recs; rec_i++)
743  {
744  int j;
745  zint sysnos[MAX_SYSNOS_PER_RECORD];
746  int no_sysnos = MAX_SYSNOS_PER_RECORD;
747  if (!poset[rec_i].sysno)
748  continue;
750  poset[rec_i].sysno,
751  sysnos, &no_sysnos);
752  assert(no_sysnos > 0);
753  yaz_log(YLOG_DEBUG, "Analyzing rec=%d ISAM sysno=" ZINT_FORMAT " chunks=%d",
754  rec_i, poset[rec_i].sysno, no_sysnos);
755  for (j = 0; j < no_sysnos && j < max_chunks; j++)
756  {
757  size_t slen;
758  const char *str;
759  struct it_key key_in;
760  Record rec = rec_get(zh->reg->records, sysnos[j]);
762  zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
763  rec->size[recInfo_delKeys], 0);
764 
765  yaz_log(YLOG_DEBUG, "rec %d " ZINT_FORMAT " %s",
766  j, sysnos[j], zebra_rec_keys_empty(keys) ? "empty" : "non-empty");
767  if (zebra_rec_keys_rewind(keys))
768  {
769  while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
770  {
771  int ord_i;
772  struct index_spec *spec;
773  for (spec = spec_list, ord_i = 0; ord_i < no_ord;
774  ord_i++, spec = spec->next)
775  {
776  int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
777  if (ord == ord_array[ord_i] &&
778  str[0] != FIRST_IN_FIELD_CHAR)
779  {
780  int *freq;
781  zebra_strmap_t sm = map_array[ord_i];
782 
783  freq = zebra_strmap_lookup(sm, str, 0, 0);
784  if (freq)
785  (*freq)++;
786  else
787  {
788  int v = 1;
789  zebra_strmap_add(sm, str, &v, sizeof v);
790  }
791  }
792  }
793  }
794  }
795  zebra_rec_keys_close(keys);
796  rec_free(&rec);
797  }
798  }
799  return 0;
800 }
801 
803  struct special_fetch_s *fi,
804  WRBUF result,
805  int num_recs, ZebraMetaRecord *poset,
806  struct index_spec *spec_list,
807  int no_ord, int *ord_array,
808  int use_xml,
810 {
811  int i;
812  int ret = 0;
813  WRBUF wr = result;
814  struct index_spec *spec;
815  yaz_timing_t timing = yaz_timing_create();
816  zebra_strmap_t *map_array
817  = nmem_malloc(fi->nmem, sizeof *map_array * no_ord);
818  for (i = 0; i < no_ord; i++)
819  map_array[i] = zebra_strmap_create();
820 
821  if (cat == zinfo_index_category_sort)
822  perform_facet_sort(zh, no_ord, ord_array, map_array,
823  num_recs, poset);
824  else
825  perform_facet_index(zh, fi, no_ord, ord_array, map_array,
826  num_recs, poset, spec_list);
827  yaz_timing_stop(timing);
828  yaz_log(YLOG_LOG, "facet first phase real=%4.2f cat=%s",
829  yaz_timing_get_real(timing),
830  (cat == zinfo_index_category_sort) ? "sort" : "index");
831  yaz_timing_start(timing);
832  for (spec = spec_list, i = 0; i < no_ord; i++, spec = spec->next)
833  {
834  int j;
835  NMEM nmem = nmem_create();
836  struct term_collect *col;
837  int no_collect_terms = 20;
838 
839  if (spec->extra)
840  no_collect_terms = atoi(spec->extra);
841  if (no_collect_terms < 1)
842  no_collect_terms = 1;
843  col = term_collect_create(map_array[i], no_collect_terms, nmem);
844  term_collect_freq(zh, col, no_collect_terms, ord_array[i],
845  resultSetRef(zh, fi->setname),
846  cat == zinfo_index_category_sort ? 1.0 : -1.0);
847 
848  if (use_xml)
849  wrbuf_printf(wr, " <facet type=\"%s\" index=\"%s\">\n",
850  spec->index_type, spec->index_name);
851  else
852  wrbuf_printf(wr, "facet %s %s\n",
853  spec->index_type, spec->index_name);
854  for (j = 0; j < no_collect_terms; j++)
855  {
856  if (col[j].term)
857  {
858  char dst_buf[IT_MAX_WORD];
859  zebra_term_untrans(zh, spec->index_type, dst_buf, col[j].term);
860  if (use_xml)
861  {
862  wrbuf_printf(wr, " <term coccur=\"%d\"", col[j].oc);
863  if (col[j].set_occur)
864  wrbuf_printf(wr, " occur=\"" ZINT_FORMAT "\"",
865  col[j].set_occur);
866  wrbuf_printf(wr, ">");
867  wrbuf_xmlputs(wr, dst_buf);
868  wrbuf_printf(wr, "</term>\n");
869  }
870  else
871  {
872  wrbuf_printf(wr, "term %d", col[j].oc);
873  if (col[j].set_occur)
874  wrbuf_printf(wr, " " ZINT_FORMAT,
875  col[j].set_occur);
876  wrbuf_printf(wr, ": %s\n", dst_buf);
877  }
878  }
879  }
880  if (use_xml)
881  wrbuf_puts(wr, " </facet>\n");
882  nmem_destroy(nmem);
883  }
884  for (i = 0; i < no_ord; i++)
885  zebra_strmap_destroy(map_array[i]);
886  yaz_timing_stop(timing);
887  yaz_log(YLOG_LOG, "facet second phase real=%4.2f",
888  yaz_timing_get_real(timing));
889  yaz_timing_destroy(&timing);
890  return ret;
891 }
892 
893 static int facet_fetch(
894  struct special_fetch_s *fi, const char *elemsetname,
895  const Odr_oid *input_format,
896  const Odr_oid **output_format,
897  WRBUF result, WRBUF addinfo)
898 {
899  zint *pos_array;
900  int i;
901  int num_recs = 10; /* number of records to analyze */
902  ZebraMetaRecord *poset;
903  ZEBRA_RES ret = ZEBRA_OK;
904  int *ord_array;
905  int use_xml = 0;
906  int no_ord = 0;
907  struct index_spec *spec, *spec_list;
908  int error;
909  ZebraHandle zh = fi->zh;
910  /* whether sort or index based */
912 
913  /* see if XML is required for response */
914  if (oid_oidcmp(input_format, yaz_oid_recsyn_xml) == 0)
915  use_xml = 1;
916 
917  spec_list = parse_index_spec(elemsetname, fi->nmem, &error);
918 
919  if (!spec_list || error)
920  {
921  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
922  }
923 
924  for (spec = spec_list; spec; spec = spec->next)
925  {
926  if (!spec->index_type)
927  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
928  no_ord++;
929  }
930 
931  /* try to see if all specs are sort based.. If not, try the
932  index based ones */
933  ord_array = nmem_malloc(fi->nmem, sizeof(*ord_array) * no_ord);
934 
935  for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
936  {
937  int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
939  spec->index_type,
940  spec->index_name);
941  if (ord == -1)
942  break;
943  ord_array[i] = ord;
944  num_recs = 10000;
945  }
946  if (spec)
947  {
949  for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
950  {
951  int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
953  spec->index_type,
954  spec->index_name);
955  if (ord == -1)
956  break;
957  ord_array[i] = ord;
958 
959  }
960  }
961  if (spec)
962  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
963 
964  res_get_int(zh->res, "facetNumRecs", &num_recs);
965 
966  pos_array = (zint *) nmem_malloc(fi->nmem, num_recs * sizeof(*pos_array));
967  for (i = 0; i < num_recs; i++)
968  pos_array[i] = i+1;
969  poset = zebra_meta_records_create(zh, fi->setname, num_recs, pos_array);
970  if (!poset)
971  {
972  wrbuf_puts(addinfo, fi->setname);
973  return YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
974  }
975  else
976  {
977  if (use_xml)
978  {
979  wrbuf_printf(result, ZEBRA_XML_HEADER_STR ">\n");
980  }
981  ret = perform_facet(zh, fi, result, num_recs, poset,
982  spec_list, no_ord, ord_array, use_xml,
983  cat);
984  if (use_xml)
985  wrbuf_puts(result, "</record>\n");
986  }
987  *output_format = yaz_oid_recsyn_xml;
988  zebra_meta_records_destroy(zh, poset, num_recs);
989  return ret;
990 }
991 
992 
994  void *handle, const char *elemsetname,
995  const Odr_oid *input_format,
996  const Odr_oid **output_format,
997  WRBUF result, WRBUF addinfo)
998 {
999  Record rec = 0;
1000  struct special_fetch_s *fi = (struct special_fetch_s *) handle;
1001  ZebraHandle zh = fi->zh;
1002  zint sysno = fi->sysno;
1003 
1004  /* processing zebra::facet */
1005  if (elemsetname && 0 == strncmp(elemsetname, "facet", 5))
1006  {
1007  return facet_fetch(fi, elemsetname + 5,
1008  input_format, output_format,
1009  result, addinfo);
1010  }
1011 
1012  if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
1013  {
1014  return snippet_fetch(fi, elemsetname + 7,
1015  input_format, output_format,
1016  result, addinfo);
1017  }
1018 
1019  /* processing zebra::meta::sysno */
1020  if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
1021  {
1022  int ret = 0;
1023  if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1024  {
1025  wrbuf_printf(result, ZINT_FORMAT, fi->sysno);
1026  *output_format = input_format;
1027  }
1028  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1029  {
1030  wrbuf_printf(result, ZEBRA_XML_HEADER_STR
1031  " sysno=\"" ZINT_FORMAT "\"/>\n",
1032  fi->sysno);
1033  *output_format = input_format;
1034  }
1035  else
1036  ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1037  return ret;
1038  }
1039 
1040  /* processing special elementsetname zebra::index:: for sort elements */
1041  if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1042  {
1043  int ret = sort_fetch(
1044  fi, elemsetname + 5,
1045  input_format, output_format,
1046  result, addinfo);
1047  if (ret != -1)
1048  return ret;
1049  /* not a sort index so we continue to get the full record */
1050  }
1051 
1052 
1053  /* fetching binary record up for all other display elementsets */
1054  rec = rec_get(zh->reg->records, sysno);
1055  if (!rec)
1056  {
1057  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1058  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1059  }
1060 
1061  /* processing special elementsetnames zebra::data */
1062  if (elemsetname && 0 == strcmp(elemsetname, "data"))
1063  {
1064  struct ZebraRecStream stream;
1065  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1066  char *b;
1067 
1068  zebra_create_record_stream(zh, &rec, &stream);
1069  *output_format = input_format;
1070 
1071  b = nmem_malloc(fi->nmem, recordAttr->recordSize);
1072  stream.readf(&stream, b, recordAttr->recordSize);
1073  wrbuf_write(result, b, recordAttr->recordSize);
1074 
1075  stream.destroy(&stream);
1076  rec_free(&rec);
1077  return 0;
1078  }
1079 
1080  /* processing special elementsetnames zebra::meta:: */
1081  if (elemsetname && 0 == strcmp(elemsetname, "meta"))
1082  {
1083  int ret = 0;
1084  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1085 
1086  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1087  {
1088  *output_format = input_format;
1089 
1090  wrbuf_printf(result, ZEBRA_XML_HEADER_STR
1091  " sysno=\"" ZINT_FORMAT "\"", sysno);
1092  retrieve_puts_attr(result, "base", rec->info[recInfo_databaseName]);
1093  retrieve_puts_attr(result, "file", rec->info[recInfo_filename]);
1094  retrieve_puts_attr(result, "type", rec->info[recInfo_fileType]);
1095  if (fi->score >= 0)
1096  retrieve_puts_attr_int(result, "score", fi->score);
1097 
1098  wrbuf_printf(result,
1099  " rank=\"" ZINT_FORMAT "\""
1100  " size=\"%i\""
1101  " set=\"zebra::%s\"/>\n",
1102  recordAttr->staticrank,
1103  recordAttr->recordSize,
1104  elemsetname);
1105  }
1106  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1107  {
1108  *output_format = input_format;
1109  wrbuf_printf(result, "sysno " ZINT_FORMAT "\n", sysno);
1110  retrieve_puts_str(result, "base", rec->info[recInfo_databaseName]);
1111  retrieve_puts_str(result, "file", rec->info[recInfo_filename]);
1112  retrieve_puts_str(result, "type", rec->info[recInfo_fileType]);
1113  if (fi->score >= 0)
1114  retrieve_puts_int(result, "score", fi->score);
1115 
1116  wrbuf_printf(result,
1117  "rank " ZINT_FORMAT "\n"
1118  "size %i\n"
1119  "set zebra::%s\n",
1120  recordAttr->staticrank,
1121  recordAttr->recordSize,
1122  elemsetname);
1123  }
1124  else
1125  ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1126 
1127  rec_free(&rec);
1128  return ret;
1129  }
1130 
1131  /* processing special elementsetnames zebra::index:: */
1132  if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1133  {
1134  int ret = special_index_fetch(
1135  fi, elemsetname + 5,
1136  input_format, output_format,
1137  result, addinfo, rec);
1138  rec_free(&rec);
1139  return ret;
1140  }
1141 
1142  if (rec)
1143  rec_free(&rec);
1144  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1145 }
1146 
1148  zint sysno, int score,
1149  ODR odr,
1150  const Odr_oid *input_format, Z_RecordComposition *comp,
1151  const Odr_oid **output_format,
1152  char **rec_bufp, int *rec_lenp, char **basenamep,
1153  WRBUF addinfo_w)
1154 {
1155  Record rec;
1156  char *fname, *file_type, *basename;
1157  const char *elemsetname;
1158  struct ZebraRecStream stream;
1159  RecordAttr *recordAttr;
1160  void *clientData;
1161  int return_code = 0;
1162  zint sysnos[MAX_SYSNOS_PER_RECORD];
1163  int no_sysnos = MAX_SYSNOS_PER_RECORD;
1164  ZEBRA_RES res;
1165  struct special_fetch_s fetch_info;
1166 
1167  res = zebra_result_recid_to_sysno(zh, setname, sysno, sysnos, &no_sysnos);
1168  if (res != ZEBRA_OK)
1169  return ZEBRA_FAIL;
1170 
1171  sysno = sysnos[0];
1172  *basenamep = 0;
1173  elemsetname = yaz_get_esn(comp);
1174 
1175  fetch_info.zh = zh;
1176  fetch_info.setname = setname;
1177  fetch_info.sysno = sysno;
1178  fetch_info.score = score;
1179  fetch_info.nmem = odr->mem;
1180 
1181  /* processing zebra special elementset names of form 'zebra:: */
1182  if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
1183  {
1184  WRBUF result = wrbuf_alloc();
1185  int r = zebra_special_fetch(&fetch_info, elemsetname + 7,
1186  input_format, output_format,
1187  result, addinfo_w);
1188  if (r == 0)
1189  {
1190  *rec_bufp = odr_strdup(odr, wrbuf_cstr(result));
1191  *rec_lenp = wrbuf_len(result);
1192  }
1193  wrbuf_destroy(result);
1194  return r;
1195  }
1196 
1197  /* processing all other element set names */
1198  rec = rec_get(zh->reg->records, sysno);
1199  if (!rec)
1200  {
1201  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1202  *basenamep = 0;
1203  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1204  }
1205 
1206 
1207  recordAttr = rec_init_attr(zh->reg->zei, rec);
1208 
1209  file_type = rec->info[recInfo_fileType];
1210  fname = rec->info[recInfo_filename];
1211  basename = rec->info[recInfo_databaseName];
1212  *basenamep = (char *) odr_malloc(odr, strlen(basename)+1);
1213  strcpy(*basenamep, basename);
1214 
1215  yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
1216  sysno, score);
1217 
1218  return_code = zebra_create_record_stream(zh, &rec, &stream);
1219 
1220  if (rec)
1221  {
1222  RecType rt;
1223  struct recRetrieveCtrl retrieveCtrl;
1224 
1225  retrieveCtrl.stream = &stream;
1226  retrieveCtrl.fname = fname;
1227  retrieveCtrl.localno = sysno;
1228  retrieveCtrl.staticrank = recordAttr->staticrank;
1229  retrieveCtrl.score = score;
1230  retrieveCtrl.recordSize = recordAttr->recordSize;
1231  retrieveCtrl.odr = odr;
1232  retrieveCtrl.input_format = retrieveCtrl.output_format = input_format;
1233  retrieveCtrl.comp = comp;
1234  retrieveCtrl.encoding = zh->record_encoding;
1235  retrieveCtrl.diagnostic = 0;
1236  retrieveCtrl.addinfo = 0;
1237  retrieveCtrl.dh = zh->reg->dh;
1238  retrieveCtrl.res = zh->res;
1239  retrieveCtrl.rec_buf = 0;
1240  retrieveCtrl.rec_len = -1;
1241  retrieveCtrl.handle = &fetch_info;
1242  retrieveCtrl.special_fetch = zebra_special_fetch;
1243 
1244  if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
1245  file_type, &clientData)))
1246  {
1247  wrbuf_printf(addinfo_w, "Could not handle record type %.40s",
1248  file_type);
1249  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1250  }
1251  else
1252  {
1253  (*rt->retrieve)(clientData, &retrieveCtrl);
1254  return_code = retrieveCtrl.diagnostic;
1255 
1256  *output_format = retrieveCtrl.output_format;
1257  *rec_bufp = (char *) retrieveCtrl.rec_buf;
1258  *rec_lenp = retrieveCtrl.rec_len;
1259  if (retrieveCtrl.addinfo)
1260  wrbuf_puts(addinfo_w, retrieveCtrl.addinfo);
1261  }
1262 
1263  stream.destroy(&stream);
1264  rec_free(&rec);
1265  }
1266 
1267  return return_code;
1268 }
1269 
1270 /*
1271  * Local variables:
1272  * c-basic-offset: 4
1273  * c-file-style: "Stroustrup"
1274  * indent-tabs-mode: nil
1275  * End:
1276  * vim: shiftwidth=4 tabstop=8 expandtab
1277  */
1278 
char * addinfo
Definition: recctrl.h:138
char * record_encoding
Definition: index.h:213
void zebra_snippets_ring(zebra_snippets *doc, const zebra_snippets *hit, int before, int after)
Definition: snippet.c:237
Record rec_get(Records p, zint sysno)
gets record - with given system number
Definition: records.c:927
int len
Definition: it_key.h:31
ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh, const char *setname, zint recid, zint *sysnos, int *no_sysnos)
Definition: zsets.c:1481
static void retrieve_puts_str(WRBUF wrbuf, const char *name, const char *value)
Definition: retrieve.c:395
static int zebra_special_fetch(void *handle, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:993
off_t recordOffset
Definition: zinfo.h:107
#define ZEBRA_OK
Definition: util.h:82
RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
Definition: zsets.c:1075
struct rset_key_control * zebra_key_control_create(ZebraHandle zh)
Definition: kcontrol.c:57
zint ISAM_P
Definition: isamc.h:28
int zebra_term_untrans(ZebraHandle zh, const char *index_type, char *dst, const char *src)
Definition: untrans.c:31
int recordSize
Definition: zinfo.h:106
char * path_reg
Definition: index.h:182
struct rset rset
static void retrieve_puts_attr(WRBUF wrbuf, const char *name, const char *value)
Definition: retrieve.c:378
ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, zint sysno, zebra_snippets *snippets)
Definition: zsets.c:1340
struct zebra_snippet_word * next
Definition: snippet.h:34
void * zebra_strmap_lookup(zebra_strmap_t st, const char *name, int no, size_t *data_len)
Definition: strmap.c:99
void zebra_create_stream_mem(struct ZebraRecStream *stream, const char *buf, size_t sz)
Definition: stream.c:123
void zebra_snippets_log(const zebra_snippets *l, int log_level, int all)
Definition: snippet.c:104
static int sort_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:155
static int special_index_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo, Record rec)
Definition: retrieve.c:245
RSET rset_dup(RSET rs)
Duplicate an RSET.
Definition: rset.c:255
zint set_occur
Definition: retrieve.c:573
#define MAX_SYSNOS_PER_RECORD
Definition: retrieve.c:42
int zebra_sort_read(zebra_sort_index_t si, zint *section_id, WRBUF w)
reads sort entry
Definition: sortidx.c:469
char * encoding
Definition: recctrl.h:125
void zebra_count_set(ZebraHandle zh, RSET rset, zint *count, zint approx_limit)
Definition: zsets.c:1498
struct zebra_register * reg
Definition: index.h:174
static void snippet_check_fields(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc, const zebra_snippet_word *doc_w, const char *w_index_type)
Definition: retrieve.c:409
ZEBRA_RES res_get_int(Res r, const char *name, int *val)
Definition: res.c:432
#define ZEBRA_FAIL
Definition: util.h:81
static int facet_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:893
data1_handle dh
Definition: index.h:142
static struct term_collect * term_collect_create(zebra_strmap_t sm, int no_terms_collect, NMEM nmem)
Definition: retrieve.c:645
const zebra_snippet_word * zebra_snippets_constlist(const zebra_snippets *l)
Definition: snippet.c:99
static void retrieve_puts_int(WRBUF wrbuf, const char *name, const int value)
Definition: retrieve.c:402
Records records
Definition: index.h:138
int zebra_rec_keys_rewind(zebra_rec_keys_t keys)
Definition: reckeys.c:240
const char * index_name
Definition: retrieve.c:90
void rec_free(Record *recpp)
frees record (from memory)
Definition: records.c:1043
void zebra_snippets_destroy(zebra_snippets *l)
Definition: snippet.c:45
zint sysno
Definition: api.h:483
static int perform_facet(ZebraHandle zh, struct special_fetch_s *fi, WRBUF result, int num_recs, ZebraMetaRecord *poset, struct index_spec *spec_list, int no_ord, int *ord_array, int use_xml, zinfo_index_category_t cat)
Definition: retrieve.c:802
void extract_snippet(ZebraHandle zh, zebra_snippets *sn, struct ZebraRecStream *stream, RecType rt, void *recTypeClientData)
Definition: extract.c:334
int(* retrieve)(void *clientData, struct recRetrieveCtrl *ctrl)
Definition: recctrl.h:159
const char * zebra_strmap_it_next(zebra_strmap_it it, void **data_buf, size_t *data_len)
Definition: strmap.c:162
#define IT_MAX_WORD
Definition: it_key.h:27
void error(const char *format,...)
Definition: agrep.c:51
#define ZEBRA_XML_HEADER_STR
Definition: retrieve.c:44
void rset_delete(RSET rs)
Destructor RSETs.
Definition: rset.c:218
ZebraMetaRecord * zebra_meta_records_create(ZebraHandle zh, const char *name, int num, zint *positions)
Definition: zsets.c:418
int zebraExplain_lookup_attr_str(ZebraExplainInfo zei, zinfo_index_category_t cat, const char *index_type, const char *str)
lookup ordinal from string index + index type
Definition: zinfo.c:1354
zint staticrank
Definition: zinfo.h:109
size_t size[REC_NO_INFO]
Definition: recindex.h:35
static int perform_facet_index(ZebraHandle zh, struct special_fetch_s *fi, int no_ord, int *ord_array, zebra_strmap_t *map_array, int num_recs, ZebraMetaRecord *poset, struct index_spec *spec_list)
Definition: retrieve.c:731
static zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set)
Definition: retrieve.c:576
void zebra_strmap_it_destroy(zebra_strmap_it it)
Definition: strmap.c:157
ZebraExplainInfo zei
Definition: index.h:139
void zebra_rec_keys_close(zebra_rec_keys_t p)
Definition: reckeys.c:143
const Odr_oid * output_format
Definition: recctrl.h:134
#define CAST_ZINT_TO_INT(x)
Definition: util.h:96
RSET rset_create_and(NMEM nmem, struct rset_key_control *kcontrol, int scope, int no_rsets, RSET *rsets)
Definition: rsmultiandor.c:280
zebra_strmap_it zebra_strmap_it_create(zebra_strmap_t st)
Definition: strmap.c:148
zebra_rec_keys_t zebra_rec_keys_open(void)
Definition: reckeys.c:88
char * info[REC_NO_INFO]
Definition: recindex.h:34
int zebra_sort_type(zebra_sort_index_t si, int type)
sets type for sort usage
Definition: sortidx.c:234
#define FIRST_IN_FIELD_CHAR
Definition: index.h:416
int(* readf)(struct ZebraRecStream *s, char *buf, size_t count)
read function
Definition: recctrl.h:75
RSET zebra_create_rset_isam(ZebraHandle zh, NMEM rset_nmem, struct rset_key_control *kctl, int scope, ISAM_P pos, TERMID termid)
Definition: rset_isam.c:32
struct index_spec * parse_index_spec(const char *elem, NMEM nmem, int *error)
Definition: retrieve.c:97
const Odr_oid * input_format
Definition: recctrl.h:123
ZebraHandle zh
Definition: retrieve.c:47
zebra_sort_index_t sort_index
Definition: index.h:134
void(* dec)(struct rset_key_control *kc)
Definition: rset.h:138
char * dict_lookup(Dict dict, const char *p)
lookup item in dictionary
Definition: lookup.c:100
void zebra_sort_sysno(zebra_sort_index_t si, zint sysno)
sets sort system number for read / add / delete
Definition: sortidx.c:339
zint mem[IT_KEY_LEVEL_MAX]
Definition: it_key.h:32
struct index_spec * next
Definition: retrieve.c:93
static int perform_facet_sort(ZebraHandle zh, int no_ord, int *ord_array, zebra_strmap_t *map_array, int num_recs, ZebraMetaRecord *poset)
Definition: retrieve.c:687
const char * extra
Definition: retrieve.c:92
const char * setname
Definition: retrieve.c:48
Z_RecordComposition * comp
Definition: recctrl.h:124
zint staticrank
Definition: recctrl.h:128
void zebra_rec_keys_set_buf(zebra_rec_keys_t p, char *buf, size_t sz, int copy_buf)
Definition: reckeys.c:109
data1_handle dh
Definition: recctrl.h:131
Dict dict
Definition: index.h:132
RecTypes recTypes
Definition: index.h:145
zinfo_index_category_t
Definition: zinfo.h:37
int zebraExplain_lookup_ord(ZebraExplainInfo zei, int ord, const char **index_type, const char **db, const char **string_index)
Definition: zinfo.c:1479
struct ZebraRecStream * stream
Definition: recctrl.h:119
int key_SU_encode(int ch, char *out)
Definition: su_codec.c:31
zebra_strmap_t zebra_strmap_create(void)
Definition: strmap.c:45
void zebra_strmap_add(zebra_strmap_t st, const char *name, void *data_buf, size_t data_len)
Definition: strmap.c:80
int zebra_get_rec_snippets(ZebraHandle zh, zint sysno, zebra_snippets *snippets)
Definition: retrieve.c:490
static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
Definition: retrieve.c:444
void * handle
Definition: recctrl.h:141
RecType recType_byName(RecTypes rts, Res res, const char *name, void **clientDataP)
Definition: recctrl.c:264
static int term_qsort_handle(const void *a, const void *b)
Definition: retrieve.c:609
long zint
Zebra integer.
Definition: util.h:66
const char * term
Definition: retrieve.c:571
void(* destroy)(struct ZebraRecStream *s)
close and destroy stream
Definition: recctrl.h:83
int zebra_rec_keys_read(zebra_rec_keys_t keys, const char **str, size_t *slen, struct it_key *key)
Definition: reckeys.c:259
void * rec_buf
Definition: recctrl.h:135
Definition: rset.h:150
int zebra_record_fetch(ZebraHandle zh, const char *setname, zint sysno, int score, ODR odr, const Odr_oid *input_format, Z_RecordComposition *comp, const Odr_oid **output_format, char **rec_bufp, int *rec_lenp, char **basenamep, WRBUF addinfo_w)
Definition: retrieve.c:1147
zebra_snippets * zebra_snippets_create(void)
Definition: snippet.c:36
int fd
Definition: tstlockscope.c:38
#define O_BINARY
Definition: agrep.c:46
char * fname
Definition: recctrl.h:130
RecordAttr * rec_init_attr(ZebraExplainInfo zei, Record rec)
Definition: zinfo.c:1595
Definition: it_key.h:30
short ZEBRA_RES
Common return type for Zebra API.
Definition: util.h:80
const char * index_type
Definition: retrieve.c:91
void zebra_strmap_destroy(zebra_strmap_t st)
Definition: strmap.c:61
static int snippet_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:526
int zebra_rec_keys_empty(zebra_rec_keys_t keys)
Definition: reckeys.c:252
zint approx_limit
Definition: index.h:180
void zebra_create_stream_fd(struct ZebraRecStream *stream, int fd, off_t start_offset)
Definition: stream.c:140
static void term_collect_freq(ZebraHandle zh, struct term_collect *col, int no_terms_collect, int ord, RSET rset, double scale_factor)
Definition: retrieve.c:625
static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name, const int value)
Definition: retrieve.c:389
record reader stream
Definition: recctrl.h:71
int(* special_fetch)(void *handle, const char *esn, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: recctrl.h:142
static int zebra_create_record_stream(ZebraHandle zh, Record *rec, struct ZebraRecStream *stream)
Definition: retrieve.c:54
void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, int num)
Definition: zsets.c:547
#define ZINT_FORMAT
Definition: util.h:72