IDZEBRA  2.1.2
kinput.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #include <fcntl.h>
24 #ifdef WIN32
25 #include <io.h>
26 #endif
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #include <stdlib.h>
31 #include <string.h>
32 #include <stdio.h>
33 #include <assert.h>
34 
35 #include "index.h"
36 
37 #define KEY_SIZE (1+sizeof(struct it_key))
38 #define INP_NAME_MAX 768
39 
40 struct key_file {
41  int no; /* file no */
42  off_t offset; /* file offset */
43  unsigned char *buf; /* buffer block */
44  size_t buf_size; /* number of read bytes in block */
45  size_t chunk; /* number of bytes allocated */
46  size_t buf_ptr; /* current position in buffer */
47  char *prev_name; /* last word read */
49  off_t length; /* length of file */
50  /* handler invoked in each read */
51  void (*readHandler)(struct key_file *keyp, void *rinfo);
52  void *readInfo;
54 };
55 
56 
57 #define PR_KEY_LOW 0
58 #define PR_KEY_TOP 0
59 
60 #if PR_KEY_LOW || PR_KEY_TOP
61 
62 static void pkey(const char *b, int mode)
63 {
64  key_logdump_txt(YLOG_LOG, b, mode ? "i" : "d");
65 }
66 #endif
67 
68 
69 void getFnameTmp(Res res, char *fname, int no)
70 {
71  const char *pre;
72 
73  pre = res_get_def(res, "keyTmpDir", ".");
74  sprintf(fname, "%s/key%d.tmp", pre, no);
75 }
76 
77 void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no)
78 {
79  const char *pre;
80 
81  pre = res_get_def(zh->res, "keyTmpDir", ".");
82  sprintf(fname, "%s/key%d.tmp", pre, no);
83 }
84 
86 {
87  int nr = 0, r = 0, fd;
88  char fname[1024];
89  getFnameTmp(f->res, fname, f->no);
90  fd = open(fname, O_BINARY|O_RDONLY);
91 
92  f->buf_ptr = 0;
93  f->buf_size = 0;
94  if (fd == -1)
95  {
96  yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot open %s", fname);
97  return ;
98  }
99  if (!f->length)
100  {
101  if ((f->length = lseek(fd, 0L, SEEK_END)) == (off_t) -1)
102  {
103  yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
104  close(fd);
105  return ;
106  }
107  }
108  if (lseek(fd, f->offset, SEEK_SET) == -1)
109  {
110  yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
111  close(fd);
112  return ;
113  }
114  while (f->chunk - nr > 0)
115  {
116  r = read(fd, f->buf + nr, f->chunk - nr);
117  if (r <= 0)
118  break;
119  nr += r;
120  }
121  if (r == -1)
122  {
123  yaz_log(YLOG_WARN|YLOG_ERRNO, "read of %s", fname);
124  close(fd);
125  return;
126  }
127  f->buf_size = nr;
128  if (f->readHandler)
129  (*f->readHandler)(f, f->readInfo);
130  close(fd);
131 }
132 
133 void key_file_destroy(struct key_file *f)
134 {
136  xfree(f->buf);
137  xfree(f->prev_name);
138  xfree(f);
139 }
140 
141 struct key_file *key_file_init(int no, int chunk, Res res)
142 {
143  struct key_file *f;
144 
145  f = (struct key_file *) xmalloc(sizeof(*f));
146  f->res = res;
147  f->decode_handle = iscz1_start();
148  f->no = no;
149  f->chunk = chunk;
150  f->offset = 0;
151  f->length = 0;
152  f->readHandler = NULL;
153  f->buf = (unsigned char *) xmalloc(f->chunk);
154  f->prev_name = (char *) xmalloc(INP_NAME_MAX);
155  *f->prev_name = '\0';
157  return f;
158 }
159 
160 int key_file_getc(struct key_file *f)
161 {
162  if (f->buf_ptr < f->buf_size)
163  return f->buf[(f->buf_ptr)++];
164  if (f->buf_size < f->chunk)
165  return EOF;
166  f->offset += f->buf_size;
168  if (f->buf_ptr < f->buf_size)
169  return f->buf[(f->buf_ptr)++];
170  else
171  return EOF;
172 }
173 
174 int key_file_read(struct key_file *f, char *key)
175 {
176  int i, c;
177  char srcbuf[128];
178  const char *src = srcbuf;
179  char *dst;
180  int j;
181 
182  c = key_file_getc(f);
183  if (c == 0)
184  {
185  strcpy(key, f->prev_name);
186  i = 1+strlen(key);
187  }
188  else if (c == EOF)
189  return 0;
190  else
191  {
192  i = 0;
193  key[i++] = c;
194  while ((c = key_file_getc(f)))
195  {
196  if (i < INP_NAME_MAX-2)
197  key[i++] = c;
198  }
199  key[i++] = '\0';
200  strcpy(f->prev_name, key);
202  }
203  c = key_file_getc(f); /* length + insert/delete combined */
204  key[i++] = c & 128;
205  c = c & 127;
206  for (j = 0; j < c; j++)
207  srcbuf[j] = key_file_getc(f);
208  dst = key + i;
209  iscz1_decode(f->decode_handle, &dst, &src);
210 
211 #if 0
212  /* debugging */
213  if (1)
214  {
215  struct it_key k;
216  memcpy(&k, key+i, sizeof(k));
217  if (!k.mem[1])
218  yaz_log(YLOG_LOG, "00 KEY");
219  }
220 #endif
221  return i + sizeof(struct it_key);
222 }
223 
224 struct heap_info {
225  struct {
226  struct key_file **file;
227  char **buf;
228  } info;
229  int heapnum;
230  int *ptr;
231  int (*cmp)(const void *p1, const void *p2);
239 };
240 
241 static struct heap_info *key_heap_malloc(void)
242 { /* malloc and clear it */
243  struct heap_info *hi;
244  hi = (struct heap_info *) xmalloc(sizeof(*hi));
245  hi->info.file = 0;
246  hi->info.buf = 0;
247  hi->heapnum = 0;
248  hi->ptr = 0;
249  hi->no_diffs = 0;
250  hi->no_diffs = 0;
251  hi->no_updates = 0;
252  hi->no_deletions = 0;
253  hi->no_insertions = 0;
254  hi->no_iterations = 0;
255  return hi;
256 }
257 
259  int nkeys,
260  int (*cmp)(const void *p1, const void *p2))
261 {
262  struct heap_info *hi;
263  int i;
264 
265  hi = key_heap_malloc();
266  hi->zh = zh;
267  hi->info.file = (struct key_file **)
268  xmalloc(sizeof(*hi->info.file) * (1+nkeys));
269  hi->info.buf = (char **) xmalloc(sizeof(*hi->info.buf) * (1+nkeys));
270  hi->ptr = (int *) xmalloc(sizeof(*hi->ptr) * (1+nkeys));
271  hi->cmp = cmp;
272  for (i = 0; i<= nkeys; i++)
273  {
274  hi->ptr[i] = i;
275  hi->info.buf[i] = (char *) xmalloc(INP_NAME_MAX);
276  }
277  return hi;
278 }
279 
280 void key_heap_destroy(struct heap_info *hi, int nkeys)
281 {
282  int i;
283  for (i = 0; i<=nkeys; i++)
284  xfree(hi->info.buf[i]);
285  xfree(hi->info.buf);
286  xfree(hi->ptr);
287  xfree(hi->info.file);
288  xfree(hi);
289 }
290 
291 static void key_heap_swap(struct heap_info *hi, int i1, int i2)
292 {
293  int swap;
294 
295  swap = hi->ptr[i1];
296  hi->ptr[i1] = hi->ptr[i2];
297  hi->ptr[i2] = swap;
298 }
299 
300 
301 static void key_heap_delete(struct heap_info *hi)
302 {
303  int cur = 1, child = 2;
304 
305  assert(hi->heapnum > 0);
306 
307  key_heap_swap(hi, 1, hi->heapnum);
308  hi->heapnum--;
309  while (child <= hi->heapnum) {
310  if (child < hi->heapnum &&
311  (*hi->cmp)(&hi->info.buf[hi->ptr[child]],
312  &hi->info.buf[hi->ptr[child+1]]) > 0)
313  child++;
314  if ((*hi->cmp)(&hi->info.buf[hi->ptr[cur]],
315  &hi->info.buf[hi->ptr[child]]) > 0)
316  {
317  key_heap_swap(hi, cur, child);
318  cur = child;
319  child = 2*cur;
320  }
321  else
322  break;
323  }
324 }
325 
326 static void key_heap_insert(struct heap_info *hi, const char *buf, int nbytes,
327  struct key_file *kf)
328 {
329  int cur, parent;
330 
331  assert(nbytes < INP_NAME_MAX);
332 
333  cur = ++(hi->heapnum);
334  memcpy(hi->info.buf[hi->ptr[cur]], buf, nbytes);
335  hi->info.file[hi->ptr[cur]] = kf;
336 
337  parent = cur/2;
338  while (parent && (*hi->cmp)(&hi->info.buf[hi->ptr[parent]],
339  &hi->info.buf[hi->ptr[cur]]) > 0)
340  {
341  key_heap_swap(hi, cur, parent);
342  cur = parent;
343  parent = cur/2;
344  }
345 }
346 
347 static int heap_read_one(struct heap_info *hi, char *name, char *key)
348 {
349  int n, r;
350  char rbuf[INP_NAME_MAX];
351  struct key_file *kf;
352 
353  if (!hi->heapnum)
354  return 0;
355  n = hi->ptr[1];
356  strcpy(name, hi->info.buf[n]);
357  kf = hi->info.file[n];
358  r = strlen(name);
359  memcpy(key, hi->info.buf[n] + r+1, KEY_SIZE);
360  key_heap_delete(hi);
361  if ((r = key_file_read(kf, rbuf)))
362  key_heap_insert(hi, rbuf, r, kf);
363  hi->no_iterations++;
364  return 1;
365 }
366 
367 
368 /* for debugging only */
369 void zebra_log_dict_entry(ZebraHandle zh, const char *s)
370 {
371  char dst[INP_NAME_MAX+1];
372  int ord;
373  int len = key_SU_decode(&ord, (const unsigned char *) s);
374  const char *index_type;
375 
376  if (!zh)
377  yaz_log(YLOG_LOG, "ord=%d", ord);
378  else
379  {
380  const char *string_index;
381  const char *db = 0;
383  ord, &index_type, &db, &string_index);
384 
385  zebra_term_untrans(zh, index_type, dst, s + len);
386 
387  yaz_log(YLOG_LOG, "ord=%d index_type=%s index=%s term=%s",
388  ord, index_type, string_index, dst);
389  }
390 }
391 
394  char cur_name[INP_NAME_MAX];
395  char *key;
396  char *key_1, *key_2;
397  int mode_1, mode_2;
398  int sz_1, sz_2;
399  struct heap_info *hi;
401  int more;
402  int ret;
404 };
405 
406 static int heap_cread_item(void *vp, char **dst, int *insertMode);
407 
408 int heap_cread_item2(void *vp, char **dst, int *insertMode)
409 {
410  struct heap_cread_info *p = (struct heap_cread_info *) vp;
411  int level = 0;
412 
413  if (p->look_level)
414  {
415  if (p->look_level > 0)
416  {
417  *insertMode = 1;
418  p->look_level--;
419  }
420  else
421  {
422  *insertMode = 0;
423  p->look_level++;
424  }
425  memcpy(*dst, p->key_1, p->sz_1);
426 #if PR_KEY_TOP
427  yaz_log(YLOG_LOG, "DUP level=%d", p->look_level);
428  pkey(*dst, *insertMode);
429 #endif
430  (*dst) += p->sz_1;
431  return 1;
432  }
433  if (p->ret == 0) /* lookahead was 0?. Return that in read next round */
434  {
435  p->ret = -1;
436  return 0;
437  }
438  else if (p->ret == -1) /* Must read new item ? */
439  {
440  char *dst_1 = p->key_1;
441  p->ret = heap_cread_item(vp, &dst_1, &p->mode_1);
442  p->sz_1 = dst_1 - p->key_1;
443  }
444  else
445  { /* lookahead in 2 . Now in 1. */
446  p->sz_1 = p->sz_2;
447  p->mode_1 = p->mode_2;
448  memcpy(p->key_1, p->key_2, p->sz_2);
449  }
450  if (p->mode_1)
451  level = 1; /* insert */
452  else
453  level = -1; /* delete */
454  while(1)
455  {
456  char *dst_2 = p->key_2;
457  p->ret = heap_cread_item(vp, &dst_2, &p->mode_2);
458  if (!p->ret)
459  {
460  if (level)
461  break;
462  p->ret = -1;
463  return 0;
464  }
465  p->sz_2 = dst_2 - p->key_2;
466 
467  if (key_compare(p->key_1, p->key_2) == 0)
468  {
469  if (p->mode_2) /* adjust level according to deletes/inserts */
470  level++;
471  else
472  level--;
473  }
474  else
475  {
476  if (level)
477  break;
478  /* all the same. new round .. */
479  p->sz_1 = p->sz_2;
480  p->mode_1 = p->mode_2;
481  memcpy(p->key_1, p->key_2, p->sz_1);
482  if (p->mode_1)
483  level = 1; /* insert */
484  else
485  level = -1; /* delete */
486  }
487  }
488  /* outcome is insert (1) or delete (0) depending on final level */
489  if (level > 0)
490  {
491  *insertMode = 1;
492  level--;
493  }
494  else
495  {
496  *insertMode = 0;
497  level++;
498  }
499  p->look_level = level;
500  memcpy(*dst, p->key_1, p->sz_1);
501 #if PR_KEY_TOP
502  yaz_log(YLOG_LOG, "TOP");
503  pkey(*dst, *insertMode);
504 #endif
505  (*dst) += p->sz_1;
506  return 1;
507 }
508 
509 int heap_cread_item(void *vp, char **dst, int *insertMode)
510 {
511  struct heap_cread_info *p = (struct heap_cread_info *) vp;
512  struct heap_info *hi = p->hi;
513 
514  if (p->first_in_list)
515  {
516  *insertMode = p->key[0];
517  memcpy(*dst, p->key+1, sizeof(struct it_key));
518 #if PR_KEY_LOW
520  pkey(*dst, *insertMode);
521 #endif
522  (*dst) += sizeof(struct it_key);
523  p->first_in_list = 0;
524  return 1;
525  }
526  strcpy(p->prev_name, p->cur_name);
527  if (!(p->more = heap_read_one(hi, p->cur_name, p->key)))
528  return 0;
529  if (*p->cur_name && strcmp(p->cur_name, p->prev_name))
530  {
531  p->first_in_list = 1;
532  return 0;
533  }
534  *insertMode = p->key[0];
535  memcpy(*dst, p->key+1, sizeof(struct it_key));
536 #if PR_KEY_LOW
538  pkey(*dst, *insertMode);
539 #endif
540  (*dst) += sizeof(struct it_key);
541  return 1;
542 }
543 
544 int heap_inpc(struct heap_cread_info *hci, struct heap_info *hi)
545 {
546  ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
547 
548  isamc_i->clientData = hci;
549  isamc_i->read_item = heap_cread_item2;
550 
551  while (hci->more)
552  {
553  char this_name[INP_NAME_MAX];
554  ISAM_P isamc_p, isamc_p2;
555  char *dict_info;
556 
557  strcpy(this_name, hci->cur_name);
558  assert(hci->cur_name[0]);
559  hi->no_diffs++;
560  if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
561  {
562  memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
563  isamc_p2 = isamc_p;
564  isamc_merge(hi->reg->isamc, &isamc_p2, isamc_i);
565  if (!isamc_p2)
566  {
567  hi->no_deletions++;
568  if (!dict_delete(hi->reg->dict, this_name))
569  abort();
570  }
571  else
572  {
573  hi->no_updates++;
574  if (isamc_p2 != isamc_p)
575  dict_insert(hi->reg->dict, this_name,
576  sizeof(ISAM_P), &isamc_p2);
577  }
578  }
579  else
580  {
581  isamc_p = 0;
582  isamc_merge(hi->reg->isamc, &isamc_p, isamc_i);
583  hi->no_insertions++;
584  if (isamc_p)
585  dict_insert(hi->reg->dict, this_name,
586  sizeof(ISAM_P), &isamc_p);
587  }
588  }
589  xfree(isamc_i);
590  return 0;
591 }
592 
593 int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi)
594 {
595  ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
596 
597  isamc_i->clientData = hci;
598  isamc_i->read_item = heap_cread_item2;
599 
600  while (hci->more)
601  {
602  char this_name[INP_NAME_MAX];
603  ISAM_P isamc_p, isamc_p2;
604  char *dict_info;
605 
606  strcpy(this_name, hci->cur_name);
607  assert(hci->cur_name[0]);
608  hi->no_diffs++;
609 
610 #if 0
611  assert(hi->zh);
612  zebra_log_dict_entry(hi->zh, hci->cur_name);
613 #endif
614  if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
615  {
616  memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
617  isamc_p2 = isamc_p;
618  isamb_merge(hi->reg->isamb, &isamc_p2, isamc_i);
619  if (!isamc_p2)
620  {
621  hi->no_deletions++;
622  if (!dict_delete(hi->reg->dict, this_name))
623  abort();
624  }
625  else
626  {
627  hi->no_updates++;
628  if (isamc_p2 != isamc_p)
629  dict_insert(hi->reg->dict, this_name,
630  sizeof(ISAM_P), &isamc_p2);
631  }
632  }
633  else
634  {
635  isamc_p = 0;
636  isamb_merge(hi->reg->isamb, &isamc_p, isamc_i);
637  hi->no_insertions++;
638  if (isamc_p)
639  dict_insert(hi->reg->dict, this_name,
640  sizeof(ISAM_P), &isamc_p);
641  }
642  }
643  xfree(isamc_i);
644  return 0;
645 }
646 
647 int heap_inps(struct heap_cread_info *hci, struct heap_info *hi)
648 {
649  ISAMS_I isams_i = (ISAMS_I) xmalloc(sizeof(*isams_i));
650 
651  isams_i->clientData = hci;
652  isams_i->read_item = heap_cread_item;
653 
654  while (hci->more)
655  {
656  char this_name[INP_NAME_MAX];
657  ISAM_P isams_p;
658  char *dict_info;
659 
660  strcpy(this_name, hci->cur_name);
661  assert(hci->cur_name[0]);
662  hi->no_diffs++;
663  if (!(dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
664  {
665  isams_p = isams_merge(hi->reg->isams, isams_i);
666  hi->no_insertions++;
667  dict_insert(hi->reg->dict, this_name, sizeof(ISAM_P), &isams_p);
668  }
669  else
670  {
671  yaz_log(YLOG_FATAL, "isams doesn't support this kind of update");
672  break;
673  }
674  }
675  xfree(isams_i);
676  return 0;
677 }
678 
679 struct progressInfo {
680  time_t startTime;
681  time_t lastTime;
682  off_t totalBytes;
683  off_t totalOffset;
684 };
685 
686 void progressFunc(struct key_file *keyp, void *info)
687 {
688  struct progressInfo *p = (struct progressInfo *) info;
689  time_t now, remaining;
690 
691  if (keyp->buf_size <= 0 || p->totalBytes <= 0)
692  return ;
693  time (&now);
694 
695  if (now >= p->lastTime+10)
696  {
697  p->lastTime = now;
698  remaining = (time_t) ((now - p->startTime)*
699  ((double) p->totalBytes/p->totalOffset - 1.0));
700  if (remaining <= 130)
701  yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining",
702  (100.0*p->totalOffset) / p->totalBytes, (long) remaining);
703  else
704  yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining",
705  (100.0*p->totalOffset) / p->totalBytes, (long) remaining/60);
706  }
707  p->totalOffset += keyp->buf_size;
708 }
709 
710 #ifndef R_OK
711 #define R_OK 4
712 #endif
713 
715 {
716  struct key_file **kf = 0;
717  char rbuf[1024];
718  int i, r;
719  struct heap_info *hi;
720  struct progressInfo progressInfo;
721  int nkeys = key_block_get_no_files(zh->reg->key_block);
722 
723  if (nkeys == 0)
724  return;
725 
726  if (nkeys < 0)
727  {
728  char fname[1024];
729  nkeys = 0;
730  while (1)
731  {
732  extract_get_fname_tmp (zh, fname, nkeys+1);
733  if (access(fname, R_OK) == -1)
734  break;
735  nkeys++;
736  }
737  if (!nkeys)
738  return ;
739  }
740  kf = (struct key_file **) xmalloc((1+nkeys) * sizeof(*kf));
741  progressInfo.totalBytes = 0;
742  progressInfo.totalOffset = 0;
743  time(&progressInfo.startTime);
744  time(&progressInfo.lastTime);
745  for (i = 1; i<=nkeys; i++)
746  {
747  kf[i] = key_file_init(i, 8192, zh->res);
748  kf[i]->readHandler = progressFunc;
749  kf[i]->readInfo = &progressInfo;
750  progressInfo.totalBytes += kf[i]->length;
751  progressInfo.totalOffset += kf[i]->buf_size;
752  }
753  hi = key_heap_init_file(zh, nkeys, key_qsort_compare);
754  hi->reg = zh->reg;
755 
756  for (i = 1; i<=nkeys; i++)
757  if ((r = key_file_read(kf[i], rbuf)))
758  key_heap_insert(hi, rbuf, r, kf[i]);
759 
760  if (1)
761  {
762  struct heap_cread_info hci;
763 
764  hci.key = (char *) xmalloc(KEY_SIZE);
765  hci.key_1 = (char *) xmalloc(KEY_SIZE);
766  hci.key_2 = (char *) xmalloc(KEY_SIZE);
767  hci.ret = -1;
768  hci.first_in_list = 1;
769  hci.hi = hi;
770  hci.look_level = 0;
771  hci.more = heap_read_one(hi, hci.cur_name, hci.key);
772 
773  if (zh->reg->isams)
774  heap_inps(&hci, hi);
775  if (zh->reg->isamc)
776  heap_inpc(&hci, hi);
777  if (zh->reg->isamb)
778  heap_inpb(&hci, hi);
779 
780  xfree(hci.key);
781  xfree(hci.key_1);
782  xfree(hci.key_2);
783  }
784 
785  for (i = 1; i<=nkeys; i++)
786  {
787  extract_get_fname_tmp (zh, rbuf, i);
788  unlink(rbuf);
789  }
790  for (i = 1; i<=nkeys; i++)
791  key_file_destroy(kf[i]);
792  xfree(kf);
793  if (hi->no_iterations)
794  { /* do not log if nothing happened */
795  yaz_log(YLOG_LOG, "Iterations: isam/dict "
797  hi->no_iterations, hi->no_diffs);
798  yaz_log(YLOG_LOG, "Dict: inserts/updates/deletions: "
799  ZINT_FORMAT "/" ZINT_FORMAT "/" ZINT_FORMAT,
800  hi->no_insertions, hi->no_updates, hi->no_deletions);
801  }
803  key_heap_destroy(hi, nkeys);
804 }
805 /*
806  * Local variables:
807  * c-basic-offset: 4
808  * c-file-style: "Stroustrup"
809  * indent-tabs-mode: nil
810  * End:
811  * vim: shiftwidth=4 tabstop=8 expandtab
812  */
813 
off_t totalBytes
Definition: kinput.c:682
char * key_1
Definition: kinput.c:396
int key_compare(const void *p1, const void *p2)
Definition: it_key.c:73
int heap_inpc(struct heap_cread_info *hci, struct heap_info *hi)
Definition: kinput.c:544
struct heap_info * hi
Definition: kinput.c:399
void progressFunc(struct key_file *keyp, void *info)
Definition: kinput.c:686
const char * res_get_def(Res r, const char *name, const char *def)
Definition: res.c:313
void(* readHandler)(struct key_file *keyp, void *rinfo)
Definition: kinput.c:51
void * iscz1_start(void)
Definition: it_key.c:129
void iscz1_reset(void *vp)
Definition: it_key.c:145
zint ISAM_P
Definition: isamc.h:28
int zebra_term_untrans(ZebraHandle zh, const char *index_type, char *dst, const char *src)
Definition: untrans.c:31
void iscz1_stop(void *p)
Definition: it_key.c:154
int(* read_item)(void *clientData, char **dst, int *insertMode)
Definition: isamc.h:53
zint no_iterations
Definition: kinput.c:238
struct ISAMS_I_s * ISAMS_I
struct heap_info * key_heap_init_file(ZebraHandle zh, int nkeys, int(*cmp)(const void *p1, const void *p2))
Definition: kinput.c:258
void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no)
Definition: kinput.c:77
size_t chunk
Definition: kinput.c:45
static int heap_read_one(struct heap_info *hi, char *name, char *key)
Definition: kinput.c:347
off_t length
Definition: kinput.c:49
char cur_name[INP_NAME_MAX]
Definition: kinput.c:394
static void key_heap_insert(struct heap_info *hi, const char *buf, int nbytes, struct key_file *kf)
Definition: kinput.c:326
struct zebra_register * reg
Definition: index.h:174
size_t buf_ptr
Definition: kinput.c:46
#define KEY_SIZE
Definition: kinput.c:37
ISAMB isamb
Definition: index.h:131
ZebraHandle zh
Definition: zebrash.c:64
struct key_file ** file
Definition: kinput.c:226
zint no_diffs
Definition: kinput.c:234
void * readInfo
Definition: kinput.c:52
int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi)
Definition: kinput.c:593
time_t lastTime
Definition: kinput.c:681
void zebra_log_dict_entry(ZebraHandle zh, const char *s)
Definition: kinput.c:369
void getFnameTmp(Res res, char *fname, int no)
Definition: kinput.c:69
int look_level
Definition: kinput.c:403
#define R_OK
Definition: kinput.c:711
void * decode_handle
Definition: kinput.c:48
int dict_insert(Dict dict, const char *p, int userlen, void *userinfo)
insert item into dictionary
Definition: insert.c:438
int(* cmp)(const void *p1, const void *p2)
Definition: kinput.c:231
char * prev_name
Definition: kinput.c:47
int heapnum
Definition: kinput.c:229
void key_heap_destroy(struct heap_info *hi, int nkeys)
Definition: kinput.c:280
ZebraExplainInfo zei
Definition: index.h:139
Definition: res.c:46
static int heap_cread_item(void *vp, char **dst, int *insertMode)
Definition: kinput.c:509
zebra_key_block_t key_block
Definition: index.h:153
int heap_inps(struct heap_cread_info *hci, struct heap_info *hi)
Definition: kinput.c:647
off_t offset
Definition: kinput.c:42
int key_SU_decode(int *ch, const unsigned char *out)
Definition: su_codec.c:64
int key_qsort_compare(const void *p1, const void *p2)
Definition: it_key.c:110
void * clientData
Definition: isams.h:45
int key_file_getc(struct key_file *f)
Definition: kinput.c:160
int(* read_item)(void *clientData, char **dst, int *insertMode)
Definition: isams.h:44
zint no_deletions
Definition: kinput.c:236
size_t buf_size
Definition: kinput.c:44
ZebraHandle zh
Definition: kinput.c:233
static struct heap_info * key_heap_malloc(void)
Definition: kinput.c:241
char prev_name[INP_NAME_MAX]
Definition: kinput.c:393
char * key
Definition: kinput.c:395
time_t startTime
Definition: kinput.c:680
char ** buf
Definition: kinput.c:227
void isamb_merge(ISAMB b, ISAM_P *pos, ISAMC_I *data)
Definition: isamb.c:1265
char * dict_lookup(Dict dict, const char *p)
lookup item in dictionary
Definition: lookup.c:100
zint mem[IT_KEY_LEVEL_MAX]
Definition: it_key.h:32
static void key_heap_delete(struct heap_info *hi)
Definition: kinput.c:301
Res res
Definition: kinput.c:53
void key_file_chunk_read(struct key_file *f)
Definition: kinput.c:85
ISAMS isams
Definition: index.h:129
char * key_2
Definition: kinput.c:396
Dict dict
Definition: index.h:132
zint no_insertions
Definition: kinput.c:237
int zebraExplain_lookup_ord(ZebraExplainInfo zei, int ord, const char **index_type, const char **db, const char **string_index)
Definition: zinfo.c:1479
int * ptr
Definition: kinput.c:230
int key_file_read(struct key_file *f, char *key)
Definition: kinput.c:174
unsigned char * buf
Definition: kinput.c:43
void zebra_index_merge(ZebraHandle zh)
Definition: kinput.c:714
long zint
Zebra integer.
Definition: util.h:66
struct key_file * key_file_init(int no, int chunk, Res res)
Definition: kinput.c:141
ISAMC isamc
Definition: index.h:130
void key_block_destroy(zebra_key_block_t *pp)
Definition: key_block.c:225
void isamc_merge(ISAMC is, ISAM_P *pos, ISAMC_I *data)
Definition: merge.c:154
#define INP_NAME_MAX
Definition: kinput.c:38
struct zebra_register * reg
Definition: kinput.c:232
int no
Definition: kinput.c:41
off_t totalOffset
Definition: kinput.c:683
void key_file_destroy(struct key_file *f)
Definition: kinput.c:133
int fd
Definition: tstlockscope.c:38
int first_in_list
Definition: kinput.c:400
#define O_BINARY
Definition: agrep.c:46
Definition: it_key.h:30
void * clientData
Definition: isamc.h:54
zint no_updates
Definition: kinput.c:235
void key_logdump_txt(int logmask, const void *p, const char *txt)
Definition: it_key.c:37
void iscz1_decode(void *vp, char **dst, const char **src)
Definition: it_key.c:237
static void key_heap_swap(struct heap_info *hi, int i1, int i2)
Definition: kinput.c:291
int heap_cread_item2(void *vp, char **dst, int *insertMode)
Definition: kinput.c:408
ISAM_P isams_merge(ISAMS is, ISAMS_I data)
Definition: isams.c:115
int dict_delete(Dict dict, const char *p)
deletes item from dictionary
Definition: delete.c:260
struct heap_info::@14 info
#define ZINT_FORMAT
Definition: util.h:72
int key_block_get_no_files(zebra_key_block_t p)
Definition: key_block.c:409