IDZEBRA  2.2.7
tstisamb.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #if HAVE_SYS_TIMES_H
24 #include <sys/times.h>
25 #endif
26 #if HAVE_SYS_TIME_H
27 #include <sys/time.h>
28 #endif
29 
30 #include <stdlib.h>
31 #include <string.h>
32 #include <yaz/log.h>
33 #include <yaz/xmalloc.h>
34 #include <idzebra/isamb.h>
35 #include <assert.h>
36 
37 static int log_level = 0;
38 
39 static void log_item(int level, const void *b, const char *txt)
40 {
41  int x;
42  memcpy(&x, b, sizeof(int));
43  yaz_log(log_level, "%s %d", txt, x);
44 }
45 
46 static void log_pr(const char *txt)
47 {
48  yaz_log(log_level, "%s", txt);
49 }
50 
51 int compare_item(const void *a, const void *b)
52 {
53  int ia, ib;
54 
55  memcpy(&ia, a, sizeof(int));
56  memcpy(&ib, b, sizeof(int));
57  if (ia > ib)
58  return 1;
59  if (ia < ib)
60  return -1;
61  return 0;
62 }
63 
64 void *code_start(void)
65 {
66  return 0;
67 }
68 
69 void code_item(void *p, char **dst, const char **src)
70 {
71  memcpy (*dst, *src, sizeof(int));
72  (*dst) += sizeof(int);
73  (*src) += sizeof(int);
74 }
75 
76 void code_reset(void *p)
77 {
78 }
79 void code_stop(void *p)
80 {
81 }
82 
83 struct read_info {
84  int val;
85  int step;
86 
87  int no;
88  int max;
89  int insertMode;
90 };
91 
92 int code_read(void *vp, char **dst, int *insertMode)
93 {
94  struct read_info *ri = (struct read_info *)vp;
95  int x;
96 
97  if (ri->no >= ri->max)
98  return 0;
99  ri->no++;
100 
101  x = ri->val;
102  memcpy (*dst, &x, sizeof(int));
103  (*dst)+=sizeof(int);
104 
105  ri->val = ri->val + ri->step;
106  *insertMode = ri->insertMode;
107 
108 #if 0
109  yaz_log(log_level, "%d %5d", ri->insertMode, x);
110 #endif
111  return 1;
112 }
113 
114 void tst_insert(ISAMB isb, int n)
115 {
116  ISAMC_I isamc_i;
117  ISAM_P isamc_p;
118  struct read_info ri;
119  ISAMB_PP pp;
120  char key_buf[20];
121  int nerrs = 0;
122 
123  /* insert a number of entries */
124  ri.no = 0;
125  ri.max = n;
126 
127  ri.val = 0;
128  ri.step = 1;
129  ri.insertMode = 1;
130 
131  isamc_i.clientData = &ri;
132  isamc_i.read_item = code_read;
133 
134  isamc_p = 0; /* new list */
135  isamb_merge (isb, &isamc_p , &isamc_i);
136 
137  /* read the entries */
138  pp = isamb_pp_open (isb, isamc_p, 1);
139 
140  ri.val = 0;
141  while(isamb_pp_read (pp, key_buf))
142  {
143  int x;
144  memcpy (&x, key_buf, sizeof(int));
145  if (x != ri.val)
146  {
147  yaz_log(YLOG_WARN, "isamb_pp_read. n=%d Got %d (expected %d)",
148  n, x, ri.val);
149  nerrs++;
150  }
151  else if (nerrs)
152  yaz_log(log_level, "isamb_pp_read. n=%d Got %d",
153  n, x);
154 
155  ri.val++;
156  }
157  if (ri.val != ri.max)
158  {
159  yaz_log(YLOG_WARN, "ri.max != ri.max (%d != %d)", ri.val, ri.max);
160  nerrs++;
161  }
162  isamb_dump(isb, isamc_p, log_pr);
163  isamb_pp_close(pp);
164 
165  if (nerrs)
166  exit(3);
167  /* delete a number of entries (even ones) */
168  ri.no = 0;
169  ri.max = n - n/2;
170 
171  ri.val = 0;
172  ri.step = 2;
173  ri.insertMode = 0;
174 
175  isamc_i.clientData = &ri;
176  isamc_i.read_item = code_read;
177 
178  isamb_merge (isb, &isamc_p , &isamc_i);
179 
180  /* delete a number of entries (odd ones) */
181  ri.no = 0;
182  ri.max = n/2;
183 
184  ri.val = 1;
185  ri.step = 2;
186  ri.insertMode = 0;
187 
188  isamc_i.clientData = &ri;
189  isamc_i.read_item = code_read;
190 
191  isamb_merge (isb, &isamc_p, &isamc_i);
192 
193  if (isamc_p)
194  {
195  yaz_log(YLOG_WARN, "isamb_merge did not return empty list n=%d",
196  n);
197  exit(3);
198  }
199 }
200 
201 void tst_forward(ISAMB isb, int n)
202 {
203  ISAMC_I isamc_i;
204  ISAM_P isamc_p;
205  struct read_info ri;
206  int i;
207  ISAMB_PP pp;
208 
209  /* insert a number of entries */
210  ri.val = 0;
211  ri.max = n;
212 
213  ri.no = 0;
214  ri.step = 1;
215  ri.insertMode = 1;
216 
217  isamc_i.clientData = &ri;
218  isamc_i.read_item = code_read;
219 
220  isamc_p = 0;
221  isamb_merge (isb, &isamc_p, &isamc_i);
222 
223  /* read the entries */
224  pp = isamb_pp_open (isb, isamc_p, 1);
225 
226  for (i = 0; i<ri.max; i +=2 )
227  {
228  int x = -1;
229  int xu = i;
230  isamb_pp_forward(pp, &x, &xu);
231  if (x != xu && xu != x+1)
232  {
233  yaz_log(YLOG_WARN, "isamb_pp_forward (1). Got %d (expected %d)",
234  x, xu);
235  exit(4);
236  }
237  ri.no++;
238  }
239  isamb_pp_close(pp);
240 
241  pp = isamb_pp_open (isb, isamc_p, 1);
242  for (i = 0; i<ri.max; i += 100)
243  {
244  int x = -1;
245  int xu = i;
246  isamb_pp_forward(pp, &x, &xu);
247  if (x != xu && xu != x+1)
248  {
249  yaz_log(YLOG_WARN, "isamb_pp_forward (2). Got %d (expected %d)",
250  x, xu);
251  exit(4);
252  }
253  ri.no++;
254  }
255  isamb_pp_close(pp);
256 
257  isamb_unlink(isb, isamc_p);
258 }
259 
260 void tst_x(ISAMB isb)
261 {
262  ISAMC_I isamc_i;
263  ISAM_P isamb_p = 0;
264  struct read_info ri;
265 
266  isamc_i.clientData = &ri;
267  isamc_i.read_item = code_read;
268  ri.no = 0;
269  ri.max = 500;
270 
271  ri.val = 1000;
272  ri.step = 1;
273  ri.insertMode = 1;
274 
275  isamb_merge (isb, &isamb_p , &isamc_i);
276 
277  ri.no = 0;
278  ri.max = 500;
279 
280  ri.val = 1;
281  ri.step = 1;
282  ri.insertMode = 1;
283 
284  isamb_merge (isb, &isamb_p , &isamc_i);
285 }
286 
287 void tst_append(ISAMB isb, int n)
288 {
289  ISAMC_I isamc_i;
290  ISAM_P isamb_p = 0;
291  struct read_info ri;
292  int i;
293  int chunk = 10;
294 
295  for (i = 0; i < n; i += chunk)
296  {
297  /* insert a number of entries */
298  ri.no = 0;
299  ri.max = i + chunk;
300 
301  ri.val = 0;
302  ri.step = 1;
303  ri.insertMode = 1;
304 
305  isamc_i.clientData = &ri;
306  isamc_i.read_item = code_read;
307 
308  isamb_merge (isb, &isamb_p , &isamc_i);
309  }
310 }
311 
312 
314  int max;
315  int idx;
316  int level;
317  int *delta;
318 };
319 
320 int tst_random_read(void *vp, char **dst, int *insertMode)
321 {
322  struct random_read_info *ri = (struct random_read_info *)vp;
323  int x;
324 
325  while(ri->idx < ri->max && ri->delta[ri->idx] == ri->level)
326  {
327  ri->idx++;
328  ri->level = 0;
329  }
330  if (ri->idx >= ri->max)
331  return 0;
332 
333  if (ri->delta[ri->idx] > 0)
334  {
335  ri->level++;
336  *insertMode = 1;
337  }
338  else
339  {
340  ri->level--;
341  *insertMode = 0;
342  }
343  x = ri->idx;
344  memcpy (*dst, &x, sizeof(int));
345  (*dst)+=sizeof(int);
346 
347  yaz_log(YLOG_DEBUG, "%d %5d", *insertMode, x);
348  return 1;
349 }
350 
351 void tst_random(ISAMB isb, int n, int rounds, int max_dups)
352 {
353  ISAM_P isamb_p = 0;
354 
355  int *freq = malloc(sizeof(int) * n);
356  int *delta = malloc(sizeof(int) * n);
357  int i, j;
358  for (i = 0; i<n; i++)
359  freq[i] = 0;
360 
361  for (j = 0; j<rounds; j++)
362  {
363  yaz_log(YLOG_DEBUG, "round %d", j);
364  for (i = 0; i<n; i++)
365  {
366  if (rand() & 1)
367  delta[i] = (rand() % (1+max_dups)) - freq[i];
368  else
369  delta[i] = 0;
370  }
371  if (n)
372  {
373  ISAMC_I isamc_i;
374  struct random_read_info ri;
375 
376  ri.delta = delta;
377  ri.idx = 0;
378  ri.max = n;
379  ri.level = 0;
380 
381  isamc_i.clientData = &ri;
382  isamc_i.read_item = tst_random_read;
383 
384  isamb_merge (isb, &isamb_p , &isamc_i);
385  }
386 
387  yaz_log(YLOG_DEBUG, "dump %d", j);
388  isamb_dump(isb, isamb_p, log_pr);
389 
390  yaz_log(YLOG_DEBUG, "----------------------------");
391  for (i = 0; i<n; i++)
392  freq[i] += delta[i];
393 
394  if (!isamb_p)
395  {
396  for (i = 0; i<n; i++)
397  if (freq[i])
398  {
399  yaz_log(YLOG_WARN, "isamb_merge returned 0, but "
400  "freq is non-empty");
401  exit(1);
402  }
403  }
404  else
405  {
406  int level = 0;
407  int idx = 0;
408  char key_buf[20];
409  ISAMB_PP pp = isamb_pp_open (isb, isamb_p, 1);
410 
411  yaz_log(YLOG_DEBUG, "test %d", j);
412 
413  while(isamb_pp_read (pp, key_buf))
414  {
415  int x;
416  memcpy (&x, key_buf, sizeof(int));
417  yaz_log(YLOG_DEBUG, "Got %d", x);
418  while (idx < n && freq[idx] == level)
419  {
420  idx++;
421  level = 0;
422  }
423  if (idx == n)
424  {
425  yaz_log(YLOG_WARN, "tst_random: Extra item: %d", x);
426  exit(1);
427  }
428  if (idx != x)
429  {
430  yaz_log(YLOG_WARN, "tst_random: Mismatch %d != %d",
431  x, idx);
432  exit(1);
433  }
434  level++;
435  }
436  while (idx < n && freq[idx] == level)
437  {
438  idx++;
439  level = 0;
440  }
441  if (idx != n)
442  {
443  yaz_log(YLOG_WARN, "tst_random: Missing item: %d", idx);
444  exit(1);
445  }
446  isamb_pp_close(pp);
447  }
448  }
449  free(freq);
450  free(delta);
451 }
452 
453 /* \fn void tst_minsert(ISAMB isb, int n)
454  \brief insert inserts n identical keys, removes n/2, then n-n/2 ..
455  \param isb ISAMB handle
456  \param n number of keys
457 */
458 void tst_minsert(ISAMB isb, int n)
459 {
460  ISAMC_I isamc_i;
461  ISAM_P isamb_p = 0;
462  struct read_info ri;
463 
464  isamc_i.clientData = &ri;
465 
466  /* all have same value = 1 */
467  ri.val = 1;
468  ri.step = 0;
469 
470  isamc_i.read_item = code_read;
471 
472  ri.no = 0;
473  ri.max = n;
474 
475  ri.insertMode = 1;
476 
477  isamb_merge (isb, &isamb_p , &isamc_i);
478 
479  isamb_dump(isb, isamb_p, log_pr);
480 
481  ri.no = 0;
482  ri.max = n - n/2;
483 
484  ri.insertMode = 0;
485 
486  isamb_merge (isb, &isamb_p , &isamc_i);
487 
488  ri.no = 0;
489  ri.max = n/2;
490 
491  ri.insertMode = 0;
492 
493  isamb_merge (isb, &isamb_p , &isamc_i);
494  if (isamb_p)
495  {
496  yaz_log(YLOG_WARN, "tst_minsert: isamb_merge should be empty n=%d",
497  n);
498  exit(1);
499  }
500 }
501 
502 /* tests for identical keys.. ISAMB does not handle that, so some of the
503  tests below fails
504 */
505 static void identical_keys_tests(ISAMB isb)
506 {
507 #if 1
508  tst_minsert(isb, 10);
509 #endif
510 #if 0
511  tst_minsert(isb, 600); /* still fails */
512 #endif
513 #if 1
514  tst_random(isb, 20, 200, 1);
515 #endif
516 #if 1
517  tst_random(isb, 5, 200, 2);
518 #endif
519 
520 #if 1
521  tst_random(isb, 250, 10, 4);
522 #endif
523 #if 1
524  /* fails if both are executed */
525  tst_random(isb, 20000, 10, 4);
526  tst_random(isb, 20000, 10, 10);
527 #endif
528 #if 1
529  tst_random(isb, 250, 100, 10);
530 #endif
531 }
532 
533 int main(int argc, char **argv)
534 {
535  BFiles bfs;
536  ISAMB isb;
537  ISAMC_M method;
538 
539  if (argc == 2)
540  yaz_log_init_level(YLOG_ALL);
541 
542  /* setup method (attributes) */
543  method.compare_item = compare_item;
544  method.log_item = log_item;
545  method.codec.start = code_start;
546  method.codec.encode = code_item;
547  method.codec.decode = code_item;
548  method.codec.reset = code_reset;
549  method.codec.stop = code_stop;
550 
551  /* create block system */
552  bfs = bfs_create(0, 0);
553  if (!bfs)
554  {
555  yaz_log(YLOG_WARN, "bfs_create failed");
556  exit(1);
557  }
558 
559  bf_reset(bfs);
560 
561  /* create isam handle */
562  isb = isamb_open (bfs, "isamb", 1, &method, 0);
563  if (!isb)
564  {
565  yaz_log(YLOG_WARN, "isamb_open failed");
566  exit(2);
567  }
568  tst_insert(isb, 1);
569  tst_insert(isb, 2);
570  tst_insert(isb, 20);
571  tst_insert(isb, 100);
572  tst_insert(isb, 500);
573  tst_insert(isb, 10000);
574 
575  tst_forward(isb, 10000);
576 
577  tst_x(isb);
578 
579  tst_append(isb, 1000);
580 
581  if (0)
583 
584  isamb_close(isb);
585 
586  /* exit block system */
587  bfs_destroy(bfs);
588  exit(0);
589  return 0;
590 }
591 /*
592  * Local variables:
593  * c-basic-offset: 4
594  * c-file-style: "Stroustrup"
595  * indent-tabs-mode: nil
596  * End:
597  * vim: shiftwidth=4 tabstop=8 expandtab
598  */
599 
void bf_reset(BFiles bfs)
Removes register and shadow completely.
Definition: bfile.c:268
BFiles bfs_create(const char *spec, const char *base)
creates a Block files collection
Definition: bfile.c:56
void bfs_destroy(BFiles bfiles)
destroys a block files handle
Definition: bfile.c:73
ISAMB_PP isamb_pp_open(ISAMB isamb, ISAM_P pos, int scope)
Definition: isamb.c:1387
ISAMB isamb_open(BFiles bfs, const char *name, int writeflag, ISAMC_M *method, int cache)
Definition: isamb.c:351
void isamb_dump(ISAMB b, ISAM_P pos, void(*pr)(const char *str))
Definition: isamb.c:1498
void isamb_close(ISAMB isamb)
Definition: isamb.c:455
int isamb_pp_forward(ISAMB_PP pp, void *buf, const void *untilbuf)
Definition: isamb.c:1525
void isamb_pp_close(ISAMB_PP pp)
Definition: isamb.c:1429
void isamb_merge(ISAMB b, ISAM_P *pos, ISAMC_I *data)
Definition: isamb.c:1266
int isamb_unlink(ISAMB b, ISAM_P pos)
Definition: isamb.c:1225
int isamb_pp_read(ISAMB_PP pp, void *buf)
Definition: isamb.c:1503
zint ISAM_P
Definition: isamc.h:28
Definition: isamb.c:95
int(* read_item)(void *clientData, char **dst, int *insertMode)
Definition: isamc.h:53
void * clientData
Definition: isamc.h:54
int(* compare_item)(const void *a, const void *b)
Definition: isamc.h:43
ISAM_CODEC codec
Definition: isamc.h:46
void(* log_item)(int logmask, const void *p, const char *txt)
Definition: isamc.h:44
void(* decode)(void *p, char **dst, const char **src)
Definition: isam-codec.h:26
void(* stop)(void *p)
Definition: isam-codec.h:25
void(* encode)(void *p, char **dst, const char **src)
Definition: isam-codec.h:27
void(* reset)(void *p)
Definition: isam-codec.h:28
void *(* start)(void)
Definition: isam-codec.h:24
int step
Definition: benchisamb.c:86
int insertMode
Definition: benchisamb.c:90
void tst_random(ISAMB isb, int n, int rounds, int max_dups)
Definition: tstisamb.c:351
static void log_item(int level, const void *b, const char *txt)
Definition: tstisamb.c:39
int main(int argc, char **argv)
Definition: tstisamb.c:533
void tst_insert(ISAMB isb, int n)
Definition: tstisamb.c:114
void * code_start(void)
Definition: tstisamb.c:64
void code_reset(void *p)
Definition: tstisamb.c:76
int compare_item(const void *a, const void *b)
Definition: tstisamb.c:51
void code_item(void *p, char **dst, const char **src)
Definition: tstisamb.c:69
void tst_append(ISAMB isb, int n)
Definition: tstisamb.c:287
static void log_pr(const char *txt)
Definition: tstisamb.c:46
int tst_random_read(void *vp, char **dst, int *insertMode)
Definition: tstisamb.c:320
void code_stop(void *p)
Definition: tstisamb.c:79
void tst_minsert(ISAMB isb, int n)
Definition: tstisamb.c:458
void tst_forward(ISAMB isb, int n)
Definition: tstisamb.c:201
static int log_level
Definition: tstisamb.c:37
void tst_x(ISAMB isb)
Definition: tstisamb.c:260
int code_read(void *vp, char **dst, int *insertMode)
Definition: tstisamb.c:92
static void identical_keys_tests(ISAMB isb)
Definition: tstisamb.c:505