IDZEBRA  2.1.2
records.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 /*
21  * Format of first block (assumes a 512 block size)
22  * next (8 bytes)
23  * ref_count (2 bytes)
24  * block (500 bytes)
25  *
26  * Format of subsequent blocks
27  * next (8 bytes)
28  * block (502 bytes)
29  *
30  * Format of each record
31  * sysno
32  * (length, data) - pairs
33  * length = 0 if same as previous
34  */
35 #if HAVE_CONFIG_H
36 #include <config.h>
37 #endif
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <assert.h>
41 #include <string.h>
42 
43 #include <yaz/yaz-util.h>
44 #include <idzebra/bfile.h>
45 #include "recindex.h"
46 
47 #if HAVE_BZLIB_H
48 #include <bzlib.h>
49 #endif
50 #if HAVE_ZLIB_H
51 #include <zlib.h>
52 #endif
53 
54 #define REC_BLOCK_TYPES 2
55 #define REC_HEAD_MAGIC "recindex"
56 #define REC_VERSION 5
57 
58 struct records_info {
59  int rw;
61 
63 
66 
67  char *tmp_buf;
68  int tmp_size;
69 
72  int cache_cur;
73  int cache_max;
74 
76 
78 
79  struct records_head {
80  char magic[8];
81  char version[4];
87 
92 
93  } head;
94 };
95 
98 
101  enum recordCacheFlag flag;
102 };
103 
104 struct record_index_entry {
105  zint next; /* first block of record info / next free entry */
106  int size; /* size of record or 0 if free entry */
107 };
108 
109 Record rec_cp(Record rec);
110 
111 /* Modify argument to if below: 1=normal, 0=sysno testing */
112 #if 1
113 /* If this is used sysno are not converted (no testing) */
114 #define FAKE_OFFSET 0
115 #define USUAL_RANGE 6000000000LL
116 
117 #else
118 /* Use a fake > 2^32 offset so we can test for proper 64-bit handling */
119 #define FAKE_OFFSET 6000000000LL
120 #define USUAL_RANGE 2000000000LL
121 #endif
122 
124 {
125  assert(sysno >= 0 && sysno <= USUAL_RANGE);
126  return sysno + FAKE_OFFSET;
127 }
128 
130 {
131  assert(sysno >= FAKE_OFFSET && sysno <= FAKE_OFFSET + USUAL_RANGE);
132  return sysno - FAKE_OFFSET;
133 }
134 
135 static void rec_tmp_expand(Records p, int size)
136 {
137  if (p->tmp_size < size + 2048 ||
139  {
140  xfree(p->tmp_buf);
141  p->tmp_size = size + (int)
142  (p->head.block_size[REC_BLOCK_TYPES-1])*2 + 2048;
143  p->tmp_buf = (char *) xmalloc(p->tmp_size);
144  }
145 }
146 
148 {
149  struct record_index_entry entry;
150  zint freeblock;
151  char block_and_ref[sizeof(zint) + sizeof(short)];
152  int dst_type;
153  int first = 1;
154 
155  if (recindex_read_indx(p->recindex, sysno, &entry, sizeof(entry), 1) != 1)
156  return ZEBRA_FAIL;
157 
158  freeblock = entry.next;
159  assert(freeblock > 0);
160  dst_type = CAST_ZINT_TO_INT(freeblock & 7);
161  assert(dst_type < REC_BLOCK_TYPES);
162  freeblock = freeblock / 8;
163  while (freeblock)
164  {
165  if (bf_read(p->data_BFile[dst_type], freeblock, 0,
166  first ? sizeof(block_and_ref) : sizeof(zint),
167  block_and_ref) != 1)
168  {
169  yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in rec_del_single");
170  return ZEBRA_FAIL;
171  }
172  if (first)
173  {
174  short ref;
175  memcpy(&ref, block_and_ref + sizeof(freeblock), sizeof(ref));
176  --ref;
177  memcpy(block_and_ref + sizeof(freeblock), &ref, sizeof(ref));
178  if (ref)
179  {
180  /* there is still a reference to this block.. */
181  if (bf_write(p->data_BFile[dst_type], freeblock, 0,
182  sizeof(block_and_ref), block_and_ref))
183  {
184  yaz_log(YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
185  return ZEBRA_FAIL;
186  }
187  return ZEBRA_OK;
188  }
189  /* the list of blocks can all be removed (ref == 0) */
190  first = 0;
191  }
192 
193  if (bf_write(p->data_BFile[dst_type], freeblock, 0, sizeof(freeblock),
194  &p->head.block_free[dst_type]))
195  {
196  yaz_log(YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
197  return ZEBRA_FAIL;
198  }
199  p->head.block_free[dst_type] = freeblock;
200  memcpy(&freeblock, block_and_ref, sizeof(freeblock));
201 
202  p->head.block_used[dst_type]--;
203  }
204  p->head.total_bytes -= entry.size;
205  return ZEBRA_OK;
206 }
207 
209 {
210  struct record_index_entry entry;
211 
212  /* all data in entry must be reset, since it's written verbatim */
213  memset(&entry, '\0', sizeof(entry));
215  return ZEBRA_FAIL;
216 
217  entry.next = p->head.index_free;
218  entry.size = 0;
220  recindex_write_indx(p->recindex, rec_sysno_to_int(rec->sysno), &entry, sizeof(entry));
221  return ZEBRA_OK;
222 }
223 
224 static ZEBRA_RES rec_write_tmp_buf(Records p, int size, zint *sysnos)
225 {
226  struct record_index_entry entry;
227  int no_written = 0;
228  char *cptr = p->tmp_buf;
229  zint block_prev = -1, block_free;
230  int dst_type = 0;
231  int i;
232 
233  /* all data in entry must be reset, since it's written verbatim */
234  memset(&entry, '\0', sizeof(entry));
235 
236  for (i = 1; i<REC_BLOCK_TYPES; i++)
237  if (size >= p->head.block_move[i])
238  dst_type = i;
239  while (no_written < size)
240  {
241  block_free = p->head.block_free[dst_type];
242  if (block_free)
243  {
244  if (bf_read(p->data_BFile[dst_type],
245  block_free, 0, sizeof(*p->head.block_free),
246  &p->head.block_free[dst_type]) != 1)
247  {
248  yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in %s at free block "
249  ZINT_FORMAT,
250  p->data_fname[dst_type], block_free);
251  return ZEBRA_FAIL;
252  }
253  }
254  else
255  block_free = p->head.block_last[dst_type]++;
256  if (block_prev == -1)
257  {
258  entry.next = block_free*8 + dst_type;
259  entry.size = size;
260  p->head.total_bytes += size;
261  while (*sysnos > 0)
262  {
263  recindex_write_indx(p->recindex, *sysnos, &entry, sizeof(entry));
264  sysnos++;
265  }
266  }
267  else
268  {
269  memcpy(cptr, &block_free, sizeof(block_free));
270  bf_write(p->data_BFile[dst_type], block_prev, 0, 0, cptr);
271  cptr = p->tmp_buf + no_written;
272  }
273  block_prev = block_free;
274  no_written += CAST_ZINT_TO_INT(p->head.block_size[dst_type])
275  - sizeof(zint);
276  p->head.block_used[dst_type]++;
277  }
278  assert(block_prev != -1);
279  block_free = 0;
280  memcpy(cptr, &block_free, sizeof(block_free));
281  bf_write(p->data_BFile[dst_type], block_prev, 0,
282  sizeof(block_free) + (p->tmp_buf+size) - cptr, cptr);
283  return ZEBRA_OK;
284 }
285 
287 {
288  switch(compression_method)
289  {
290  case REC_COMPRESS_ZLIB:
291 #if HAVE_ZLIB_H
292  return 1;
293 #else
294  return 0;
295 #endif
296  case REC_COMPRESS_BZIP2:
297 #if HAVE_BZLIB_H
298  return 1;
299 #else
300  return 0;
301 #endif
302  case REC_COMPRESS_NONE:
303  return 1;
304  }
305  return 0;
306 }
307 
309 {
310  Records p;
311  int i, r;
312  int version;
313  ZEBRA_RES ret = ZEBRA_OK;
314 
315  p = (Records) xmalloc(sizeof(*p));
316  memset(&p->head, '\0', sizeof(p->head));
318  p->rw = rw;
319  p->tmp_size = 4096;
320  p->tmp_buf = (char *) xmalloc(p->tmp_size);
321  p->compression_chunk_size = 0;
322  if (compression_method == REC_COMPRESS_BZIP2)
323  p->compression_chunk_size = 90000;
324  p->recindex = recindex_open(bfs, rw, 0 /* 1=isamb for recindex */);
325  r = recindex_read_head(p->recindex, p->tmp_buf);
326  switch (r)
327  {
328  case 0:
329  memcpy(p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
330  sprintf(p->head.version, "%3d", REC_VERSION);
331  p->head.index_free = 0;
332  p->head.index_last = 1;
333  p->head.no_records = 0;
334  p->head.total_bytes = 0;
335  for (i = 0; i<REC_BLOCK_TYPES; i++)
336  {
337  p->head.block_free[i] = 0;
338  p->head.block_last[i] = 1;
339  p->head.block_used[i] = 0;
340  }
341  p->head.block_size[0] = 256;
342  p->head.block_move[0] = 0;
343  for (i = 1; i<REC_BLOCK_TYPES; i++)
344  {
345  p->head.block_size[i] = p->head.block_size[i-1] * 8;
346  p->head.block_move[i] = p->head.block_size[i] * 2;
347  }
348  if (rw)
349  {
351  &p->head, sizeof(p->head)) != ZEBRA_OK)
352  ret = ZEBRA_FAIL;
353  }
354  break;
355  case 1:
356  memcpy(&p->head, p->tmp_buf, sizeof(p->head));
357  if (memcmp(p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
358  {
359  yaz_log(YLOG_FATAL, "file %s has bad format",
361  ret = ZEBRA_FAIL;
362  }
363  version = atoi(p->head.version);
364  if (version != REC_VERSION)
365  {
366  yaz_log(YLOG_FATAL, "file %s is version %d, but version"
367  " %d is required",
368  recindex_get_fname(p->recindex), version, REC_VERSION);
369  ret = ZEBRA_FAIL;
370  }
371  break;
372  }
373  for (i = 0; i<REC_BLOCK_TYPES; i++)
374  {
375  char str[80];
376  sprintf(str, "recd%c", i + 'A');
377  p->data_fname[i] = (char *) xmalloc(strlen(str)+1);
378  strcpy(p->data_fname[i], str);
379  p->data_BFile[i] = NULL;
380  }
381  for (i = 0; i<REC_BLOCK_TYPES; i++)
382  {
383  if (!(p->data_BFile[i] =
384  bf_open(bfs, p->data_fname[i],
385  CAST_ZINT_TO_INT(p->head.block_size[i]), rw)))
386  {
387  yaz_log(YLOG_FATAL|YLOG_ERRNO, "bf_open %s", p->data_fname[i]);
388  ret = ZEBRA_FAIL;
389  break;
390  }
391  }
392  p->cache_max = 400;
393  p->cache_cur = 0;
394  p->record_cache = (struct record_cache_entry *)
395  xmalloc(sizeof(*p->record_cache)*p->cache_max);
396  zebra_mutex_init(&p->mutex);
397  if (ret == ZEBRA_FAIL)
398  rec_close(&p);
399  return p;
400 }
401 
402 static void rec_encode_unsigned(unsigned n, unsigned char *buf, int *len)
403 {
404  (*len) = 0;
405  while (n > 127)
406  {
407  buf[*len] = 128 + (n & 127);
408  n = n >> 7;
409  (*len)++;
410  }
411  buf[*len] = n;
412  (*len)++;
413 }
414 
415 static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len)
416 {
417  unsigned n = 0;
418  unsigned w = 1;
419  (*len) = 0;
420 
421  while (buf[*len] > 127)
422  {
423  n += w*(buf[*len] & 127);
424  w = w << 7;
425  (*len)++;
426  }
427  n += w * buf[*len];
428  (*len)++;
429  *np = n;
430 }
431 
432 static void rec_encode_zint(zint n, unsigned char *buf, int *len)
433 {
434  (*len) = 0;
435  while (n > 127)
436  {
437  buf[*len] = (unsigned) (128 + (n & 127));
438  n = n >> 7;
439  (*len)++;
440  }
441  buf[*len] = (unsigned) n;
442  (*len)++;
443 }
444 
445 static void rec_decode_zint(zint *np, unsigned char *buf, int *len)
446 {
447  zint n = 0;
448  zint w = 1;
449  (*len) = 0;
450 
451  while (buf[*len] > 127)
452  {
453  n += w*(buf[*len] & 127);
454  w = w << 7;
455  (*len)++;
456  }
457  n += w * buf[*len];
458  (*len)++;
459  *np = n;
460 }
461 
462 static void rec_cache_flush_block1(Records p, Record rec, Record last_rec,
463  char **out_buf, int *out_size,
464  int *out_offset)
465 {
466  int i;
467  int len;
468 
469  for (i = 0; i<REC_NO_INFO; i++)
470  {
471  if (*out_offset + CAST_ZINT_TO_INT(rec->size[i]) + 20 > *out_size)
472  {
473  int new_size = *out_offset + rec->size[i] + 65536;
474  char *np = (char *) xmalloc(new_size);
475  if (*out_offset)
476  memcpy(np, *out_buf, *out_offset);
477  xfree(*out_buf);
478  *out_size = new_size;
479  *out_buf = np;
480  }
481  if (i == 0)
482  {
484  (unsigned char *) *out_buf + *out_offset, &len);
485  (*out_offset) += len;
486  }
487  if (rec->size[i] == 0)
488  {
489  rec_encode_unsigned(1, (unsigned char *) *out_buf + *out_offset,
490  &len);
491  (*out_offset) += len;
492  }
493  else if (last_rec && rec->size[i] == last_rec->size[i] &&
494  !memcmp(rec->info[i], last_rec->info[i], rec->size[i]))
495  {
496  rec_encode_unsigned(0, (unsigned char *) *out_buf + *out_offset,
497  &len);
498  (*out_offset) += len;
499  }
500  else
501  {
502  rec_encode_unsigned(rec->size[i]+1,
503  (unsigned char *) *out_buf + *out_offset,
504  &len);
505  (*out_offset) += len;
506  memcpy(*out_buf + *out_offset, rec->info[i], rec->size[i]);
507  (*out_offset) += rec->size[i];
508  }
509  }
510 }
511 
512 static ZEBRA_RES rec_flush_shared(Records p, short ref_count, zint *sysnos,
513  char *out_buf, int out_offset)
514 {
515  ZEBRA_RES ret = ZEBRA_OK;
516  if (ref_count)
517  {
518 #if HAVE_BZLIB_H
519  int i;
520 #endif
521  unsigned int csize = 0; /* indicate compression "not performed yet" */
523  switch (compression_method)
524  {
525  case REC_COMPRESS_ZLIB:
526 #if HAVE_ZLIB_H
527  csize = out_offset + (out_offset >> 6) + 620;
528  while (1)
529  {
530  int r;
531  uLongf destLen = csize;
532  rec_tmp_expand(p, csize);
533  r = compress((Bytef *) p->tmp_buf+sizeof(zint)+sizeof(short)+
534  sizeof(char),
535  &destLen, (const Bytef *) out_buf, out_offset);
536  csize = destLen;
537  if (r == Z_OK)
538  {
539  break;
540  }
541  if (r != Z_MEM_ERROR)
542  {
543  yaz_log(YLOG_WARN, "compress error: %d", r);
544  csize = 0;
545  break;
546  }
547  csize = csize * 2;
548  }
549 #endif
550  break;
551  case REC_COMPRESS_BZIP2:
552 #if HAVE_BZLIB_H
553  csize = out_offset + (out_offset >> 6) + 620;
554  rec_tmp_expand(p, csize);
555 #ifdef BZ_CONFIG_ERROR
556  i = BZ2_bzBuffToBuffCompress
557 #else
558  i = bzBuffToBuffCompress
559 #endif
560  (p->tmp_buf+sizeof(zint)+sizeof(short)+
561  sizeof(char),
562  &csize, out_buf, out_offset, 1, 0, 30);
563  if (i != BZ_OK)
564  {
565  yaz_log(YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
566  csize = 0;
567  }
568 #endif
569  break;
570  case REC_COMPRESS_NONE:
571  break;
572  }
573  if (!csize)
574  {
575  /* either no compression or compression not supported ... */
576  csize = out_offset;
577  rec_tmp_expand(p, csize);
578  memcpy(p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char),
579  out_buf, out_offset);
580  csize = out_offset;
581  compression_method = REC_COMPRESS_NONE;
582  }
583  memcpy(p->tmp_buf + sizeof(zint), &ref_count, sizeof(ref_count));
584  memcpy(p->tmp_buf + sizeof(zint)+sizeof(short),
585  &compression_method, sizeof(compression_method));
586 
587  /* -------- compression */
588  if (rec_write_tmp_buf(p, csize + sizeof(short) + sizeof(char), sysnos)
589  != ZEBRA_OK)
590  ret = ZEBRA_FAIL;
591  }
592  return ret;
593 }
594 
595 static ZEBRA_RES rec_write_multiple(Records p, int saveCount)
596 {
597  int i;
598  short ref_count = 0;
599  Record last_rec = 0;
600  int out_size = 1000;
601  int out_offset = 0;
602  char *out_buf = (char *) xmalloc(out_size);
603  zint *sysnos = (zint *) xmalloc(sizeof(*sysnos) * (p->cache_cur + 1));
604  zint *sysnop = sysnos;
605  ZEBRA_RES ret = ZEBRA_OK;
606 
607  for (i = 0; i<p->cache_cur - saveCount; i++)
608  {
609  struct record_cache_entry *e = p->record_cache + i;
610  switch (e->flag)
611  {
612  case recordFlagNew:
613  rec_cache_flush_block1(p, e->rec, last_rec, &out_buf,
614  &out_size, &out_offset);
615  *sysnop++ = rec_sysno_to_int(e->rec->sysno);
616  ref_count++;
617  e->flag = recordFlagNop;
618  last_rec = e->rec;
619  break;
620  case recordFlagWrite:
622  != ZEBRA_OK)
623  ret = ZEBRA_FAIL;
624 
625  rec_cache_flush_block1(p, e->rec, last_rec, &out_buf,
626  &out_size, &out_offset);
627  *sysnop++ = rec_sysno_to_int(e->rec->sysno);
628  ref_count++;
629  e->flag = recordFlagNop;
630  last_rec = e->rec;
631  break;
632  case recordFlagDelete:
633  if (rec_delete_single(p, e->rec) != ZEBRA_OK)
634  ret = ZEBRA_FAIL;
635 
636  e->flag = recordFlagNop;
637  break;
638  case recordFlagNop:
639  break;
640  default:
641  break;
642  }
643  }
644 
645  *sysnop = -1;
646  rec_flush_shared(p, ref_count, sysnos, out_buf, out_offset);
647  xfree(out_buf);
648  xfree(sysnos);
649  return ret;
650 }
651 
652 static ZEBRA_RES rec_cache_flush(Records p, int saveCount)
653 {
654  int i, j;
655  ZEBRA_RES ret;
656 
657  if (saveCount >= p->cache_cur)
658  saveCount = 0;
659 
660  ret = rec_write_multiple(p, saveCount);
661 
662  for (i = 0; i<p->cache_cur - saveCount; i++)
663  {
664  struct record_cache_entry *e = p->record_cache + i;
665  rec_free(&e->rec);
666  }
667  /* i still being used ... */
668  for (j = 0; j<saveCount; j++, i++)
669  memcpy(p->record_cache+j, p->record_cache+i,
670  sizeof(*p->record_cache));
671  p->cache_cur = saveCount;
672  return ret;
673 }
674 
676  enum recordCacheFlag flag)
677 {
678  int i;
679  for (i = 0; i<p->cache_cur; i++)
680  {
681  struct record_cache_entry *e = p->record_cache + i;
682  if (e->rec->sysno == sysno)
683  {
684  if (flag != recordFlagNop && e->flag == recordFlagNop)
685  e->flag = flag;
686  return &e->rec;
687  }
688  }
689  return NULL;
690 }
691 
693 {
694  struct record_cache_entry *e;
695  ZEBRA_RES ret = ZEBRA_OK;
696 
697  if (p->cache_cur == p->cache_max)
698  ret = rec_cache_flush(p, 1);
699  else if (p->cache_cur > 0)
700  {
701  int i, j;
702  int used = 0;
703  for (i = 0; i<p->cache_cur; i++)
704  {
705  Record r = (p->record_cache + i)->rec;
706  for (j = 0; j<REC_NO_INFO; j++)
707  used += r->size[j];
708  }
709  if (used > p->compression_chunk_size)
710  ret = rec_cache_flush(p, 1);
711  }
712  assert(p->cache_cur < p->cache_max);
713 
714  e = p->record_cache + (p->cache_cur)++;
715  e->flag = flag;
716  e->rec = rec_cp(rec);
717  return ret;
718 }
719 
721 {
722  Records p = *pp;
723  int i;
724  ZEBRA_RES ret = ZEBRA_OK;
725 
726  if (!p)
727  return ret;
728 
730  if (rec_cache_flush(p, 0) != ZEBRA_OK)
731  ret = ZEBRA_FAIL;
732 
733  xfree(p->record_cache);
734 
735  if (p->rw)
736  {
737  if (recindex_write_head(p->recindex, &p->head, sizeof(p->head)) != ZEBRA_OK)
738  ret = ZEBRA_FAIL;
739  }
740 
742 
743  for (i = 0; i<REC_BLOCK_TYPES; i++)
744  {
745  if (p->data_BFile[i])
746  bf_close(p->data_BFile[i]);
747  xfree(p->data_fname[i]);
748  }
749  xfree(p->tmp_buf);
750  xfree(p);
751  *pp = NULL;
752  return ret;
753 }
754 
755 static Record rec_get_int(Records p, zint sysno)
756 {
757  int i, in_size, r;
758  Record rec, *recp;
759  struct record_index_entry entry;
760  zint freeblock;
761  int dst_type;
762  char *nptr, *cptr;
763  char *in_buf = 0;
764  char *bz_buf = 0;
765  char compression_method;
766 
767  assert(sysno > 0);
768  assert(p);
769 
770  if ((recp = rec_cache_lookup(p, sysno, recordFlagNop)))
771  return rec_cp(*recp);
772 
773  if (recindex_read_indx(p->recindex, rec_sysno_to_int(sysno), &entry, sizeof(entry), 1) < 1)
774  return NULL; /* record is not there! */
775 
776  if (!entry.size)
777  return NULL; /* record is deleted */
778 
779  dst_type = (int) (entry.next & 7);
780  assert(dst_type < REC_BLOCK_TYPES);
781  freeblock = entry.next / 8;
782 
783  assert(freeblock > 0);
784 
785  rec_tmp_expand(p, entry.size);
786 
787  cptr = p->tmp_buf;
788  r = bf_read(p->data_BFile[dst_type], freeblock, 0, 0, cptr);
789  if (r < 0)
790  return 0;
791  memcpy(&freeblock, cptr, sizeof(freeblock));
792 
793  while (freeblock)
794  {
795  zint tmp;
796 
797  cptr += p->head.block_size[dst_type] - sizeof(freeblock);
798 
799  memcpy(&tmp, cptr, sizeof(tmp));
800  r = bf_read(p->data_BFile[dst_type], freeblock, 0, 0, cptr);
801  if (r < 0)
802  return 0;
803  memcpy(&freeblock, cptr, sizeof(freeblock));
804  memcpy(cptr, &tmp, sizeof(tmp));
805  }
806 
807  rec = (Record) xmalloc(sizeof(*rec));
808  rec->sysno = sysno;
809  memcpy(&compression_method, p->tmp_buf + sizeof(zint) + sizeof(short),
810  sizeof(compression_method));
811  in_buf = p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char);
812  in_size = entry.size - sizeof(short) - sizeof(char);
813  switch (compression_method)
814  {
815  case REC_COMPRESS_ZLIB:
816 #if HAVE_ZLIB_H
817  if (1)
818  {
819  unsigned int bz_size = entry.size * 20 + 100;
820  while (1)
821  {
822  uLongf destLen = bz_size;
823  bz_buf = (char *) xmalloc(bz_size);
824  i = uncompress((Bytef *) bz_buf, &destLen,
825  (const Bytef *) in_buf, in_size);
826  if (i == Z_OK)
827  {
828  bz_size = destLen;
829  break;
830  }
831  yaz_log(YLOG_LOG, "failed");
832  xfree(bz_buf);
833  bz_size *= 2;
834  }
835  in_buf = bz_buf;
836  in_size = bz_size;
837  }
838 #else
839  yaz_log(YLOG_FATAL, "cannot decompress record(s) in ZLIB format");
840  return 0;
841 #endif
842  break;
843  case REC_COMPRESS_BZIP2:
844 #if HAVE_BZLIB_H
845  if (1)
846  {
847  unsigned int bz_size = entry.size * 20 + 100;
848  while (1)
849  {
850  bz_buf = (char *) xmalloc(bz_size);
851 #ifdef BZ_CONFIG_ERROR
852  i = BZ2_bzBuffToBuffDecompress
853 #else
854  i = bzBuffToBuffDecompress
855 #endif
856  (bz_buf, &bz_size, in_buf, in_size, 0, 0);
857  if (i == BZ_OK)
858  break;
859  yaz_log(YLOG_LOG, "failed");
860  xfree(bz_buf);
861  bz_size *= 2;
862  }
863  in_buf = bz_buf;
864  in_size = bz_size;
865  }
866 #else
867  yaz_log(YLOG_FATAL, "cannot decompress record(s) in BZIP2 format");
868  return 0;
869 #endif
870  break;
871  case REC_COMPRESS_NONE:
872  break;
873  }
874  for (i = 0; i<REC_NO_INFO; i++)
875  rec->info[i] = 0;
876 
877  nptr = in_buf; /* skip ref count */
878  while (nptr < in_buf + in_size)
879  {
880  zint this_sysno;
881  int len;
882  rec_decode_zint(&this_sysno, (unsigned char *) nptr, &len);
883  nptr += len;
884 
885  for (i = 0; i < REC_NO_INFO; i++)
886  {
887  unsigned int this_size;
888  rec_decode_unsigned(&this_size, (unsigned char *) nptr, &len);
889  nptr += len;
890 
891  if (this_size == 0)
892  continue;
893  rec->size[i] = this_size-1;
894 
895  if (rec->size[i])
896  {
897  rec->info[i] = nptr;
898  nptr += rec->size[i];
899  }
900  else
901  rec->info[i] = NULL;
902  }
903  if (this_sysno == rec_sysno_to_int(sysno))
904  break;
905  }
906  for (i = 0; i<REC_NO_INFO; i++)
907  {
908  if (rec->info[i] && rec->size[i])
909  {
910  char *np = xmalloc(rec->size[i]+1);
911  memcpy(np, rec->info[i], rec->size[i]);
912  np[rec->size[i]] = '\0';
913  rec->info[i] = np;
914  }
915  else
916  {
917  assert(rec->info[i] == 0);
918  assert(rec->size[i] == 0);
919  }
920  }
921  xfree(bz_buf);
922  if (rec_cache_insert(p, rec, recordFlagNop) != ZEBRA_OK)
923  return 0;
924  return rec;
925 }
926 
928 {
929  Record rec;
930  zebra_mutex_lock(&p->mutex);
931 
932  rec = rec_get_int(p, sysno);
934  return rec;
935 }
936 
938 {
939  return rec_get(p, rec_sysno_to_ext(1));
940 }
941 
943 {
944  Record next = 0;
945  zint next_sysno_int = rec_sysno_to_int(rec->sysno);
946 
947  while (!next)
948  {
949  ++next_sysno_int;
950  if (next_sysno_int == p->head.index_last)
951  break;
952  next = rec_get(p, rec_sysno_to_ext(next_sysno_int));
953  }
954  return next;
955 }
956 
958 {
959  int i;
960  zint sysno;
961  Record rec;
962 
963  assert(p);
964  rec = (Record) xmalloc(sizeof(*rec));
965  if (1 || p->head.index_free == 0)
966  sysno = (p->head.index_last)++;
967  else
968  {
969  struct record_index_entry entry;
970 
971  if (recindex_read_indx(p->recindex, p->head.index_free, &entry, sizeof(entry), 0) < 1)
972  {
973  xfree(rec);
974  return 0;
975  }
976  sysno = p->head.index_free;
977  p->head.index_free = entry.next;
978  }
979  (p->head.no_records)++;
980  rec->sysno = rec_sysno_to_ext(sysno);
981  for (i = 0; i < REC_NO_INFO; i++)
982  {
983  rec->info[i] = NULL;
984  rec->size[i] = 0;
985  }
987  return rec;
988 }
989 
991 {
992  Record rec;
993  zebra_mutex_lock(&p->mutex);
994 
995  rec = rec_new_int(p);
997  return rec;
998 }
999 
1001 {
1002  Record *recp;
1003  ZEBRA_RES ret = ZEBRA_OK;
1004 
1005  zebra_mutex_lock(&p->mutex);
1006  (p->head.no_records)--;
1007  if ((recp = rec_cache_lookup(p, (*recpp)->sysno, recordFlagDelete)))
1008  {
1009  rec_free(recp);
1010  *recp = *recpp;
1011  }
1012  else
1013  {
1014  ret = rec_cache_insert(p, *recpp, recordFlagDelete);
1015  rec_free(recpp);
1016  }
1018  *recpp = NULL;
1019  return ret;
1020 }
1021 
1023 {
1024  Record *recp;
1025  ZEBRA_RES ret = ZEBRA_OK;
1026 
1027  zebra_mutex_lock(&p->mutex);
1028  if ((recp = rec_cache_lookup(p, (*recpp)->sysno, recordFlagWrite)))
1029  {
1030  rec_free(recp);
1031  *recp = *recpp;
1032  }
1033  else
1034  {
1035  ret = rec_cache_insert(p, *recpp, recordFlagWrite);
1036  rec_free(recpp);
1037  }
1039  *recpp = NULL;
1040  return ret;
1041 }
1042 
1043 void rec_free(Record *recpp)
1044 {
1045  int i;
1046 
1047  if (!*recpp)
1048  return ;
1049  for (i = 0; i < REC_NO_INFO; i++)
1050  xfree((*recpp)->info[i]);
1051  xfree(*recpp);
1052  *recpp = NULL;
1053 }
1054 
1056 {
1057  Record n;
1058  int i;
1059 
1060  n = (Record) xmalloc(sizeof(*n));
1061  n->sysno = rec->sysno;
1062  for (i = 0; i < REC_NO_INFO; i++)
1063  if (!rec->info[i])
1064  {
1065  n->info[i] = NULL;
1066  n->size[i] = 0;
1067  }
1068  else
1069  {
1070  n->size[i] = rec->size[i];
1071  n->info[i] = (char *) xmalloc(rec->size[i]+1);
1072  memcpy(n->info[i], rec->info[i], rec->size[i]);
1073  n->info[i][rec->size[i]] = '\0';
1074  }
1075  return n;
1076 }
1077 
1078 
1079 char *rec_strdup(const char *s, size_t *len)
1080 {
1081  char *p;
1082 
1083  if (!s)
1084  {
1085  *len = 0;
1086  return NULL;
1087  }
1088  *len = strlen(s)+1;
1089  p = (char *) xmalloc(*len);
1090  strcpy(p, s);
1091  return p;
1092 }
1093 
1094 void rec_prstat(Records records, int verbose)
1095 {
1096  int i;
1097  zint total_bytes = 0;
1098 
1099  yaz_log (YLOG_LOG,
1100  "Total records %8" ZINT_FORMAT0,
1101  records->head.no_records);
1102 
1103  for (i = 0; i< REC_BLOCK_TYPES; i++)
1104  {
1105  yaz_log (YLOG_LOG, "Record blocks of size "ZINT_FORMAT,
1106  records->head.block_size[i]);
1107  yaz_log (YLOG_LOG,
1108  " Used/Total/Bytes used "
1110  records->head.block_used[i], records->head.block_last[i]-1,
1111  records->head.block_used[i] * records->head.block_size[i]);
1112  total_bytes +=
1113  records->head.block_used[i] * records->head.block_size[i];
1114 
1115  yaz_log(YLOG_LOG, " Block Last " ZINT_FORMAT, records->head.block_last[i]);
1116  if (verbose)
1117  { /* analyse free lists */
1118  zint no_free = 0;
1119  zint block_free = records->head.block_free[i];
1120  WRBUF w = wrbuf_alloc();
1121  while (block_free)
1122  {
1123  zint nblock;
1124  no_free++;
1125  wrbuf_printf(w, " " ZINT_FORMAT, block_free);
1126  if (bf_read(records->data_BFile[i],
1127  block_free, 0, sizeof(nblock), &nblock) != 1)
1128  {
1129  yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in %s at free block "
1130  ZINT_FORMAT,
1131  records->data_fname[i], block_free);
1132  break;
1133  }
1134  block_free = nblock;
1135  }
1136  yaz_log (YLOG_LOG,
1137  " Number in free list %8" ZINT_FORMAT0, no_free);
1138  if (no_free)
1139  yaz_log(YLOG_LOG, "%s", wrbuf_cstr(w));
1140  wrbuf_destroy(w);
1141  }
1142  }
1143  yaz_log (YLOG_LOG,
1144  "Total size of record index in bytes %8" ZINT_FORMAT0,
1145  records->head.total_bytes);
1146  yaz_log (YLOG_LOG,
1147  "Total size with overhead %8" ZINT_FORMAT0,
1148  total_bytes);
1149 }
1150 
1151 /*
1152  * Local variables:
1153  * c-basic-offset: 4
1154  * c-file-style: "Stroustrup"
1155  * indent-tabs-mode: nil
1156  * End:
1157  * vim: shiftwidth=4 tabstop=8 expandtab
1158  */
1159 
static ZEBRA_RES rec_cache_flush(Records p, int saveCount)
Definition: records.c:652
zint block_free[REC_BLOCK_TYPES]
Definition: records.c:83
static ZEBRA_RES rec_cache_insert(Records p, Record rec, enum recordCacheFlag flag)
Definition: records.c:692
void rec_free(Record *recpp)
frees record (from memory)
Definition: records.c:1043
#define ZEBRA_OK
Definition: util.h:82
#define REC_HEAD_MAGIC
Definition: records.c:55
zint block_size[REC_BLOCK_TYPES]
Definition: records.c:82
int rec_check_compression_method(int compression_method)
check whether a compression method is supported
Definition: records.c:286
recindex_t recindex
Definition: records.c:62
struct records_info::records_head head
void rec_prstat(Records records, int verbose)
Definition: records.c:1094
int zebra_mutex_unlock(Zebra_mutex *p)
Definition: zebra-lock.c:74
#define REC_NO_INFO
Definition: recindex.h:29
Record rec_new(Records p)
creates new record (to be written to file storage)
Definition: records.c:990
#define ZEBRA_FAIL
Definition: util.h:81
int tmp_size
Definition: records.c:68
Zebra Block File Layer.
int size
Definition: recindex.c:44
ZEBRA_RES rec_put(Records p, Record *recpp)
puts record (writes into file storage)
Definition: records.c:1022
enum recordCacheFlag flag
Definition: records.c:101
zint rec_sysno_to_int(zint sysno)
Definition: records.c:129
int recindex_read_indx(recindex_t p, zint sysno, void *buf, int itemsize, int ignoreError)
Definition: recindex.c:225
Zebra_mutex mutex
Definition: records.c:77
static ZEBRA_RES rec_write_multiple(Records p, int saveCount)
Definition: records.c:595
int zebra_mutex_destroy(Zebra_mutex *p)
Definition: zebra-lock.c:43
void recindex_close(recindex_t p)
closes record index handle
Definition: recindex.c:182
recordCacheFlag
Definition: records.c:96
char * tmp_buf
Definition: records.c:67
static void rec_decode_zint(zint *np, unsigned char *buf, int *len)
Definition: records.c:445
int recindex_read_head(recindex_t p, void *buf)
Definition: recindex.c:198
#define REC_COMPRESS_ZLIB
zlib compression (faster and works off small chunks)
Definition: recindex.h:115
size_t size[REC_NO_INFO]
Definition: recindex.h:35
static ZEBRA_RES rec_write_tmp_buf(Records p, int size, zint *sysnos)
Definition: records.c:224
zint next
Definition: recindex.c:43
struct records_info * Records
Definition: recindex.h:41
#define CAST_ZINT_TO_INT(x)
Definition: util.h:96
static ZEBRA_RES rec_release_blocks(Records p, zint sysno)
Definition: records.c:147
#define REC_VERSION
Definition: records.c:56
static void rec_encode_unsigned(unsigned n, unsigned char *buf, int *len)
Definition: records.c:402
Record rec
Definition: records.c:100
const char * recindex_get_fname(recindex_t p)
Definition: recindex.c:203
zint block_last[REC_BLOCK_TYPES]
Definition: records.c:84
Record rec_get_root(Records p)
gets root record
Definition: records.c:937
int compression_method
Definition: records.c:60
ZEBRA_RES rec_del(Records p, Record *recpp)
marks record for deletion (on file storage)
Definition: records.c:1000
char * info[REC_NO_INFO]
Definition: recindex.h:34
ZEBRA_RES recindex_write_head(recindex_t p, const void *buf, size_t len)
Definition: recindex.c:208
#define ZINT_FORMAT0
Definition: util.h:67
Record rec_get(Records p, zint sysno)
gets record - with given system number
Definition: records.c:927
Record rec_cp(Record rec)
Definition: records.c:1055
struct record_cache_entry * record_cache
Definition: records.c:70
char * data_fname[REC_BLOCK_TYPES]
Definition: records.c:64
void bf_close(BFile bf)
closes a Block file (may call exit)
Definition: bfile.c:151
static Record rec_get_int(Records p, zint sysno)
Definition: records.c:755
static void rec_tmp_expand(Records p, int size)
Definition: records.c:135
static zint rec_sysno_to_ext(zint sysno)
Definition: records.c:123
int zebra_mutex_lock(Zebra_mutex *p)
Definition: zebra-lock.c:59
#define REC_COMPRESS_BZIP2
BZIP2 compression (slow and requires big chunks)
Definition: recindex.h:113
Definition: recindex.c:42
int bf_read(BFile bf, zint no, int offset, int nbytes, void *buf)
read from block file (may call exit)
Definition: bfile.c:319
#define REC_COMPRESS_NONE
No compression ("none")
Definition: recindex.h:111
static ZEBRA_RES rec_flush_shared(Records p, short ref_count, zint *sysnos, char *out_buf, int out_offset)
Definition: records.c:512
ZEBRA_RES rec_close(Records *pp)
Definition: records.c:720
int cache_max
Definition: records.c:73
static Record * rec_cache_lookup(Records p, zint sysno, enum recordCacheFlag flag)
Definition: records.c:675
struct record_info * Record
zint block_move[REC_BLOCK_TYPES]
Definition: records.c:86
static Record rec_new_int(Records p)
Definition: records.c:957
zint sysno
Definition: recindex.h:32
BFile data_BFile[REC_BLOCK_TYPES]
Definition: records.c:65
Record rec_get_next(Records p, Record rec)
gets next record - with given records
Definition: records.c:942
int zebra_mutex_init(Zebra_mutex *p)
Definition: zebra-lock.c:31
long zint
Zebra integer.
Definition: util.h:66
#define FAKE_OFFSET
Definition: records.c:114
int cache_cur
Definition: records.c:72
int bf_write(BFile bf, zint no, int offset, int nbytes, const void *buf)
writes block of bytes to file (may call exit)
Definition: bfile.c:346
static void rec_encode_zint(zint n, unsigned char *buf, int *len)
Definition: records.c:432
Records rec_open(BFiles bfs, int rw, int compression_method)
opens records system
Definition: records.c:308
recindex_t recindex_open(BFiles bfs, int rw, int use_isamb)
opens record index handle
Definition: recindex.c:134
static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len)
Definition: records.c:415
#define REC_BLOCK_TYPES
Definition: records.c:54
Definition: records.c:99
static void rec_cache_flush_block1(Records p, Record rec, Record last_rec, char **out_buf, int *out_size, int *out_offset)
Definition: records.c:462
char * rec_strdup(const char *s, size_t *len)
Definition: records.c:1079
short ZEBRA_RES
Common return type for Zebra API.
Definition: util.h:80
int cache_size
Definition: records.c:71
zint block_used[REC_BLOCK_TYPES]
Definition: records.c:85
#define USUAL_RANGE
Definition: records.c:115
int compression_chunk_size
Definition: records.c:75
BFile bf_open(BFiles bfs, const char *name, int block_size, int wflag)
opens and returns a Block file handle
Definition: bfile.c:261
static ZEBRA_RES rec_delete_single(Records p, Record rec)
Definition: records.c:208
void recindex_write_indx(recindex_t p, zint sysno, void *buf, int itemsize)
Definition: recindex.c:312
#define ZINT_FORMAT
Definition: util.h:72