IDZEBRA  2.1.2
bfile.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 #include <assert.h>
27 #ifdef WIN32
28 #include <io.h>
29 #endif
30 
31 #if HAVE_UNISTD_H
32 #include <unistd.h>
33 #endif
34 
35 #include <yaz/xmalloc.h>
36 #include <idzebra/util.h>
37 #include <idzebra/bfile.h>
38 #include "mfile.h"
39 #include "cfile.h"
40 
42 {
45  struct CFile_struct *cf;
46  char *alloc_buf;
52  char *magic;
54 };
55 
56 struct BFiles_struct {
59  char *base;
60  char *cache_fname;
61 };
62 
63 BFiles bfs_create (const char *spec, const char *base)
64 {
65  BFiles bfs = (BFiles) xmalloc(sizeof(*bfs));
66  bfs->commit_area = 0;
67  bfs->base = 0;
68  bfs->cache_fname = 0;
69  if (base)
70  bfs->base = xstrdup(base);
71  bfs->register_area = mf_init("register", spec, base, 0);
72  if (!bfs->register_area)
73  {
74  bfs_destroy(bfs);
75  return 0;
76  }
77  return bfs;
78 }
79 
81 {
82  if (!bfs)
83  return;
84  xfree(bfs->cache_fname);
85  xfree(bfs->base);
86  mf_destroy(bfs->commit_area);
88  xfree(bfs);
89 }
90 
91 static FILE *open_cache(BFiles bfs, const char *flags)
92 {
93  FILE *file;
94 
95  file = fopen(bfs->cache_fname, flags);
96  return file;
97 }
98 
99 static void unlink_cache(BFiles bfs)
100 {
101  if (bfs->cache_fname)
102  unlink(bfs->cache_fname);
103 }
104 
105 ZEBRA_RES bf_cache(BFiles bfs, const char *spec)
106 {
107  if (spec)
108  {
109  yaz_log(YLOG_LOG, "enabling shadow spec=%s", spec);
110  if (!bfs->commit_area)
111  bfs->commit_area = mf_init("shadow", spec, bfs->base, 1);
112  if (bfs->commit_area)
113  {
114  bfs->cache_fname = xmalloc(strlen(bfs->commit_area->dirs->name)+
115  8);
116  strcpy(bfs->cache_fname, bfs->commit_area->dirs->name);
117  strcat(bfs->cache_fname, "/cache");
118  yaz_log(YLOG_LOG, "cache_fname = %s", bfs->cache_fname);
119  }
120  else
121  {
122  yaz_log(YLOG_WARN, "shadow could not be enabled");
123  return ZEBRA_FAIL;
124  }
125  }
126  else
127  bfs->commit_area = 0;
128  return ZEBRA_OK;
129 }
130 
132 {
133  int ret = 0;
135  if (bf->cf)
136  {
137  if (cf_close(bf->cf))
138  ret = -1;
139  }
140  if (bf->mf)
141  {
142  if (mf_close(bf->mf))
143  ret = -1;
144  }
145  xfree(bf->alloc_buf);
146  xfree(bf->magic);
147  xfree(bf);
148  return ret;
149 }
150 
151 void bf_close(BFile bf)
152 {
153  if (bf_close2(bf))
154  {
155  zebra_exit("bf_close");
156  }
157 }
158 
159 
160 #define HEADER_SIZE 256
161 
162 BFile bf_xopen(BFiles bfs, const char *name, int block_size, int wrflag,
163  const char *magic, int *read_version,
164  const char **more_info)
165 {
166  char read_magic[40];
167  int l = 0;
168  int i = 0;
169  char *hbuf;
170  zint pos = 0;
171  BFile bf = bf_open(bfs, name, block_size, wrflag);
172 
173  if (!bf)
174  return 0;
175  /* HEADER_SIZE is considered enough for our header */
176  if (bf->block_size < HEADER_SIZE)
178  else
179  bf->alloc_buf_size = bf->block_size;
180 
181  hbuf = bf->alloc_buf = xmalloc(bf->alloc_buf_size);
182 
183  /* fill-in default values */
184  bf->free_list = 0;
185  bf->root_block = bf->last_block = HEADER_SIZE / bf->block_size + 1;
186  bf->magic = xstrdup(magic);
187 
188  if (!bf_read(bf, pos, 0, 0, hbuf + pos * bf->block_size))
189  {
190  if (wrflag)
191  bf->header_dirty = 1;
192  return bf;
193  }
194  while(hbuf[pos * bf->block_size + i] != '\0')
195  {
196  if (i == bf->block_size)
197  { /* end of block at pos reached .. */
198  if (bf->alloc_buf_size < (pos+1) * bf->block_size)
199  {
200  /* not room for all in alloc_buf_size */
201  yaz_log(YLOG_WARN, "bad header for %s (3)", magic);
202  bf_close(bf);
203  return 0;
204  }
205  /* read next block in header */
206  pos++;
207  if (!bf_read(bf, pos, 0, 0, hbuf + pos * bf->block_size))
208  {
209  yaz_log(YLOG_WARN, "missing header block %s (4)", magic);
210  bf_close(bf);
211  return 0;
212  }
213  i = 0; /* pos within block is reset */
214  }
215  else
216  i++;
217  }
218  if (sscanf(hbuf, "%39s %d " ZINT_FORMAT " " ZINT_FORMAT "%n",
219  read_magic, read_version, &bf->last_block,
220  &bf->free_list, &l) < 4 && l) /* if %n is counted, it's 5 */
221  {
222  yaz_log(YLOG_WARN, "bad header for %s (1)", magic);
223  bf_close(bf);
224  return 0;
225  }
226  if (strcmp(read_magic, magic))
227  {
228  yaz_log(YLOG_WARN, "bad header for %s (2)", magic);
229  bf_close(bf);
230  return 0;
231  }
232  if (hbuf[l] == ' ')
233  l++;
234  if (more_info)
235  *more_info = hbuf + l;
236  return bf;
237 }
238 
239 int bf_xclose(BFile bf, int version, const char *more_info)
240 {
241  if (bf->header_dirty)
242  {
243  zint pos = 0;
244  assert(bf->alloc_buf);
245  assert(bf->magic);
246  sprintf(bf->alloc_buf, "%s %d " ZINT_FORMAT " " ZINT_FORMAT " ",
247  bf->magic, version, bf->last_block, bf->free_list);
248  if (more_info)
249  strcat(bf->alloc_buf, more_info);
250  while (1)
251  {
252  bf_write(bf, pos, 0, 0, bf->alloc_buf + pos * bf->block_size);
253  pos++;
254  if (pos * bf->block_size > strlen(bf->alloc_buf))
255  break;
256  }
257  }
258  return bf_close2(bf);
259 }
260 
261 BFile bf_open(BFiles bfs, const char *name, int block_size, int wflag)
262 {
263  BFile bf = (BFile) xmalloc(sizeof(*bf));
264 
265  bf->alloc_buf = 0;
266  bf->magic = 0;
267  bf->block_size = block_size;
268  bf->header_dirty = 0;
269  bf->cf = 0;
270  bf->mf = 0;
272 
273  if (bfs->commit_area)
274  {
275  int first_time;
276 
277  bf->mf = mf_open(bfs->register_area, name, block_size, 0);
278  bf->cf = cf_open(bf->mf, bfs->commit_area, name, block_size,
279  wflag, &first_time);
280  if (!bf->cf)
281  {
282  yaz_log(YLOG_FATAL, "cf_open failed for %s", name);
283  bf_close(bf);
284  return 0;
285  }
286  if (first_time)
287  {
288  FILE *outf;
289 
290  outf = open_cache(bfs, "ab");
291  if (!outf)
292  {
293  yaz_log(YLOG_FATAL|YLOG_ERRNO, "open %s", bfs->cache_fname);
294  bf_close(bf);
295  return 0;
296  }
297  fprintf(outf, "%s %d\n", name, block_size);
298  if (fclose(outf))
299  {
300  yaz_log(YLOG_FATAL|YLOG_ERRNO, "fclose %s", bfs->cache_fname);
301  bf_close(bf);
302  return 0;
303  }
304  }
305  }
306  else
307  {
308  bf->mf = mf_open(bfs->register_area, name, block_size, wflag);
309  }
310  if (!bf->mf)
311  {
312  yaz_log(YLOG_FATAL, "mf_open failed for %s", name);
313  bf_close(bf);
314  return 0;
315  }
316  return bf;
317 }
318 
319 int bf_read(BFile bf, zint no, int offset, int nbytes, void *buf)
320 {
321  int ret = bf_read2(bf, no, offset, nbytes, buf);
322 
323  if (ret == -1)
324  {
325  zebra_exit("bf_read");
326  }
327  return ret;
328 }
329 
330 int bf_read2(BFile bf, zint no, int offset, int nbytes, void *buf)
331 {
332  int ret;
333 
335  if (bf->cf)
336  {
337  if ((ret = cf_read(bf->cf, no, offset, nbytes, buf)) == 0)
338  ret = mf_read(bf->mf, no, offset, nbytes, buf);
339  }
340  else
341  ret = mf_read(bf->mf, no, offset, nbytes, buf);
343  return ret;
344 }
345 
346 int bf_write(BFile bf, zint no, int offset, int nbytes, const void *buf)
347 {
348  int ret = bf_write2(bf, no, offset, nbytes, buf);
349 
350  if (ret == -1)
351  {
352  zebra_exit("bf_write");
353  }
354  return ret;
355 }
356 
357 int bf_write2(BFile bf, zint no, int offset, int nbytes, const void *buf)
358 {
359  int r;
361  if (bf->cf)
362  r = cf_write(bf->cf, no, offset, nbytes, buf);
363  else
364  r = mf_write(bf->mf, no, offset, nbytes, buf);
366  return r;
367 }
368 
370 {
371  FILE *inf;
372 
373  inf = open_cache(bfs, "rb");
374  if (inf)
375  {
376  fclose(inf);
377  return 1;
378  }
379  return 0;
380 }
381 
382 void bf_reset(BFiles bfs)
383 {
384  if (!bfs)
385  return;
386  mf_reset(bfs->commit_area, 1);
387  mf_reset(bfs->register_area, 1);
388  unlink_cache(bfs);
389 }
390 
392 {
393  FILE *inf;
394  int block_size;
395  char path[256];
396  MFile mf;
397  CFile cf;
398  int first_time;
399  int r = 0;
400 
401  assert(bfs->commit_area);
402  if (!(inf = open_cache(bfs, "rb")))
403  {
404  yaz_log(YLOG_LOG, "No commit file");
405  return -1;
406  }
407  while (fscanf(inf, "%s %d", path, &block_size) == 2)
408  {
409  mf = mf_open(bfs->register_area, path, block_size, 1);
410  if (!mf)
411  {
412  r = -1;
413  break;
414  }
415  cf = cf_open(mf, bfs->commit_area, path, block_size, 0, &first_time);
416  if (!cf)
417  {
418  mf_close(mf);
419  r = -1;
420  break;
421  }
422 
423  r = cf_commit(cf);
424 
425  cf_close(cf);
426  mf_close(mf);
427 
428  if (r)
429  break;
430  }
431  fclose(inf);
432  return r;
433 }
434 
435 void bf_commitClean(BFiles bfs, const char *spec)
436 {
437  int mustDisable = 0;
438 
439  if (!bfs->commit_area)
440  {
441  bf_cache(bfs, spec);
442  mustDisable = 1;
443  }
444 
445  mf_reset(bfs->commit_area, 1);
446 
447  unlink_cache(bfs);
448  if (mustDisable)
449  bf_cache(bfs, 0);
450 }
451 
452 int bf_alloc(BFile bf, int no, zint *blocks)
453 {
454  int i;
455  assert(bf->alloc_buf);
456  bf->header_dirty = 1;
457  for (i = 0; i < no; i++)
458  {
459  if (!bf->free_list)
460  blocks[i] = bf->last_block++;
461  else
462  {
463  char buf[16];
464  const char *cp = buf;
465  memset(buf, '\0', sizeof(buf));
466 
467  blocks[i] = bf->free_list;
468  if (bf_read(bf, bf->free_list, 0, sizeof(buf), buf) != 1)
469  {
470  yaz_log(YLOG_WARN, "Bad freelist entry " ZINT_FORMAT,
471  bf->free_list);
472  return -1;
473  }
474  zebra_zint_decode(&cp, &bf->free_list);
475  }
476  }
477  return 0;
478 }
479 
480 int bf_free(BFile bf, int no, const zint *blocks)
481 {
482  int i;
483  assert(bf->alloc_buf);
484  bf->header_dirty = 1;
485  for (i = 0; i < no; i++)
486  {
487  char buf[16];
488  char *cp = buf;
489  memset(buf, '\0', sizeof(buf));
490  zebra_zint_encode(&cp, bf->free_list);
491  bf->free_list = blocks[i];
492  bf_write(bf, bf->free_list, 0, sizeof(buf), buf);
493  }
494  return 0;
495 }
496 
497 int bfs_register_directory_stat(BFiles bfs, int no, const char **directory,
498  double *used_bytes, double *max_bytes)
499 {
500  return mf_area_directory_stat(bfs->register_area, no, directory,
501  used_bytes, max_bytes);
502 }
503 
504 
505 int bfs_shadow_directory_stat(BFiles bfs, int no, const char **directory,
506  double *used_bytes, double *max_bytes)
507 {
508  if (!bfs->commit_area)
509  return 0;
510  return mf_area_directory_stat(bfs->commit_area, no, directory,
511  used_bytes, max_bytes);
512 }
513 /*
514  * Local variables:
515  * c-basic-offset: 4
516  * c-file-style: "Stroustrup"
517  * indent-tabs-mode: nil
518  * End:
519  * vim: shiftwidth=4 tabstop=8 expandtab
520  */
521 
static FILE * outf
Definition: readfile.c:38
int zebra_lock_rdwr_wlock(Zebra_lock_rdwr *p)
Definition: zebra-lock.c:123
int header_dirty
Definition: bfile.c:53
int zebra_lock_rdwr_runlock(Zebra_lock_rdwr *p)
Definition: zebra-lock.c:135
int zebra_lock_rdwr_rlock(Zebra_lock_rdwr *p)
Definition: zebra-lock.c:111
#define ZEBRA_OK
Definition: util.h:82
MFile mf
Definition: bfile.c:43
zint free_list
Definition: bfile.c:50
int bf_close2(BFile bf)
closes a Block file
Definition: bfile.c:131
struct BFile_struct * BFile
A Block File.
Definition: bfile.h:43
BFile bf_xopen(BFiles bfs, const char *name, int block_size, int wrflag, const char *magic, int *read_version, const char **more_info)
opens and returns an extended Block file handle
Definition: bfile.c:162
static FILE * inf
Definition: readfile.c:37
void bf_commitClean(BFiles bfs, const char *spec)
Cleans shadow files (remove them)
Definition: bfile.c:435
MFile_area mf_init(const char *name, const char *spec, const char *base, int only_shadow_files)
creates a metafile area
Definition: mfile.c:195
#define ZEBRA_FAIL
Definition: util.h:81
Zebra Block File Layer.
CFile cf_open(MFile mf, MFile_area area, const char *fname, int block_size, int wflag, int *firstp)
Definition: cfile.c:81
int bfs_register_directory_stat(BFiles bfs, int no, const char **directory, double *used_bytes, double *max_bytes)
Definition: bfile.c:497
#define HEADER_SIZE
Definition: bfile.c:160
MFile_area register_area
Definition: bfile.c:58
int mf_close(MFile mf)
closes metafile
Definition: mfile.c:429
static FILE * open_cache(BFiles bfs, const char *flags)
Definition: bfile.c:91
int cf_close(CFile cf)
Definition: cfile.c:615
void bf_reset(BFiles bfs)
Removes register and shadow completely.
Definition: bfile.c:382
int bf_write(BFile bf, zint no, int offset, int nbytes, const void *buf)
writes block of bytes to file (may call exit)
Definition: bfile.c:346
void bfs_destroy(BFiles bfs)
destroys a block files handle
Definition: bfile.c:80
int cf_write(CFile cf, zint no, int offset, int nbytes, const void *buf)
writes block to commit area
Definition: cfile.c:578
void mf_destroy(MFile_area ma)
destroys metafile area handle
Definition: mfile.c:307
int zebra_lock_rdwr_wunlock(Zebra_lock_rdwr *p)
Definition: zebra-lock.c:155
void zebra_zint_decode(const char **src, zint *pos)
Definition: zint.c:39
int alloc_buf_size
Definition: bfile.c:48
int mf_write(MFile mf, zint no, int offset, int nbytes, const void *buf)
writes block to metafile
Definition: mfile.c:484
int bfs_shadow_directory_stat(BFiles bfs, int no, const char **directory, double *used_bytes, double *max_bytes)
Definition: bfile.c:505
char * alloc_buf
Definition: bfile.c:46
int cf_commit(CFile cf) ZEBRA_GCC_ATTR((warn_unused_result))
Definition: commit.c:257
char * base
Definition: bfile.c:59
int bf_free(BFile bf, int no, const zint *blocks)
Releases one or more blocks in an extended block file.
Definition: bfile.c:480
All in-memory information per CFile.
Definition: cfile.h:68
BFiles bfs_create(const char *spec, const char *base)
creates a Block files collection
Definition: bfile.c:63
void zebra_exit(const char *msg)
Definition: exit.c:26
void zebra_zint_encode(char **dst, zint pos)
Definition: zint.c:26
zint last_block
Definition: bfile.c:49
int mf_area_directory_stat(MFile_area ma, int no, const char **directory, double *used_bytes, double *max_bytes)
metafile area statistics
Definition: mfile.c:609
char * cache_fname
Definition: bfile.c:60
void mf_reset(MFile_area ma, int unlink_flag)
reset all files in a metafile area (optionally delete them as well)
Definition: mfile.c:324
int bf_commitExists(BFiles bfs)
Check if there is content in shadow area (to be committed).
Definition: bfile.c:369
int zebra_lock_rdwr_init(Zebra_lock_rdwr *p)
Definition: zebra-lock.c:89
Zebra_lock_rdwr rdwr_lock
Definition: bfile.c:44
int bf_xclose(BFile bf, int version, const char *more_info)
closes an extended Block file handle..
Definition: bfile.c:239
char * magic
Definition: bfile.c:52
BFile bf_open(BFiles bfs, const char *name, int block_size, int wflag)
opens and returns a Block file handle
Definition: bfile.c:261
int mf_read(MFile mf, zint no, int offset, int nbytes, void *buf)
reads block from metafile
Definition: mfile.c:451
static void unlink_cache(BFiles bfs)
Definition: bfile.c:99
void bf_close(BFile bf)
closes a Block file (may call exit)
Definition: bfile.c:151
int cf_read(CFile cf, zint no, int offset, int nbytes, void *buf)
reads block from commit area
Definition: cfile.c:541
long zint
Zebra integer.
Definition: util.h:66
ZEBRA_RES bf_cache(BFiles bfs, const char *spec)
enables or disables shadow for block files
Definition: bfile.c:105
MFile mf_open(MFile_area ma, const char *name, int block_size, int wflag)
opens metafile
Definition: mfile.c:351
int bf_commitExec(BFiles bfs)
Executes commit operation.
Definition: bfile.c:391
int bf_alloc(BFile bf, int no, zint *blocks)
Allocates one or more blocks in an extended block file.
Definition: bfile.c:452
int bf_read2(BFile bf, zint no, int offset, int nbytes, void *buf)
read from block file
Definition: bfile.c:330
int zebra_lock_rdwr_destroy(Zebra_lock_rdwr *p)
Definition: zebra-lock.c:100
int bf_write2(BFile bf, zint no, int offset, int nbytes, const void *buf)
writes block of bytes to file
Definition: bfile.c:357
int bf_read(BFile bf, zint no, int offset, int nbytes, void *buf)
read from block file (may call exit)
Definition: bfile.c:319
short ZEBRA_RES
Common return type for Zebra API.
Definition: util.h:80
int block_size
Definition: bfile.c:47
mf_dir * dirs
Definition: mfile.h:96
char name[FILENAME_MAX+1]
Definition: mfile.h:57
struct BFiles_struct * BFiles
A collection of BFile(s).
Definition: bfile.h:38
zint root_block
Definition: bfile.c:51
struct CFile_struct * cf
Definition: bfile.c:45
MFile_area commit_area
Definition: bfile.c:57
#define ZINT_FORMAT
Definition: util.h:72