35 #include <yaz/xmalloc.h>
36 #include <yaz/snprintf.h>
48 char **thread_key_buf;
49 size_t thread_ptr_top;
53 pthread_mutex_t mutex;
55 pthread_cond_t work_available;
57 pthread_cond_t cond_sorting;
62 #define ENCODE_BUFLEN 768
69 #define USE_SHELLSORT 0
72 static void shellsort(
void *ar,
int r,
size_t s,
73 int (*cmp)(
const void *a,
const void *b))
78 static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936,
79 13776, 4592, 1968, 861, 336,
80 112, 48, 21, 7, 3, 1 };
81 for ( k = 0; k < 16; k++)
82 for (h = incs[k], i = h; i < r; i++)
86 while (j > h && (*cmp)(a + s*(j-h), v) > 0)
88 memcpy (a + s*j, a + s*(j-h), s);
106 char *bp = i->
buf, *bp0;
107 const char *src = (
char *) &key;
108 size_t klen = strlen(k);
110 if (fwrite (k, klen+1, 1,
outf) != 1)
112 yaz_log (YLOG_FATAL|YLOG_ERRNO,
"fwrite");
119 memcpy (&key, k+1,
sizeof(
struct it_key));
125 assert(key.
mem[0] >= 0);
130 *bp0 = (*k * 128) + bp - bp0 - 1;
131 if (fwrite (i->
buf, bp - i->
buf, 1,
outf) != 1)
133 yaz_log (YLOG_FATAL|YLOG_ERRNO,
"fwrite");
142 const char *src = bp0+1;
143 char *dst = (
char*) &key2;
160 char **key_buf,
size_t ptr_top,
size_t ptr_i);
162 #if YAZ_POSIX_THREADS
163 static void *thread_func(
void *vp)
168 pthread_mutex_lock(&p->mutex);
170 while (!p->is_sorting && !p->exit_flag)
171 pthread_cond_wait(&p->work_available, &p->mutex);
176 pthread_mutex_unlock(&p->mutex);
179 p->thread_ptr_top, p->thread_ptr_i);
181 pthread_mutex_lock(&p->mutex);
183 pthread_cond_signal(&p->cond_sorting);
184 pthread_mutex_unlock(&p->mutex);
186 pthread_mutex_unlock(&p->mutex);
196 #if YAZ_POSIX_THREADS
211 #if YAZ_POSIX_THREADS
215 pthread_mutex_init(&p->mutex, 0);
216 pthread_cond_init(&p->work_available, 0);
217 pthread_cond_init(&p->cond_sorting, 0);
218 pthread_create(&p->thread_id, 0, thread_func, p);
222 yaz_log(YLOG_DEBUG,
"key_block_create t=%d", p->
use_threads);
233 #if YAZ_POSIX_THREADS
234 pthread_mutex_lock(&p->mutex);
236 while (p->is_sorting)
237 pthread_cond_wait(&p->cond_sorting, &p->mutex);
241 pthread_cond_broadcast(&p->work_available);
243 pthread_mutex_unlock(&p->mutex);
244 pthread_join(p->thread_id, 0);
245 pthread_cond_destroy(&p->work_available);
246 pthread_cond_destroy(&p->cond_sorting);
247 pthread_mutex_destroy(&p->mutex);
260 int cmd,
const char *str_buf,
size_t str_len,
261 zint staticrank,
int static_rank_enable)
270 assert(p->
ptr_i > 0);
291 if (static_rank_enable)
293 assert(staticrank >= 0);
294 key_out.
mem[j++] = staticrank;
298 key_out.
mem[j++] = key_in->
mem[1];
300 key_out.
mem[j++] = sysno;
301 for (i = 2; i < key_in->
len; i++)
302 key_out.
mem[j++] = key_in->
mem[i];
306 &key_out,
sizeof(key_out));
312 char **key_buf,
size_t ptr_top,
size_t ptr_i)
323 yaz_log(YLOG_DEBUG,
"sorting section %d", (p->
key_file_no));
328 shellsort(key_buf + ptr_top - ptr_i, ptr_i,
331 qsort(key_buf + ptr_top - ptr_i, ptr_i,
334 yaz_snprintf(out_fname,
sizeof(out_fname),
"%s/key%d.tmp",
337 if (!(
outf = fopen (out_fname,
"wb")))
339 yaz_log (YLOG_FATAL|YLOG_ERRNO,
"fopen %s", out_fname);
342 yaz_log(YLOG_DEBUG,
"writing section %d", p->
key_file_no);
343 prevcp = cp = (key_buf)[ptr_top - ptr_i];
350 cp = (key_buf)[ptr_top - ptr_i];
351 if (strcmp (cp, prevcp))
364 yaz_log (YLOG_FATAL|YLOG_ERRNO,
"fclose %s", out_fname);
367 yaz_log(YLOG_DEBUG,
"finished section %d", p->
key_file_no);
377 #if YAZ_POSIX_THREADS
380 pthread_mutex_lock(&p->mutex);
382 while (p->is_sorting)
383 pthread_cond_wait(&p->cond_sorting, &p->mutex);
387 p->thread_ptr_top = p->
ptr_top;
388 p->thread_ptr_i = p->
ptr_i;
389 p->thread_key_buf = p->
key_buf;
395 pthread_cond_signal(&p->work_available);
399 while (p->is_sorting)
400 pthread_cond_wait(&p->cond_sorting, &p->mutex);
402 pthread_mutex_unlock(&p->mutex);
void iscz1_decode(void *vp, char **dst, const char **src)
void iscz1_encode(void *vp, char **dst, const char **src)
int key_qsort_compare(const void *p1, const void *p2)
void key_logdump_txt(int logmask, const void *p, const char *txt)
int key_SU_encode(int ch, char *out)
void key_block_flush(zebra_key_block_t p, int is_final)
void key_block_flush_int(zebra_key_block_t p, char **key_buf, size_t ptr_top, size_t ptr_i)
zebra_key_block_t key_block_create(size_t mem, const char *key_tmp_dir, int use_threads)
static void encode_key_flush(struct encode_info *i, FILE *outf)
void key_block_write(zebra_key_block_t p, zint sysno, struct it_key *key_in, int cmd, const char *str_buf, size_t str_len, zint staticrank, int static_rank_enable)
static void encode_key_init(struct encode_info *i)
void key_block_destroy(zebra_key_block_t *pp)
int key_block_get_no_files(zebra_key_block_t p)
static void encode_key_write(const char *k, struct encode_info *i, FILE *outf)
struct zebra_key_block * zebra_key_block_t
zint mem[IT_KEY_LEVEL_MAX]
void zebra_exit(const char *msg)
#define CAST_ZINT_TO_INT(x)