Kouhei Sutou
null+****@clear*****
Tue Mar 8 19:26:28 JST 2016
Kouhei Sutou 2016-03-08 19:26:28 +0900 (Tue, 08 Mar 2016) New Revision: 0fef6f1d18c6eba3d75cc3fad6630793af787431 https://github.com/groonga/groonga/commit/0fef6f1d18c6eba3d75cc3fad6630793af787431 Merged e25859e: Merge pull request #498 from groonga/support-4gib-over-total-key-size-in-hash Message: hash: support 4GiB over total key size If you add KEY_LARGE flag to table: table_create Users TABLE_HASH_KEY|KEY_LARGE ShortText you can put many keys to the table. By default, 4GiB is the max total key size. With KEY_LARGE flag, 1TiB is the max total key size. Because 4KiB * (2 ** 28 - 1) < 1TiB. ((2 ** 28 - 1) is the max number of records in a table.) It doesn't break backward compatibility. You can use the existing database without dump and restore. Added files: test/command/suite/dump/schema/table/hash/key_large_flag.expected test/command/suite/dump/schema/table/hash/key_large_flag.test test/command/suite/table_create/key_large/hash_key.expected test/command/suite/table_create/key_large/hash_key.test test/command/suite/table_create/key_large/patricia_trie.expected test/command/suite/table_create/key_large/patricia_trie.test Modified files: include/groonga/groonga.h lib/db.c lib/dump.c lib/grn_hash.h lib/hash.c lib/proc/proc_table.c Modified: include/groonga/groonga.h (+2 -0) =================================================================== --- include/groonga/groonga.h 2016-03-08 16:12:16 +0900 (d6cb07f) +++ include/groonga/groonga.h 2016-03-08 19:26:28 +0900 (147479c) @@ -335,6 +335,8 @@ typedef uint16_t grn_obj_flags; #define GRN_OBJ_UNIT_USERDEF_SECTION (0x07<<8) #define GRN_OBJ_UNIT_USERDEF_POSITION (0x08<<8) +#define GRN_OBJ_KEY_LARGE (0x01<<12) + #define GRN_OBJ_NO_SUBREC (0x00<<13) #define GRN_OBJ_WITH_SUBREC (0x01<<13) Modified: lib/db.c (+11 -0) =================================================================== --- lib/db.c 2016-03-08 16:12:16 +0900 (651e06a) +++ lib/db.c 2016-03-08 19:26:28 +0900 (320611d) @@ -933,6 +933,17 @@ grn_table_create_validate(grn_ctx *ctx, const char *name, unsigned int name_size return ctx->rc; } + if ((flags & GRN_OBJ_KEY_LARGE) && + table_type != GRN_OBJ_TABLE_HASH_KEY) { + ERR(GRN_INVALID_ARGUMENT, + "[table][create] " + "large key support is available only for TABLE_HASH_KEY key table: " + "<%.*s>(%s)", + name_size, name, + table_type_name); + return ctx->rc; + } + return ctx->rc; } Modified: lib/dump.c (+3 -0) =================================================================== --- lib/dump.c 2016-03-08 16:12:16 +0900 (3d44731) +++ lib/dump.c 2016-03-08 19:26:28 +0900 (594213c) @@ -39,6 +39,9 @@ grn_dump_table_create_flags(grn_ctx *ctx, GRN_TEXT_PUTS(ctx, buffer, "TABLE_NO_KEY"); break; } + if (flags & GRN_OBJ_KEY_LARGE) { + GRN_TEXT_PUTS(ctx, buffer, "|KEY_LARGE"); + } if (flags & GRN_OBJ_KEY_WITH_SIS) { GRN_TEXT_PUTS(ctx, buffer, "|KEY_WITH_SIS"); } Modified: lib/grn_hash.h (+5 -2) =================================================================== --- lib/grn_hash.h 2016-03-08 16:12:16 +0900 (3a0637c) +++ lib/grn_hash.h 2016-03-08 19:26:28 +0900 (1048252) @@ -254,7 +254,7 @@ struct _grn_hash { uint32_t value_size;\ grn_id tokenizer;\ uint32_t curr_rec;\ - int32_t curr_key;\ + uint32_t curr_key;\ uint32_t idx_offset;\ uint32_t entry_size;\ uint32_t max_offset;\ @@ -263,7 +263,8 @@ struct _grn_hash { uint32_t lock;\ grn_id normalizer;\ uint32_t truncated;\ - uint32_t reserved[14] + uint64_t curr_key_large;\ + uint32_t reserved[12] struct _grn_hash_header_common { GRN_HASH_HEADER_COMMON_FIELDS; @@ -365,6 +366,8 @@ grn_id grn_array_at(grn_ctx *ctx, grn_array *array, grn_id id); void grn_hash_check(grn_ctx *ctx, grn_hash *hash); +grn_bool grn_hash_is_large_total_key_size(grn_ctx *ctx, grn_hash *hash); + uint64_t grn_hash_total_key_size(grn_ctx *ctx, grn_hash *hash); uint64_t grn_hash_max_total_key_size(grn_ctx *ctx, grn_hash *hash); Modified: lib/hash.c (+186 -62) =================================================================== --- lib/hash.c 2016-03-08 16:12:16 +0900 (4b4d3fa) +++ lib/hash.c 2016-03-08 19:26:28 +0900 (500dd77) @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2009-2015 Brazil + Copyright(C) 2009-2016 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -262,7 +262,7 @@ grn_tiny_bitmap_put_and_set(grn_tiny_bitmap *bitmap, grn_id bit_id, inline static void * grn_io_array_at_inline(grn_ctx *ctx, grn_io *io, uint32_t segment_id, - uint32_t offset, int flags) + uint64_t offset, int flags) { void *ptr; GRN_IO_ARRAY_AT(io, segment_id, offset, &flags, ptr); @@ -1244,9 +1244,13 @@ grn_array_unblock(grn_ctx *ctx, grn_array *array) (GRN_HASH_MAX_KEY_SIZE_LARGE - GRN_HASH_MAX_KEY_SIZE_NORMAL))) #define GRN_HASH_SEGMENT_SIZE 0x400000 #define GRN_HASH_KEY_MAX_N_SEGMENTS 0x400 +#define GRN_HASH_KEY_MAX_N_SEGMENTS_LARGE 0x40000 #define W_OF_KEY_IN_A_SEGMENT 22 #define GRN_HASH_KEY_MAX_TOTAL_SIZE\ (((uint64_t)(1) << W_OF_KEY_IN_A_SEGMENT) * GRN_HASH_KEY_MAX_N_SEGMENTS - 1) +#define GRN_HASH_KEY_MAX_TOTAL_SIZE_LARGE\ + (((uint64_t)(1) << W_OF_KEY_IN_A_SEGMENT) *\ + GRN_HASH_KEY_MAX_N_SEGMENTS_LARGE - 1) #define IDX_MASK_IN_A_SEGMENT 0xfffff typedef struct { @@ -1275,6 +1279,17 @@ typedef struct { uint16_t flag; uint16_t key_size; union { + uint8_t buf[sizeof(uint64_t)]; + uint64_t offset; + } key; + uint8_t value[1]; +} grn_io_hash_entry_large; + +typedef struct { + uint32_t hash_value; + uint16_t flag; + uint16_t key_size; + union { uint8_t buf[sizeof(void *)]; void *ptr; } key; @@ -1298,6 +1313,7 @@ typedef union { grn_plain_hash_entry plain_entry; grn_rich_hash_entry rich_entry; grn_io_hash_entry io_entry; + grn_io_hash_entry_large io_entry_large; grn_tiny_hash_entry tiny_entry; } grn_hash_entry; @@ -1396,7 +1412,7 @@ grn_hash_idx_at(grn_ctx *ctx, grn_hash *hash, grn_id id) } inline static void * -grn_io_hash_key_at(grn_ctx *ctx, grn_hash *hash, uint32_t pos) +grn_io_hash_key_at(grn_ctx *ctx, grn_hash *hash, uint64_t pos) { return grn_io_array_at_inline(ctx, hash->io, GRN_HASH_KEY_SEGMENT, pos, GRN_TABLE_ADD); @@ -1421,10 +1437,20 @@ grn_hash_entry_get_key(grn_ctx *ctx, grn_hash *hash, grn_hash_entry *entry) { if (hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) { if (grn_hash_is_io_hash(hash)) { - if (entry->io_entry.flag & HASH_IMMEDIATE) { - return (char *)entry->io_entry.key.buf; + if (grn_hash_is_large_total_key_size(ctx, hash)) { + if (entry->io_entry_large.flag & HASH_IMMEDIATE) { + return (char *)entry->io_entry_large.key.buf; + } else { + return (char *)grn_io_hash_key_at(ctx, hash, + entry->io_entry_large.key.offset); + } } else { - return (char *)grn_io_hash_key_at(ctx, hash, entry->io_entry.key.offset); + if (entry->io_entry.flag & HASH_IMMEDIATE) { + return (char *)entry->io_entry.key.buf; + } else { + return (char *)grn_io_hash_key_at(ctx, hash, + entry->io_entry.key.offset); + } } } else { if (entry->tiny_entry.flag & HASH_IMMEDIATE) { @@ -1443,11 +1469,15 @@ grn_hash_entry_get_key(grn_ctx *ctx, grn_hash *hash, grn_hash_entry *entry) } inline static void * -grn_hash_entry_get_value(grn_hash *hash, grn_hash_entry *entry) +grn_hash_entry_get_value(grn_ctx *ctx, grn_hash *hash, grn_hash_entry *entry) { if (hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) { if (grn_hash_is_io_hash(hash)) { - return entry->io_entry.value; + if (grn_hash_is_large_total_key_size(ctx, hash)) { + return entry->io_entry_large.value; + } else { + return entry->io_entry.value; + } } else { return entry->tiny_entry.value; } @@ -1462,15 +1492,34 @@ grn_hash_entry_get_value(grn_hash *hash, grn_hash_entry *entry) inline static grn_rc grn_io_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, - grn_io_hash_entry *entry, + grn_hash_entry *entry, const void *key, unsigned int key_size) { - uint32_t key_offset; - if (entry->key_size) { - key_offset = entry->key.offset; + grn_bool is_large_mode; + grn_bool key_exist; + uint64_t key_offset; + grn_io_hash_entry *io_entry = &(entry->io_entry); + grn_io_hash_entry_large *io_entry_large = &(entry->io_entry_large); + + is_large_mode = grn_hash_is_large_total_key_size(ctx, hash); + + if (is_large_mode) { + key_exist = (io_entry_large->key_size > 0); + } else { + key_exist = (io_entry->key_size > 0); + } + + if (key_exist > 0) { + if (is_large_mode) { + key_offset = io_entry_large->key.offset; + } else { + key_offset = io_entry->key.offset; + } } else { - uint32_t segment_id; + uint64_t segment_id; grn_hash_header_common *header; + uint64_t curr_key; + uint64_t max_total_size; header = hash->header.common; if (key_size >= GRN_HASH_SEGMENT_SIZE) { @@ -1484,26 +1533,47 @@ grn_io_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, key_size); return ctx->rc; } - if (key_size > (GRN_HASH_KEY_MAX_TOTAL_SIZE - header->curr_key)) { + + if (is_large_mode) { + curr_key = header->curr_key_large; + max_total_size = GRN_HASH_KEY_MAX_TOTAL_SIZE_LARGE; + } else { + curr_key = header->curr_key; + max_total_size = GRN_HASH_KEY_MAX_TOTAL_SIZE; + } + + if (key_size > (max_total_size - curr_key)) { char name[GRN_TABLE_MAX_KEY_SIZE]; int name_size; name_size = grn_hash_name(ctx, hash, name, GRN_TABLE_MAX_KEY_SIZE); ERR(GRN_NOT_ENOUGH_SPACE, "[hash][key][put] total key size is over: <%.*s>: " - "max=%" GRN_FMT_INT64U ": current=%u: new key size=%u", + "max=%" GRN_FMT_INT64U ": " + "current=%" GRN_FMT_INT64U ": " + "new key size=%u", name_size, name, - GRN_HASH_KEY_MAX_TOTAL_SIZE, - header->curr_key, + max_total_size, + curr_key, key_size); return ctx->rc; } - key_offset = header->curr_key; + key_offset = curr_key; segment_id = (key_offset + key_size) >> W_OF_KEY_IN_A_SEGMENT; if ((key_offset >> W_OF_KEY_IN_A_SEGMENT) != segment_id) { - key_offset = header->curr_key = segment_id << W_OF_KEY_IN_A_SEGMENT; + key_offset = segment_id << W_OF_KEY_IN_A_SEGMENT; + if (is_large_mode) { + header->curr_key_large = key_offset; + } else { + header->curr_key = key_offset; + } + } + if (is_large_mode) { + header->curr_key_large += key_size; + io_entry_large->key.offset = key_offset; + } else { + header->curr_key += key_size; + io_entry->key.offset = key_offset; } - header->curr_key += key_size; - entry->key.offset = key_offset; } { @@ -1514,7 +1584,8 @@ grn_io_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, name_size = grn_hash_name(ctx, hash, name, GRN_TABLE_MAX_KEY_SIZE); ERR(GRN_NO_MEMORY_AVAILABLE, "[hash][key][put] failed to allocate for new key: <%.*s>: " - "new offset:%u key size=%u", + "new offset:%" GRN_FMT_INT64U " " + "key size:%u", name_size, name, key_offset, key_size); @@ -1532,20 +1603,35 @@ grn_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, { if (hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) { if (grn_hash_is_io_hash(hash)) { - if (key_size <= sizeof(entry->io_entry.key.buf)) { - grn_memcpy(entry->io_entry.key.buf, key, key_size); - entry->io_entry.flag = HASH_IMMEDIATE; + if (grn_hash_is_large_total_key_size(ctx, hash)) { + if (key_size <= sizeof(entry->io_entry_large.key.buf)) { + grn_memcpy(entry->io_entry_large.key.buf, key, key_size); + entry->io_entry_large.flag = HASH_IMMEDIATE; + } else { + const grn_rc rc = + grn_io_hash_entry_put_key(ctx, hash, entry, key, key_size); + if (rc) { + return rc; + } + entry->io_entry_large.flag = 0; + } + entry->io_entry_large.hash_value = hash_value; + entry->io_entry_large.key_size = key_size; } else { - const grn_rc rc = - grn_io_hash_entry_put_key(ctx, hash, (grn_io_hash_entry *)entry, - key, key_size); - if (rc) { - return rc; + if (key_size <= sizeof(entry->io_entry.key.buf)) { + grn_memcpy(entry->io_entry.key.buf, key, key_size); + entry->io_entry.flag = HASH_IMMEDIATE; + } else { + const grn_rc rc = + grn_io_hash_entry_put_key(ctx, hash, entry, key, key_size); + if (rc) { + return rc; + } + entry->io_entry.flag = 0; } - entry->io_entry.flag = 0; + entry->io_entry.hash_value = hash_value; + entry->io_entry.key_size = key_size; } - entry->io_entry.hash_value = hash_value; - entry->io_entry.key_size = key_size; } else { if (key_size <= sizeof(entry->tiny_entry.key.buf)) { grn_memcpy(entry->tiny_entry.key.buf, key, key_size); @@ -1588,12 +1674,22 @@ grn_hash_entry_compare_key(grn_ctx *ctx, grn_hash *hash, return GRN_FALSE; } if (grn_hash_is_io_hash(hash)) { - if (entry->io_entry.flag & HASH_IMMEDIATE) { - return !memcmp(key, entry->io_entry.key.buf, key_size); + if (grn_hash_is_large_total_key_size(ctx, hash)) { + if (entry->io_entry_large.flag & HASH_IMMEDIATE) { + return !memcmp(key, entry->io_entry_large.key.buf, key_size); + } else { + const void * const entry_key_ptr = + grn_io_hash_key_at(ctx, hash, entry->io_entry_large.key.offset); + return !memcmp(key, entry_key_ptr, key_size); + } } else { - const void * const entry_key_ptr = - grn_io_hash_key_at(ctx, hash, entry->io_entry.key.offset); - return !memcmp(key, entry_key_ptr, key_size); + if (entry->io_entry.flag & HASH_IMMEDIATE) { + return !memcmp(key, entry->io_entry.key.buf, key_size); + } else { + const void * const entry_key_ptr = + grn_io_hash_key_at(ctx, hash, entry->io_entry.key.offset); + return !memcmp(key, entry_key_ptr, key_size); + } } } else { if (entry->tiny_entry.flag & HASH_IMMEDIATE) { @@ -1621,9 +1717,9 @@ get_key(grn_ctx *ctx, grn_hash *hash, entry_str *n) } inline static void * -get_value(grn_hash *hash, entry_str *n) +get_value(grn_ctx *ctx, grn_hash *hash, entry_str *n) { - return grn_hash_entry_get_value(hash, (grn_hash_entry *)n); + return grn_hash_entry_get_value(ctx, hash, (grn_hash_entry *)n); } inline static grn_rc @@ -1661,7 +1757,8 @@ grn_io_hash_calculate_entry_size(uint32_t key_size, uint32_t value_size, static grn_io * grn_io_hash_create_io(grn_ctx *ctx, const char *path, - uint32_t header_size, uint32_t entry_size) + uint32_t header_size, uint32_t entry_size, + uint32_t flags) { uint32_t w_of_element = 0; grn_io_array_spec array_spec[4]; @@ -1671,7 +1768,13 @@ grn_io_hash_create_io(grn_ctx *ctx, const char *path, } array_spec[GRN_HASH_KEY_SEGMENT].w_of_element = 0; - array_spec[GRN_HASH_KEY_SEGMENT].max_n_segments = GRN_HASH_KEY_MAX_N_SEGMENTS; + if (flags & GRN_OBJ_KEY_LARGE) { + array_spec[GRN_HASH_KEY_SEGMENT].max_n_segments = + GRN_HASH_KEY_MAX_N_SEGMENTS_LARGE; + } else { + array_spec[GRN_HASH_KEY_SEGMENT].max_n_segments = + GRN_HASH_KEY_MAX_N_SEGMENTS; + } array_spec[GRN_HASH_ENTRY_SEGMENT].w_of_element = w_of_element; array_spec[GRN_HASH_ENTRY_SEGMENT].max_n_segments = 1U << (30 - (22 - w_of_element)); @@ -1700,7 +1803,7 @@ grn_io_hash_init(grn_ctx *ctx, grn_hash *hash, const char *path, } entry_size = grn_io_hash_calculate_entry_size(key_size, value_size, flags); - io = grn_io_hash_create_io(ctx, path, header_size, entry_size); + io = grn_io_hash_create_io(ctx, path, header_size, entry_size, flags); if (!io) { return GRN_NO_MEMORY_AVAILABLE; } @@ -1724,6 +1827,7 @@ grn_io_hash_init(grn_ctx *ctx, grn_hash *hash, const char *path, header->key_size = key_size; header->curr_rec = 0; header->curr_key = 0; + header->curr_key_large = 0; header->lock = 0; header->idx_offset = 0; header->value_size = value_size; @@ -2235,7 +2339,11 @@ grn_io_hash_add(grn_ctx *ctx, grn_hash *hash, uint32_t hash_value, garbages[key_size - 1] = *(grn_id *)entry; if (hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) { /* keep entry->io_entry's hash_value, flag, key_size and key. */ - memset(entry->io_entry.value, 0, header->value_size); + if (grn_hash_is_large_total_key_size(ctx, hash)) { + memset(entry->io_entry_large.value, 0, header->value_size); + } else { + memset(entry->io_entry.value, 0, header->value_size); + } } else { memset(entry, 0, header->entry_size); } @@ -2258,7 +2366,7 @@ grn_io_hash_add(grn_ctx *ctx, grn_hash *hash, uint32_t hash_value, } if (value) { - *value = grn_hash_entry_get_value(hash, entry); + *value = grn_hash_entry_get_value(ctx, hash, entry); } return entry_id; } @@ -2291,7 +2399,7 @@ grn_tiny_hash_add(grn_ctx *ctx, grn_hash *hash, uint32_t hash_value, } if (value) { - *value = grn_hash_entry_get_value(hash, entry); + *value = grn_hash_entry_get_value(ctx, hash, entry); } return entry_id; } @@ -2359,7 +2467,7 @@ grn_hash_add(grn_ctx *ctx, grn_hash *hash, const void *key, if (grn_hash_entry_compare_key(ctx, hash, entry, hash_value, key, key_size)) { if (value) { - *value = grn_hash_entry_get_value(hash, entry); + *value = grn_hash_entry_get_value(ctx, hash, entry); } if (added) { *added = 0; @@ -2434,7 +2542,7 @@ grn_hash_get(grn_ctx *ctx, grn_hash *hash, const void *key, if (grn_hash_entry_compare_key(ctx, hash, entry, hash_value, key, key_size)) { if (value) { - *value = grn_hash_entry_get_value(hash, entry); + *value = grn_hash_entry_get_value(ctx, hash, entry); } return id; } @@ -2520,7 +2628,7 @@ grn_hash_get_value(grn_ctx *ctx, grn_hash *hash, grn_id id, void *valuebuf) if (!entry) { return 0; } - value = grn_hash_entry_get_value(hash, entry); + value = grn_hash_entry_get_value(ctx, hash, entry); if (!value) { return 0; } @@ -2542,7 +2650,7 @@ grn_hash_get_value_(grn_ctx *ctx, grn_hash *hash, grn_id id, uint32_t *size) if (!entry) { return NULL; } - value = grn_hash_entry_get_value(hash, entry); + value = grn_hash_entry_get_value(ctx, hash, entry); if (!value) { return NULL; } @@ -2568,7 +2676,7 @@ grn_hash_get_key_value(grn_ctx *ctx, grn_hash *hash, grn_id id, if (bufsize >= key_size) { grn_memcpy(keybuf, grn_hash_entry_get_key(ctx, hash, entry), key_size); } - value = grn_hash_entry_get_value(hash, entry); + value = grn_hash_entry_get_value(ctx, hash, entry); if (!value) { return 0; } @@ -2593,7 +2701,7 @@ _grn_hash_get_key_value(grn_ctx *ctx, grn_hash *hash, grn_id id, } key_size = grn_hash_entry_get_key_size(hash, entry); *key = grn_hash_entry_get_key(ctx, hash, entry); - *value = grn_hash_entry_get_value(hash, entry); + *value = grn_hash_entry_get_value(ctx, hash, entry); return *value ? key_size : 0; } @@ -2613,7 +2721,7 @@ grn_hash_set_value(grn_ctx *ctx, grn_hash *hash, grn_id id, if (!entry) { return GRN_NO_MEMORY_AVAILABLE; } - entry_value = grn_hash_entry_get_value(hash, entry); + entry_value = grn_hash_entry_get_value(ctx, hash, entry); if (!entry_value) { return GRN_NO_MEMORY_AVAILABLE; } @@ -2898,7 +3006,7 @@ grn_hash_cursor_get_value(grn_ctx *ctx, grn_hash_cursor *c, void **value) entry_str *ee; if (!c) { return 0; } ee = grn_hash_entry_at(ctx, c->hash, c->curr_rec, 0); - if (ee && (v = get_value(c->hash, ee))) { + if (ee && (v = get_value(ctx, c->hash, ee))) { *value = v; return c->hash->value_size; } @@ -2917,7 +3025,7 @@ grn_hash_cursor_get_key_value(grn_ctx *ctx, grn_hash_cursor *c, *key_size = (c->hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) ? ee->size : c->hash->key_size; } if (key) { *key = get_key(ctx, c->hash, ee); } - if (value) { *value = get_value(c->hash, ee); } + if (value) { *value = get_value(ctx, c->hash, ee); } return c->hash->value_size; } @@ -2941,7 +3049,7 @@ grn_hash_cursor_delete(grn_ctx *ctx, grn_hash_cursor *c, #define PREPARE_VAL(e,ep,es) do {\ if ((arg->flags & GRN_TABLE_SORT_BY_VALUE)) {\ - ep = ((const uint8_t *)(get_value(hash, (entry_str *)(e))));\ + ep = ((const uint8_t *)(get_value(ctx, hash, (entry_str *)(e))));\ es = hash->value_size;\ } else {\ ep = ((const uint8_t *)(get_key(ctx, hash, (entry_str *)(e))));\ @@ -3099,7 +3207,7 @@ typedef struct { (ep)->v = (arg->flags & GRN_TABLE_SORT_BY_ID)\ ? (int32_t) id\ : (*((int32_t *)((byte *)((arg->flags & GRN_TABLE_SORT_BY_VALUE)\ - ? get_value(hash, (e))\ + ? get_value(ctx, hash, (e))\ : get_key(ctx, hash, (e))) + arg->offset)));\ } while (0) @@ -3306,7 +3414,7 @@ grn_hash_check(grn_ctx *ctx, grn_hash *hash) return; } GRN_OUTPUT_ARRAY_OPEN("RESULT", 1); - GRN_OUTPUT_MAP_OPEN("SUMMARY", 25); + GRN_OUTPUT_MAP_OPEN("SUMMARY", 26); GRN_OUTPUT_CSTR("flags"); grn_itoh(h->flags, buf, 8); GRN_OUTPUT_STR(buf, 8); @@ -3322,6 +3430,8 @@ grn_hash_check(grn_ctx *ctx, grn_hash *hash) GRN_OUTPUT_INT64(h->curr_rec); GRN_OUTPUT_CSTR("curr_key"); GRN_OUTPUT_INT64(h->curr_key); + GRN_OUTPUT_CSTR("curr_key_large"); + GRN_OUTPUT_UINT64(h->curr_key_large); GRN_OUTPUT_CSTR("idx_offset"); GRN_OUTPUT_INT64(h->idx_offset); GRN_OUTPUT_CSTR("entry_size"); @@ -3528,7 +3638,7 @@ grn_rhash_group(grn_hash *s, int limit, grn_group_optarg *optarg) } grn_rc -grn_rhash_subrec_info(grn_hash *s, grn_id rh, int index, +grn_rhash_subrec_info(grn_ctx *ctx, grn_hash *s, grn_id rh, int index, grn_id *rid, int *section, int *pos, int *score, void **subrec) { grn_rset_posinfo *pi; @@ -3542,7 +3652,7 @@ grn_rhash_subrec_info(grn_hash *s, grn_id rh, int index, ee = grn_hash_entry_at(ctx, s, rh, 0); if (!ee) { return GRN_INVALID_ARGUMENT; } pi = (grn_rset_posinfo *)get_key(ctx, s, ee); - ri = get_value(s, ee); + ri = get_value(ctx, s, ee); if (!pi || !ri) { return GRN_INVALID_ARGUMENT; } } if (index >= ri->n_subrecs) { return GRN_INVALID_ARGUMENT; } @@ -3590,14 +3700,28 @@ grn_rhash_subrec_info(grn_hash *s, grn_id rh, int index, } #endif /* USE_GRN_INDEX2 */ +grn_bool +grn_hash_is_large_total_key_size(grn_ctx *ctx, grn_hash *hash) +{ + return (hash->header.common->flags & GRN_OBJ_KEY_LARGE) == GRN_OBJ_KEY_LARGE; +} + uint64_t grn_hash_total_key_size(grn_ctx *ctx, grn_hash *hash) { - return hash->header.common->curr_key; + if (grn_hash_is_large_total_key_size(ctx, hash)) { + return hash->header.common->curr_key_large; + } else { + return hash->header.common->curr_key; + } } uint64_t grn_hash_max_total_key_size(grn_ctx *ctx, grn_hash *hash) { - return GRN_HASH_KEY_MAX_TOTAL_SIZE; + if (grn_hash_is_large_total_key_size(ctx, hash)) { + return GRN_HASH_KEY_MAX_TOTAL_SIZE_LARGE; + } else { + return GRN_HASH_KEY_MAX_TOTAL_SIZE; + } } Modified: lib/proc/proc_table.c (+3 -0) =================================================================== --- lib/proc/proc_table.c 2016-03-08 16:12:16 +0900 (9be68cf) +++ lib/proc/proc_table.c 2016-03-08 19:26:28 +0900 (0ec3be7) @@ -52,6 +52,9 @@ command_table_create_parse_flags(grn_ctx *ctx, } else if (!memcmp(nptr, "KEY_WITH_SIS", 12)) { flags |= GRN_OBJ_KEY_WITH_SIS; nptr += 12; + } else if (!memcmp(nptr, "KEY_LARGE", 9)) { + flags |= GRN_OBJ_KEY_LARGE; + nptr += 9; } else { GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, Added: test/command/suite/dump/schema/table/hash/key_large_flag.expected (+4 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/table/hash/key_large_flag.expected 2016-03-08 19:26:28 +0900 (1b096e3) @@ -0,0 +1,4 @@ +table_create Users TABLE_HASH_KEY|KEY_LARGE ShortText +[[0,0.0,0.0],true] +dump +table_create Users TABLE_HASH_KEY|KEY_LARGE ShortText Added: test/command/suite/dump/schema/table/hash/key_large_flag.test (+3 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/dump/schema/table/hash/key_large_flag.test 2016-03-08 19:26:28 +0900 (c32329b) @@ -0,0 +1,3 @@ +table_create Users TABLE_HASH_KEY|KEY_LARGE ShortText + +dump Added: test/command/suite/table_create/key_large/hash_key.expected (+46 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/table_create/key_large/hash_key.expected 2016-03-08 19:26:28 +0900 (9a9e1c3) @@ -0,0 +1,46 @@ +table_create Users TABLE_HASH_KEY|KEY_LARGE ShortText +[[0,0.0,0.0],true] +load --table Users +[ +{"_key": "Alice"}, +{"_key": "Bob"}, +{"_key": "Charlie"} +] +[[0,0.0,0.0],3] +select Users +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 3 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "_key", + "ShortText" + ] + ], + [ + 1, + "Alice" + ], + [ + 2, + "Bob" + ], + [ + 3, + "Charlie" + ] + ] + ] +] Added: test/command/suite/table_create/key_large/hash_key.test (+10 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/table_create/key_large/hash_key.test 2016-03-08 19:26:28 +0900 (ec21332) @@ -0,0 +1,10 @@ +table_create Users TABLE_HASH_KEY|KEY_LARGE ShortText + +load --table Users +[ +{"_key": "Alice"}, +{"_key": "Bob"}, +{"_key": "Charlie"} +] + +select Users Added: test/command/suite/table_create/key_large/patricia_trie.expected (+13 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/table_create/key_large/patricia_trie.expected 2016-03-08 19:26:28 +0900 (3f4830e) @@ -0,0 +1,13 @@ +table_create Users TABLE_PAT_KEY|KEY_LARGE ShortText +[ + [ + [ + -22, + 0.0, + 0.0 + ], + "[table][create] large key support is available only for TABLE_HASH_KEY key table: <Users>(TABLE_PAT_KEY)" + ], + false +] +#|e| [table][create] large key support is available only for TABLE_HASH_KEY key table: <Users>(TABLE_PAT_KEY) Added: test/command/suite/table_create/key_large/patricia_trie.test (+1 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/table_create/key_large/patricia_trie.test 2016-03-08 19:26:28 +0900 (6c73481) @@ -0,0 +1 @@ +table_create Users TABLE_PAT_KEY|KEY_LARGE ShortText -------------- next part -------------- HTML����������������������������...Télécharger
