Groonga - fulltext search engine.

[Groonga-commit] groonga/grnci at a5e1784 [master] Add CommandReader to read commands from a dump.

Susumu Yata null+****@clear*****
Wed Jul 5 18:46:57 JST 2017

• Previous message: [Groonga-commit] Broken: ranguba/chupa-text#156 (master - 8729061)
• Next message: [Groonga-commit] ranguba/chupa-text at d180cf5 [master] test: add screenshot data
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Susumu Yata	2017-07-05 18:46:57 +0900 (Wed, 05 Jul 2017)

  New Revision: a5e1784c7f7dbd5317ce8942117379a0cc11900d
  https://github.com/groonga/grnci/commit/a5e1784c7f7dbd5317ce8942117379a0cc11900d

  Message:
    Add CommandReader to read commands from a dump.
    
    GitHub: fix #41

  Modified files:
    v2/command.go
    v2/command_test.go

  Modified: v2/command.go (+213 -3)
===================================================================
--- v2/command.go    2017-07-05 11:25:27 +0900 (5f85bf2)
+++ v2/command.go    2017-07-05 18:46:57 +0900 (4ebf626)
@@ -1,6 +1,7 @@
 package grnci
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"reflect"
@@ -480,9 +481,9 @@ var commandFormats = map[string]*commandFormat{
 		nil,
 		newParamFormat("name", nil, true),
 	),
-	"query_expand": newCommandFormat(nil), // TODO
+	"query_expand": newCommandFormat(nil), // TODO: not documented.
 	"quit":         newCommandFormat(nil),
-	"range_filter": newCommandFormat(nil), // TODO
+	"range_filter": newCommandFormat(nil), // TODO: not documented.
 	"register": newCommandFormat(
 		nil,
 		newParamFormat("path", nil, true),
@@ -710,7 +711,15 @@ func tokenizeCommand(cmd string) ([]string, error) {
 							"error":   "The command ends with an escape character.",
 						})
 					}
-					token = append(token, unescapeCommandByte(s[i]))
+					switch s[i] {
+					case '\n':
+					case '\r':
+						if i+1 < len(s) && s[i+1] == '\n' {
+							i++
+						}
+					default:
+						token = append(token, unescapeCommandByte(s[i]))
+					}
 				default:
 					token = append(token, s[i])
 				}
@@ -894,3 +903,204 @@ func (c *Command) String() string {
 	}
 	return string(cmd)
 }
+
+// commandBodyReader is a reader for command bodies.
+type commandBodyReader struct {
+	reader *CommandReader // Underlying reader
+	stack  []byte         // Stack for special symbols
+	line   []byte         // Current line
+	left   []byte         // Remaining bytes of the current line
+	err    error          // Last error
+}
+
+// newCommandBodyReader returns a new commandBodyReader.
+func newCommandBodyReader(cr *CommandReader) *commandBodyReader {
+	return &commandBodyReader{
+		reader: cr,
+		stack:  make([]byte, 0, 8),
+	}
+}
+
+// checkLine checks the current line.
+func (br *commandBodyReader) checkLine() error {
+	var top byte
+	if len(br.stack) != 0 {
+		top = br.stack[len(br.stack)-1]
+	}
+	for i := 0; i < len(br.line); i++ {
+		switch top {
+		case 0: // The first non-space byte must be '[' or '{'.
+			switch br.line[i] {
+			case '[', '{':
+				top = br.line[i] + 2 // Convert a bracket from left to right.
+				br.stack = append(br.stack, top)
+			case ' ', '\t', '\r', '\n':
+			default:
+				return io.EOF
+			}
+		case '"', '\'':
+			switch br.line[i] {
+			case '\\':
+				if i+1 < len(br.line) { // Skip the next byte if possible.
+					i++
+				}
+			case top: // Close the quoted string.
+				br.stack = br.stack[len(br.stack)-1:]
+				top = br.stack[len(br.stack)-1]
+			}
+		default:
+			switch br.line[i] {
+			case '"', '\'':
+				top = br.line[i]
+				br.stack = append(br.stack, top)
+			case '[', '{':
+				top = br.line[i] + 2 // Convert a bracket from left to right.
+				br.stack = append(br.stack, top)
+			case ']', '}':
+				if br.line[i] != top {
+					return io.EOF
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// Read reads up to len(p) bytes into p.
+func (br *commandBodyReader) Read(p []byte) (n int, err error) {
+	if len(br.left) == 0 && br.err != nil {
+		return 0, br.err
+	}
+	cr := br.reader
+	for n < len(p) {
+		if len(br.left) == 0 {
+			if err = br.checkLine(); err != nil {
+				cr.err = err
+				return
+			}
+			br.line, err = cr.readLine()
+			if err != nil {
+				return
+			}
+			br.left = br.line
+		}
+		m := copy(p[n:], br.left)
+		br.left = br.left[m:]
+		n += m
+	}
+	return
+}
+
+// CommandReader is a reader for commands.
+type CommandReader struct {
+	reader io.Reader // Underlying reader
+	buf    []byte    // Buffer
+	left   []byte    // Unprocessed bytes in buf
+	err    error     // Last error
+}
+
+// NewCommandReader returns a new CommandReader.
+func NewCommandReader(r io.Reader) *CommandReader {
+	return &CommandReader{
+		reader: r,
+		buf:    make([]byte, 1024),
+	}
+}
+
+// fill reads data from the underlying reader and fills the buffer.
+func (cr *CommandReader) fill() error {
+	if cr.err != nil {
+		return cr.err
+	}
+	if len(cr.left) == len(cr.buf) {
+		cr.buf = make([]byte, len(cr.buf)*2)
+	}
+	copy(cr.buf, cr.left)
+	n, err := cr.reader.Read(cr.buf[len(cr.left):])
+	if err != nil {
+		cr.err = err
+		if err != io.EOF {
+			cr.err = NewError(InvalidCommand, map[string]interface{}{
+				"error": err.Error(),
+			})
+		}
+	}
+	if n == 0 {
+		return cr.err
+	}
+	cr.left = cr.buf[:len(cr.left)+n]
+	return nil
+}
+
+// readLine reads the next line.
+func (cr *CommandReader) readLine() ([]byte, error) {
+	if len(cr.left) == 0 && cr.err != nil {
+		return nil, cr.err
+	}
+	i := 0
+	for {
+		if i == len(cr.left) {
+			cr.fill()
+			if i == len(cr.left) {
+				if i == 0 {
+					return nil, cr.err
+				}
+				line := cr.left
+				cr.left = cr.left[len(cr.left):]
+				return line, nil
+			}
+		}
+		switch cr.left[i] {
+		case '\\':
+			i++
+			if i == len(cr.left) {
+				cr.fill()
+			}
+			if i == len(cr.left) {
+				line := cr.left
+				cr.left = cr.left[len(cr.left):]
+				return line, nil
+			}
+		case '\r':
+			if i+1 == len(cr.left) {
+				cr.fill()
+			}
+			if i+1 < len(cr.left) && cr.left[i+1] == '\n' {
+				i++
+			}
+			line := cr.left[:i+1]
+			cr.left = cr.left[i+1:]
+			return line, nil
+		case '\n':
+			line := cr.left[:i+1]
+			cr.left = cr.left[i+1:]
+			return line, nil
+		}
+		i++
+	}
+}
+
+// Read reads the next command.
+// If the command has a body, its whole content must be read before the next Read.
+func (cr *CommandReader) Read() (*Command, error) {
+	if len(cr.left) == 0 && cr.err != nil {
+		return nil, cr.err
+	}
+	for {
+		line, err := cr.readLine()
+		if err != nil {
+			return nil, err
+		}
+		cmd := bytes.TrimLeft(line, " \t\r\n")
+		if len(cmd) != 0 {
+			cmd, err := ParseCommand(string(cmd))
+			if err != nil {
+				return nil, err
+			}
+			if cmd.NeedsBody() {
+				cmd.SetBody(newCommandBodyReader(cr))
+			}
+			return cmd, nil
+		}
+	}
+}

  Modified: v2/command_test.go (+134 -26)
===================================================================
--- v2/command_test.go    2017-07-05 11:25:27 +0900 (3aef0b4)
+++ v2/command_test.go    2017-07-05 18:46:57 +0900 (3cda9db)
@@ -1,6 +1,9 @@
 package grnci
 
 import (
+	"io"
+	"io/ioutil"
+	"strings"
 	"testing"
 )
 
@@ -209,28 +212,61 @@ func TestFormatParamSelect(t *testing.T) {
 
 func TestNewCommand(t *testing.T) {
 	params := map[string]interface{}{
-		"table":     "Tbl",
-		"filter":    "value < 100",
-		"sort_keys": "value",
-		"cache":     false,
-		"offset":    0,
-		"limit":     -1,
+		"table":                 "Tbl",
+		"match_columns":         []string{"title", "body"},
+		"query":                 "Japan",
+		"filter":                "value < 100",
+		"sort_keys":             []string{"value", "_key"},
+		"output_columns":        []string{"_id", "_key", "value", "percent"},
+		"cache":                 false,
+		"offset":                0,
+		"limit":                 -1,
+		"column[percent].stage": "output",
+		"column[percent].type":  "Float",
+		"column[percent].value": "value / 100",
 	}
 	cmd, err := NewCommand("select", params)
 	if err != nil {
 		t.Fatalf("NewCommand failed: %v", err)
 	}
-	if cmd.Name() != "select" {
-		t.Fatalf("NewCommand failed: name = %s, want = %s", cmd.Name(), "select")
+	if actual, want := cmd.Name(), "select"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
-	if key, want := "table", "Tbl"; cmd.Params()[key] != want {
-		t.Fatalf("NewCommand failed: params[\"%s\"] = %s, want = %v", key, cmd.Params()[key], want)
+	if actual, want := cmd.params["table"], "Tbl"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
-	if key, want := "cache", "no"; cmd.Params()[key] != want {
-		t.Fatalf("NewCommand failed: params[\"%s\"] = %s, want = %v", key, cmd.Params()[key], want)
+	if actual, want := cmd.params["match_columns"], "title||body"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
-	if key, want := "limit", "-1"; cmd.Params()[key] != want {
-		t.Fatalf("NewCommand failed: params[\"%s\"] = %s, want = %v", key, cmd.Params()[key], want)
+	if actual, want := cmd.params["query"], "Japan"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["filter"], "value < 100"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["sort_keys"], "value,_key"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["output_columns"], "_id,_key,value,percent"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["cache"], "no"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["offset"], "0"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["limit"], "-1"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["column[percent].stage"], "output"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["column[percent].type"], "Float"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.params["column[percent].value"], "value / 100"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
 }
 
@@ -239,17 +275,20 @@ func TestParseCommand(t *testing.T) {
 	if err != nil {
 		t.Fatalf("ParseCommand failed: %v", err)
 	}
-	if want := "select"; cmd.Name() != want {
-		t.Fatalf("ParseCommand failed: name = %s, want = %s", cmd.Name(), want)
+	if actual, want := cmd.Name(), "select"; actual != want {
+		t.Fatalf("ParseCommand failed: actual = %s, want = %s", actual, want)
+	}
+	if actual, want := cmd.Params()["table"], "Tbl"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
-	if key, want := "table", "Tbl"; cmd.Params()[key] != want {
-		t.Fatalf("NewCommand failed: params[\"%s\"] = %s, want = %v", key, cmd.Params()[key], want)
+	if actual, want := cmd.Params()["query"], "\"apple juice\""; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
-	if key, want := "query", `"apple juice"`; cmd.Params()[key] != want {
-		t.Fatalf("NewCommand failed: params[\"%s\"] = %s, want = %v", key, cmd.Params()[key], want)
+	if actual, want := cmd.Params()["filter"], "price < 100"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
-	if key, want := "cache", "no"; cmd.Params()[key] != want {
-		t.Fatalf("NewCommand failed: params[\"%s\"] = %s, want = %v", key, cmd.Params()[key], want)
+	if actual, want := cmd.Params()["cache"], "no"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
 	}
 }
 
@@ -261,29 +300,43 @@ func TestCommandSetParam(t *testing.T) {
 	if err := cmd.SetParam("", "Tbl"); err != nil {
 		t.Fatalf("cmd.SetParam failed: %v", err)
 	}
+	if actual, want := cmd.Params()["table"], "Tbl"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
 	if err := cmd.SetParam("cache", false); err != nil {
 		t.Fatalf("cmd.SetParam failed: %v", err)
 	}
+	if actual, want := cmd.Params()["cache"], "no"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
 	if err := cmd.SetParam("cache", true); err != nil {
 		t.Fatalf("cmd.SetParam failed: %v", err)
 	}
+	if actual, want := cmd.Params()["cache"], "yes"; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
 	if err := cmd.SetParam("cache", nil); err != nil {
 		t.Fatalf("cmd.SetParam failed: %v", err)
 	}
+	if actual, want := cmd.Params()["cache"], ""; actual != want {
+		t.Fatalf("NewCommand failed: actual = %s, want = %s", actual, want)
+	}
 }
 
 func TestCommandString(t *testing.T) {
 	params := map[string]interface{}{
-		"table": "Tbl",
-		"cache": "no",
-		"limit": -1,
+		"table":         "Tbl",
+		"cache":         "no",
+		"limit":         -1,
+		"match_columns": []string{"title", "body"},
+		"query":         `"de facto"`,
 	}
 	cmd, err := NewCommand("select", params)
 	if err != nil {
 		t.Fatalf("NewCommand failed: %v", err)
 	}
 	actual := cmd.String()
-	want := "select --cache 'no' --limit '-1' --table 'Tbl'"
+	want := `select --cache 'no' --limit '-1' --match_columns 'title||body' --query '"de facto"' --table 'Tbl'`
 	if actual != want {
 		t.Fatalf("cmd.String failed: actual = %s, want = %s", actual, want)
 	}
@@ -307,3 +360,58 @@ func TestCommandNeedsBody(t *testing.T) {
 		}
 	}
 }
+
+func TestCommandReader(t *testing.T) {
+	dump := `table_create Tbl TABLE_NO_KEY
+column_create Tbl col COLUMN_SCALAR Text
+
+table_create Idx TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram --normalizer NormalizerAuto
+column_create Idx col COLUMN_INDEX|WITH_POSITION Tbl col
+
+load --table Tbl
+[
+["col"],
+["Hello, world!"],
+["'{' is called a left brace."]
+]
+`
+	cr := NewCommandReader(strings.NewReader(dump))
+	if cmd, err := cr.Read(); err != nil {
+		t.Fatalf("cr.Read failed: %v", err)
+	} else if actual, want := cmd.Name(), "table_create"; actual != want {
+		t.Fatalf("cr.Read failed: actual = %s, want = %s", actual, want)
+	}
+	if cmd, err := cr.Read(); err != nil {
+		t.Fatalf("cr.Read failed: %v", err)
+	} else if actual, want := cmd.Name(), "column_create"; actual != want {
+		t.Fatalf("cr.Read failed: actual = %s, want = %s", actual, want)
+	}
+	if cmd, err := cr.Read(); err != nil {
+		t.Fatalf("cr.Read failed: %v", err)
+	} else if actual, want := cmd.Name(), "table_create"; actual != want {
+		t.Fatalf("cr.Read failed: actual = %s, want = %s", actual, want)
+	}
+	if cmd, err := cr.Read(); err != nil {
+		t.Fatalf("cr.Read failed: %v", err)
+	} else if actual, want := cmd.Name(), "column_create"; actual != want {
+		t.Fatalf("cr.Read failed: actual = %s, want = %s", actual, want)
+	}
+	if cmd, err := cr.Read(); err != nil {
+		t.Fatalf("cr.Read failed: %v", err)
+	} else if actual, want := cmd.Name(), "load"; actual != want {
+		t.Fatalf("cr.Read failed: actual = %s, want = %s", actual, want)
+	} else if body, err := ioutil.ReadAll(cmd.Body()); err != nil {
+		t.Fatalf("io.ReadAll failed: %v", err)
+	} else if actual, want := string(body), `[
+["col"],
+["Hello, world!"],
+["'{' is called a left brace."]
+]
+`; actual != want {
+		t.Fatalf("io.ReadAll failed: actual = %s, want = %s", actual, want)
+	}
+	if _, err := cr.Read(); err != io.EOF {
+		t.Fatalf("cr.Read wongly succeeded")
+	}
+}
-------------- next part --------------
HTML����������������������������...
Télécharger 

• Previous message: [Groonga-commit] Broken: ranguba/chupa-text#156 (master - 8729061)
• Next message: [Groonga-commit] ranguba/chupa-text at d180cf5 [master] test: add screenshot data
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]