Skip to content

Commit 72225a5

Browse files
webbjurahul2393
webbju
and
rahul2393
authored
feat(spanner/spansql): Add support for TOKENIZE_JSON. (#12338)
Co-authored-by: rahul2393 <irahul@google.com>
1 parent f5cb67b commit 72225a5

3 files changed

Lines changed: 27 additions & 8 deletions

File tree

‎spanner/spansql/keywords.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ var funcNames = []string{
245245
"TOKEN",
246246
"TOKENIZE_BOOL",
247247
"TOKENIZE_FULLTEXT",
248+
"TOKENIZE_JSON",
248249
"TOKENIZE_NGRAMS",
249250
"TOKENIZE_NUMBER",
250251
"TOKENIZE_SUBSTRING",

‎spanner/spansql/parser_test.go‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2225,6 +2225,8 @@ func TestParseDDL(t *testing.T) {
22252225
ValueThree_Tokens TOKENLIST AS (TOKENIZE_NGRAMS(ValueThree)) HIDDEN,
22262226
ValueFour STRING(MAX) NOT NULL,
22272227
ValueFour_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(ValueFour || "concat")) HIDDEN,
2228+
ValueFive JSON NOT NULL,
2229+
ValueFive_Tokens TOKENLIST AS (TOKENIZE_JSON(ValueFive)) HIDDEN,
22282230
Combined_Tokens TOKENLIST AS (TOKENLIST_CONCAT([Name_Tokens, ValueFour_Tokens])) HIDDEN,
22292231
Argument_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(Name, token_category => "small")) HIDDEN,
22302232
ManyArgument_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Value, comparison_type => "all", min => 1, max => 5)) HIDDEN,
@@ -2285,11 +2287,18 @@ func TestParseDDL(t *testing.T) {
22852287
Hidden: true,
22862288
Position: line(13),
22872289
},
2290+
{Name: "ValueFive", Type: Type{Base: JSON}, NotNull: true, Position: line(14)},
2291+
{
2292+
Name: "ValueFive_Tokens", Type: Type{Base: Tokenlist},
2293+
Generated: Func{Name: "TOKENIZE_JSON", Args: []Expr{ID("ValueFive")}},
2294+
Hidden: true,
2295+
Position: line(15),
2296+
},
22882297
{
22892298
Name: "Combined_Tokens", Type: Type{Base: Tokenlist},
22902299
Generated: Func{Name: "TOKENLIST_CONCAT", Args: []Expr{Array{ID("Name_Tokens"), ID("ValueFour_Tokens")}}},
22912300
Hidden: true,
2292-
Position: line(14),
2301+
Position: line(16),
22932302
},
22942303
{
22952304
Name: "Argument_Tokens", Type: Type{Base: Tokenlist},
@@ -2298,7 +2307,7 @@ func TestParseDDL(t *testing.T) {
22982307
Value: StringLiteral("small"),
22992308
}}},
23002309
Hidden: true,
2301-
Position: line(15),
2310+
Position: line(17),
23022311
},
23032312
{
23042313
Name: "ManyArgument_Tokens", Type: Type{Base: Tokenlist},
@@ -2318,7 +2327,7 @@ func TestParseDDL(t *testing.T) {
23182327
},
23192328
}},
23202329
Hidden: true,
2321-
Position: line(16),
2330+
Position: line(18),
23222331
},
23232332
},
23242333
PrimaryKey: []KeyPart{{Column: "Name"}},
@@ -2334,7 +2343,7 @@ func TestParseDDL(t *testing.T) {
23342343
Storing: []ID{"ValueTwo"},
23352344
PartitionBy: []ID{"Value", "ValueTwo"},
23362345
OrderBy: []Order{{Expr: ID("Value"), Desc: true}, {Expr: ID("ValueTwo"), Desc: false}},
2337-
Position: line(19),
2346+
Position: line(21),
23382347
Interleave: ID("SomeTable"),
23392348
Options: SearchIndexOptions{SortOrderSharding: func(b bool) *bool { return &b }(true)},
23402349
},
@@ -2351,7 +2360,7 @@ func TestParseDDL(t *testing.T) {
23512360
23522361
CREATE SEARCH INDEX TableTokensSearch
23532362
ON TableTokens(Name_Tokens, Value_Tokens);
2354-
2363+
23552364
ALTER SEARCH INDEX TableTokensSearch ADD STORED COLUMN Value_Tokens;
23562365
ALTER SEARCH INDEX TableTokensSearch DROP STORED COLUMN Value_Tokens;
23572366
DROP SEARCH INDEX IF EXISTS TableTokensSearch;`,

‎spanner/spansql/sql_test.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -833,11 +833,18 @@ func TestSQL(t *testing.T) {
833833
Hidden: true,
834834
Position: line(13),
835835
},
836+
{Name: "ValueFive", Type: Type{Base: JSON}, NotNull: true, Position: line(14)},
837+
{
838+
Name: "ValueFive_Tokens", Type: Type{Base: Tokenlist},
839+
Generated: Func{Name: "TOKENIZE_JSON", Args: []Expr{ID("ValueFive")}},
840+
Hidden: true,
841+
Position: line(15),
842+
},
836843
{
837844
Name: "Combined_Tokens", Type: Type{Base: Tokenlist},
838845
Generated: Func{Name: "TOKENLIST_CONCAT", Args: []Expr{Array{ID("Name_Tokens"), ID("ValueFour_Tokens")}}},
839846
Hidden: true,
840-
Position: line(14),
847+
Position: line(16),
841848
},
842849
{
843850
Name: "Argument_Tokens", Type: Type{Base: Tokenlist},
@@ -846,7 +853,7 @@ func TestSQL(t *testing.T) {
846853
Value: StringLiteral("small"),
847854
}}},
848855
Hidden: true,
849-
Position: line(15),
856+
Position: line(17),
850857
},
851858
{
852859
Name: "ManyArgument_Tokens", Type: Type{Base: Tokenlist},
@@ -866,7 +873,7 @@ func TestSQL(t *testing.T) {
866873
},
867874
}},
868875
Hidden: true,
869-
Position: line(16),
876+
Position: line(18),
870877
},
871878
},
872879
PrimaryKey: []KeyPart{{Column: "Name"}},
@@ -885,6 +892,8 @@ func TestSQL(t *testing.T) {
885892
ValueThree_Tokens TOKENLIST AS (TOKENIZE_NGRAMS(ValueThree)) HIDDEN,
886893
ValueFour STRING(MAX) NOT NULL,
887894
ValueFour_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(ValueFour)) HIDDEN,
895+
ValueFive JSON NOT NULL,
896+
ValueFive_Tokens TOKENLIST AS (TOKENIZE_JSON(ValueFive)) HIDDEN,
888897
Combined_Tokens TOKENLIST AS (TOKENLIST_CONCAT([Name_Tokens, ValueFour_Tokens])) HIDDEN,
889898
Argument_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(Name, token_category => "small")) HIDDEN,
890899
ManyArgument_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Value, comparison_type => "all", min => 1, max => 5)) HIDDEN,

0 commit comments

Comments
 (0)