From 13374e36e91ebac430666f49578e6b780059cefe Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Mon, 27 Jan 2025 10:35:44 +0200 Subject: [PATCH 01/10] feat(flink): implement suggestion support for table properties in CREATE TABLE --- src/parser/common/types.ts | 4 +++ src/parser/flink/index.ts | 10 ++++++ .../flink/suggestion/tokenSuggestion.test.ts | 31 ++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/parser/common/types.ts b/src/parser/common/types.ts index 8d2d2e67..a248abf4 100644 --- a/src/parser/common/types.ts +++ b/src/parser/common/types.ts @@ -44,6 +44,10 @@ export enum EntityContextType { COLUMN = 'column', /** column name that will be created */ COLUMN_CREATE = 'columnCreate', + /** table property key when creating table*/ + TABLE_PROPERTY_KEY = 'tablePropertyKey', + /** table property value when creating table*/ + TABLE_PROPERTY_VALUE = 'tablePropertyValue', } /** diff --git a/src/parser/flink/index.ts b/src/parser/flink/index.ts index 54f1203b..e59a715a 100644 --- a/src/parser/flink/index.ts +++ b/src/parser/flink/index.ts @@ -41,6 +41,8 @@ export class FlinkSQL extends BasicSQL { expect(suggestion).toContain('NOT'); expect(suggestion).toContain('NOT EXISTS'); }); + + test('Create Statement table properties', () => { + const tokenSql = `CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka');`; + const scenarios = [ + { + caretPosition: { + lineNumber: 1, + column: 45, + }, + entityContextType: EntityContextType.TABLE_PROPERTY_KEY, + }, + { + caretPosition: { + lineNumber: 1, + column: 55, + }, + entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, + }, + ]; + + scenarios.forEach((scenario) => { + const suggestion = flink.getSuggestionAtCaretPosition( + tokenSql, + scenario.caretPosition + )?.syntax; + + expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType); + }); + }); }); From 802a84364b53f446e3031ccedc0882e1f1a7e422 Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Thu, 13 Mar 2025 09:42:23 +0200 Subject: [PATCH 02/10] fix(suggestion): move SQL to fixtures --- .../parser/flink/suggestion/fixtures/tokenSuggestion.sql | 9 ++++----- test/parser/flink/suggestion/tokenSuggestion.test.ts | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql index 60ce11c3..795c807b 100644 --- a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql @@ -1,10 +1,9 @@ SELECT * FROM aa.bb; -USE +USE ; CREATE ; -SHOW -; -CREATE TABLE IF NOT EXISTS -; \ No newline at end of file +SHOW + +CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); \ No newline at end of file diff --git a/test/parser/flink/suggestion/tokenSuggestion.test.ts b/test/parser/flink/suggestion/tokenSuggestion.test.ts index 3a18171f..17ba3668 100644 --- a/test/parser/flink/suggestion/tokenSuggestion.test.ts +++ b/test/parser/flink/suggestion/tokenSuggestion.test.ts @@ -95,18 +95,17 @@ describe('Flink SQL Token Suggestion', () => { }); test('Create Statement table properties', () => { - const tokenSql = `CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka');`; const scenarios = [ { caretPosition: { - lineNumber: 1, + lineNumber: 9, column: 45, }, entityContextType: EntityContextType.TABLE_PROPERTY_KEY, }, { caretPosition: { - lineNumber: 1, + lineNumber: 9, column: 55, }, entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, @@ -115,7 +114,7 @@ describe('Flink SQL Token Suggestion', () => { scenarios.forEach((scenario) => { const suggestion = flink.getSuggestionAtCaretPosition( - tokenSql, + commentOtherLine(tokenSql, scenario.caretPosition.lineNumber), scenario.caretPosition )?.syntax; From 94ec619a9e19e97890761e83653c2193bad79d42 Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Thu, 13 Mar 2025 09:43:24 +0200 Subject: [PATCH 03/10] chore(sql): add EOL --- test/parser/flink/suggestion/fixtures/tokenSuggestion.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql index 795c807b..dff1eb6c 100644 --- a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql @@ -6,4 +6,4 @@ CREATE ; SHOW -CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); \ No newline at end of file +CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); From ed7e81e7a54a14c94c1ebecb9abf997ee7512631 Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Mon, 27 Jan 2025 10:35:44 +0200 Subject: [PATCH 04/10] feat(flink): implement suggestion support for table properties in CREATE TABLE # Conflicts: # test/parser/flink/suggestion/tokenSuggestion.test.ts --- .../flink/suggestion/tokenSuggestion.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/parser/flink/suggestion/tokenSuggestion.test.ts b/test/parser/flink/suggestion/tokenSuggestion.test.ts index 17ba3668..3b96317e 100644 --- a/test/parser/flink/suggestion/tokenSuggestion.test.ts +++ b/test/parser/flink/suggestion/tokenSuggestion.test.ts @@ -121,4 +121,33 @@ describe('Flink SQL Token Suggestion', () => { expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType); }); }); + + test('Create Statement table properties', () => { + const tokenSql = `CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka');`; + const scenarios = [ + { + caretPosition: { + lineNumber: 1, + column: 45, + }, + entityContextType: EntityContextType.TABLE_PROPERTY_KEY, + }, + { + caretPosition: { + lineNumber: 1, + column: 55, + }, + entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, + }, + ]; + + scenarios.forEach((scenario) => { + const suggestion = flink.getSuggestionAtCaretPosition( + tokenSql, + scenario.caretPosition + )?.syntax; + + expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType); + }); + }); }); From 9d5fc13da9e37eb44ffe682960f5bb48f1283d5c Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Thu, 13 Mar 2025 09:42:23 +0200 Subject: [PATCH 05/10] fix(suggestion): move SQL to fixtures --- test/parser/flink/suggestion/tokenSuggestion.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/parser/flink/suggestion/tokenSuggestion.test.ts b/test/parser/flink/suggestion/tokenSuggestion.test.ts index 3b96317e..f3ef4972 100644 --- a/test/parser/flink/suggestion/tokenSuggestion.test.ts +++ b/test/parser/flink/suggestion/tokenSuggestion.test.ts @@ -123,18 +123,17 @@ describe('Flink SQL Token Suggestion', () => { }); test('Create Statement table properties', () => { - const tokenSql = `CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka');`; const scenarios = [ { caretPosition: { - lineNumber: 1, + lineNumber: 9, column: 45, }, entityContextType: EntityContextType.TABLE_PROPERTY_KEY, }, { caretPosition: { - lineNumber: 1, + lineNumber: 9, column: 55, }, entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, @@ -143,7 +142,7 @@ describe('Flink SQL Token Suggestion', () => { scenarios.forEach((scenario) => { const suggestion = flink.getSuggestionAtCaretPosition( - tokenSql, + commentOtherLine(tokenSql, scenario.caretPosition.lineNumber), scenario.caretPosition )?.syntax; From c51a2d808dc32a534655759b40d9fc16932ade68 Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Mon, 12 May 2025 07:31:29 +0300 Subject: [PATCH 06/10] fix(syntaxSuggestion): resolve conflicts and move tests --- .../suggestion/fixtures/syntaxSuggestion.sql | 2 + .../suggestion/fixtures/tokenSuggestion.sql | 2 - .../flink/suggestion/syntaxSuggestion.test.ts | 28 +++++++++ .../flink/suggestion/tokenSuggestion.test.ts | 58 +------------------ 4 files changed, 31 insertions(+), 59 deletions(-) diff --git a/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql b/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql index 74cbb786..fa24f76b 100644 --- a/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql @@ -40,6 +40,8 @@ CREATE TABLE yourTable (ts TIMESTAMP(3), WATERMARK FOR ); CREATE TABLE newTable ( ); +CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); + SELECT SUM(amount) FROM Orders GROUP BY length(users) HAVING SUM(amount) > 50; SELECT * FROM Orders ORDER BY orderTime LIMIT length(order_id); diff --git a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql index adc68fee..9102b1be 100644 --- a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql @@ -13,5 +13,3 @@ CREATE TABLE IF NOT EXISTS CREATE TABLE tb (id ); - -CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); diff --git a/test/parser/flink/suggestion/syntaxSuggestion.test.ts b/test/parser/flink/suggestion/syntaxSuggestion.test.ts index ebe47c7f..fde52a91 100644 --- a/test/parser/flink/suggestion/syntaxSuggestion.test.ts +++ b/test/parser/flink/suggestion/syntaxSuggestion.test.ts @@ -469,4 +469,32 @@ describe('Flink SQL Syntax Suggestion', () => { expect(suggestion).not.toBeUndefined(); expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['age']); }); + + test('Create Statement table properties', () => { + const scenarios = [ + { + caretPosition: { + lineNumber: 9, + column: 45, + }, + entityContextType: EntityContextType.TABLE_PROPERTY_KEY, + }, + { + caretPosition: { + lineNumber: 9, + column: 55, + }, + entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, + }, + ]; + + scenarios.forEach((scenario) => { + const suggestion = flink.getSuggestionAtCaretPosition( + commentOtherLine(syntaxSql, scenario.caretPosition.lineNumber), + scenario.caretPosition + )?.syntax; + + expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType); + }); + }); }); diff --git a/test/parser/flink/suggestion/tokenSuggestion.test.ts b/test/parser/flink/suggestion/tokenSuggestion.test.ts index a3946a72..2b2ab1b3 100644 --- a/test/parser/flink/suggestion/tokenSuggestion.test.ts +++ b/test/parser/flink/suggestion/tokenSuggestion.test.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import { FlinkSQL } from 'src/parser/flink'; -import { CaretPosition, EntityContextType } from 'src/parser/common/types'; +import { CaretPosition } from 'src/parser/common/types'; import { commentOtherLine } from 'test/helper'; const tokenSql = fs.readFileSync(path.join(__dirname, 'fixtures', 'tokenSuggestion.sql'), 'utf-8'); @@ -94,62 +94,6 @@ describe('Flink SQL Token Suggestion', () => { expect(suggestion).toContain('NOT EXISTS'); }); - test('Create Statement table properties', () => { - const scenarios = [ - { - caretPosition: { - lineNumber: 9, - column: 45, - }, - entityContextType: EntityContextType.TABLE_PROPERTY_KEY, - }, - { - caretPosition: { - lineNumber: 9, - column: 55, - }, - entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, - }, - ]; - - scenarios.forEach((scenario) => { - const suggestion = flink.getSuggestionAtCaretPosition( - commentOtherLine(tokenSql, scenario.caretPosition.lineNumber), - scenario.caretPosition - )?.syntax; - - expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType); - }); - }); - - test('Create Statement table properties', () => { - const scenarios = [ - { - caretPosition: { - lineNumber: 9, - column: 45, - }, - entityContextType: EntityContextType.TABLE_PROPERTY_KEY, - }, - { - caretPosition: { - lineNumber: 9, - column: 55, - }, - entityContextType: EntityContextType.TABLE_PROPERTY_VALUE, - }, - ]; - - scenarios.forEach((scenario) => { - const suggestion = flink.getSuggestionAtCaretPosition( - commentOtherLine(tokenSql, scenario.caretPosition.lineNumber), - scenario.caretPosition - )?.syntax; - - expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType); - }); - }); - test('Suggestion in new line', () => { const pos: CaretPosition = { lineNumber: 13, From c0e29a373690ba3988714ca2154e13ce91a12527 Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Mon, 12 May 2025 07:33:32 +0300 Subject: [PATCH 07/10] fix: revert old changes --- test/parser/flink/suggestion/fixtures/tokenSuggestion.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql index 9102b1be..4296aad0 100644 --- a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql @@ -4,7 +4,6 @@ USE ; CREATE ; - SHOW ; CREATE TABLE IF NOT EXISTS From a19b3c935746385585d21d4b6a0a0b827e69a0d1 Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Mon, 12 May 2025 07:35:22 +0300 Subject: [PATCH 08/10] fix: revert old changes --- test/parser/flink/suggestion/fixtures/tokenSuggestion.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql index 4296aad0..01a0ae89 100644 --- a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql @@ -1,6 +1,6 @@ SELECT * FROM aa.bb; -USE +USE ; CREATE ; From 53c34cb7d4e41966ca027f60181ebbcad0af1f8c Mon Sep 17 00:00:00 2001 From: Eduardas Kazakas Date: Mon, 12 May 2025 07:37:50 +0300 Subject: [PATCH 09/10] fix: merge issues --- src/parser/flink/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/parser/flink/index.ts b/src/parser/flink/index.ts index 293b1897..53b2bdcc 100644 --- a/src/parser/flink/index.ts +++ b/src/parser/flink/index.ts @@ -151,6 +151,7 @@ export class FlinkSQL extends BasicSQL Date: Mon, 12 May 2025 07:40:01 +0300 Subject: [PATCH 10/10] fix: unit tests --- test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql | 6 +++--- test/parser/flink/suggestion/syntaxSuggestion.test.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql b/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql index fa24f76b..2bf8d560 100644 --- a/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql +++ b/test/parser/flink/suggestion/fixtures/syntaxSuggestion.sql @@ -40,10 +40,10 @@ CREATE TABLE yourTable (ts TIMESTAMP(3), WATERMARK FOR ); CREATE TABLE newTable ( ); -CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); - SELECT SUM(amount) FROM Orders GROUP BY length(users) HAVING SUM(amount) > 50; SELECT * FROM Orders ORDER BY orderTime LIMIT length(order_id); -SELECT age CASE WHEN age < 18 THEN 1 ELSE 0 END AS is_minor FROM dt_catalog.dt_db.subscriptions; \ No newline at end of file +SELECT age CASE WHEN age < 18 THEN 1 ELSE 0 END AS is_minor FROM dt_catalog.dt_db.subscriptions; + +CREATE TABLE tmp_table (col INT) WITH ('connector'='kafka'); \ No newline at end of file diff --git a/test/parser/flink/suggestion/syntaxSuggestion.test.ts b/test/parser/flink/suggestion/syntaxSuggestion.test.ts index fde52a91..7a6dd004 100644 --- a/test/parser/flink/suggestion/syntaxSuggestion.test.ts +++ b/test/parser/flink/suggestion/syntaxSuggestion.test.ts @@ -474,14 +474,14 @@ describe('Flink SQL Syntax Suggestion', () => { const scenarios = [ { caretPosition: { - lineNumber: 9, + lineNumber: 49, column: 45, }, entityContextType: EntityContextType.TABLE_PROPERTY_KEY, }, { caretPosition: { - lineNumber: 9, + lineNumber: 49, column: 55, }, entityContextType: EntityContextType.TABLE_PROPERTY_VALUE,