Merge branch 'main' into string-3

Author: sundy-li

.github/workflows/dev-perf.yml

@@ -84,7 +84,7 @@ jobs:
           python3 scripts/ci/wait_tcp.py --timeout 5 --port 3307
       - name: Setup BendSQL
         run: |
-          curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/v0.2.3/bendsql-linux-amd64.tar.gz
+          curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/v0.3.0/bendsql-linux-amd64.tar.gz
           tar -xvzf ./bendsql.tar.gz
           chmod +x ./linux-amd64/bendsql
           sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
@@ -97,6 +97,8 @@ jobs:
         run: |
           mkdir -p /tmp/databend-perf/
           bendsql benchmark \
+            --warm 2 \
+            --count 5 \
             --test-dir ./scripts/benchmark/query/benchmarks/ \
             --output-format json,md \
             --output-dir /tmp/databend-perf/ \

.github/workflows/trusted-perf.yml

@@ -15,6 +15,8 @@ jobs:
       github.event.workflow_run.conclusion == 'success'
     steps:
       - name: Download benchmark results
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
           gh run download ${{github.event.workflow_run.id }} --dir benchmark/results --name benchmark-results
           while read line; do
@@ -24,7 +26,7 @@ jobs:
         uses: everpcpc/comment-on-pr-action@v1
         with:
           number: ${{ env.PR_NUMBER }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ github.token }}
           identifier: benchmark
           body: |
             ![platform](https://img.shields.io/static/v1?label=AWS%20EC2&message=${{ env.INSTANCE_TYPE }}&color=orange&logo=amazonec2)

Cargo.lock

@@ -90,6 +90,9 @@ sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafus
 opendal = { version = "0.26.2" }
 ordered-float = { version = "3.4.0", default-features = false }
 
+# type helper
+derive_more = "0.99.17"
+
 # error
 anyhow = { version = "1.0.65" }
 anyerror = { version = "=0.1.7" }

Cargo.toml

@@ -25,7 +25,7 @@ Plus, we assume that your databend destination is **S3 Compatible** since we use
 To check whether your backend support the integration, you could simply run the following command
 
 ```sql
-CREATE STAGE IF NOT EXISTS airbyte_stage;
+CREATE STAGE IF NOT EXISTS airbyte_stage FILE_FORMAT = (TYPE = CSV);
 PRESIGN UPLOAD @airbyte_stage/test.csv;
 ```
 

docs/doc/11-integrations/10-data-tool/01-airbyte.md

@@ -26,7 +26,7 @@ DESC STAGE my_int_stage;
 +--------------+------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------+---------+
 | name         | stage_type | stage_params                                                                                                                                                | copy_options                                  | file_format_options                                                                                                | comment |
 +--------------+------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------+---------+
-| my_int_stage | Internal   | StageParams { storage: S3(StageS3Storage { bucket: "", path: "", credentials_aws_key_id: "", credentials_aws_secret_key: "", encryption_master_key: "" }) } | CopyOptions { on_error: None, size_limit: 0 } | FileFormatOptions { format: Csv, skip_header: 0, field_delimiter: ",", record_delimiter: "\n", compression: None } |         |
+| my_int_stage | Internal   | StageParams { storage: S3(StageS3Storage { bucket: "", path: "", credentials_aws_key_id: "", credentials_aws_secret_key: "", encryption_master_key: "" }) } | CopyOptions { on_error: None, size_limit: 0 } | FileFormatOptions { format: Parquet, skip_header: 0, field_delimiter: ",", record_delimiter: "\n", compression: None } |         |
 +--------------+------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------+---------+
 ```
 
@@ -180,7 +180,7 @@ COPY INTO books FROM @my_int_stage files=('books.csv') file_format = (type = CSV
 <TabItem value="parquet" label="Parquet">
 
 ```sql
-COPY INTO books FROM @my_int_stage files=('books.parquet') file_format = (type = 'Parquet');
+COPY INTO books FROM @my_int_stage files=('books.parquet') FILE_FORMAT = (TYPE = PARQUET);
 ```
 
 </TabItem>

docs/doc/12-load-data/00-stage.md

@@ -103,9 +103,9 @@ COPY INTO books
 ```sql
 COPY INTO books
   FROM 's3://databend-bohu/data/'
-  credentials=(aws_key_id='<your-access-key-id>' aws_secret_key='<your-secret-access-key>')
-  pattern ='.*[.]parquet'
-  file_format = (type = PARQUET);
+  CONNECTION=(aws_key_id='<your-access-key-id>' aws_secret_key='<your-secret-access-key>')
+  PATTERN ='.*[.]parquet'
+  FILE_FORMAT = (TYPE = PARQUET);
 ```
 
 </TabItem>
@@ -119,10 +119,10 @@ If the file(s) is large and we want to check the file format is ok to parse, we
 ```sql
 COPY INTO books
   FROM 's3://databend-bohu/data/'
-  credentials=(aws_key_id='<your-access-key-id>' aws_secret_key='<your-secret-access-key>')
-  pattern ='.*[.]csv'
-  file_format = (type = CSV field_delimiter = ','  record_delimiter = '\n' skip_header = 0)
-  size_limit = 1; -- only load 1 rows
+  CONNECTION=(aws_key_id='<your-access-key-id>' aws_secret_key='<your-secret-access-key>')
+  PATTERN ='.*[.]csv'
+  FILE_FORMAT = (TYPE = CSV field_delimiter = ','  record_delimiter = '\n' skip_header = 0)
+  SIZE_LIMIT = 1; -- only load 1 rows
 ```
 
 :::

docs/doc/12-load-data/01-s3.md

@@ -14,6 +14,8 @@ FILE_FORMAT = ( TYPE = { CSV | TSV | NDJSON | PARQUET | XML } [ formatTypeOption
 Databend currently supports XML as a source ONLY. Unloading data into an XML file is not supported yet.
 :::
 
+If `FILE_FORMAT` is not specified, use `FILE_FORMAT = (TYPE = PARQUET)` by default.
+
 `formatTypeOptions`: Includes one or more options to describe other format details about the file. The options vary depending on the file format. See the sections below to find out the available options for each supported file format.
 
 ```sql

docs/doc/13-sql-reference/75-file-format-options.md

@@ -9,14 +9,14 @@ Creates a stage.
 ```sql
 -- Internal stage
 CREATE STAGE [ IF NOT EXISTS ] <internal_stage_name>
-  [ FILE_FORMAT = ( { TYPE = { CSV | PARQUET } [ formatTypeOptions ] ) } ]
+  [ FILE_FORMAT = ( { TYPE = { PARQUET | CSV | TSV | NDJSON } [ formatTypeOptions ] ) } ]
   [ COPY_OPTIONS = ( copyOptions ) ]
   [ COMMENT = '<string_literal>' ]
 
 -- External stage
 CREATE STAGE [ IF NOT EXISTS ] <external_stage_name>
     externalStageParams
-  [ FILE_FORMAT = ( { TYPE = { CSV | PARQUET } [ formatTypeOptions ] ) } ]
+  [ FILE_FORMAT = ( { TYPE = { PARQUET | CSV | TSV | NDJSON } [ formatTypeOptions ] ) } ]
   [ COPY_OPTIONS = ( copyOptions ) ]
   [ COMMENT = '<string_literal>' ]
 ```
@@ -68,18 +68,8 @@ externalStageParams ::=
 | ACCOUNT_KEY                	| Your account key for connecting the Azure Blob storage.  	| Optional 	|
 
 ### formatTypeOptions
-```
-formatTypeOptions ::=
-  RECORD_DELIMITER = '<character>'
-  FIELD_DELIMITER = '<character>'
-  SKIP_HEADER = <integer>
-```
 
-| Parameters  | Description | Required |
-| ----------- | ----------- | --- |
-| `RECORD_DELIMITER = '<character>'`  | One characters that separate records in an input file. Default `'\n'` | Optional |
-| `FIELD_DELIMITER = '<character>'`  | One characters that separate fields in an input file. Default `','` | Optional |
-| `SKIP_HEADER = <integer>`  | Number of lines at the start of the file to skip. Default `0` | Optional |
+For details about `FILE_FORMAT`, see [Input & Output File Formats](../../../13-sql-reference/75-file-format-options.md).
 
 ### copyOptions
 ```

docs/doc/14-sql-commands/00-ddl/40-stage/01-ddl-create-stage.md

@@ -21,6 +21,6 @@ DESC STAGE my_int_stage;
 +--------------+------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------+---------+
 | name         | stage_type | stage_params                                                                                                                                                | copy_options                                  | file_format_options                                                                                                | comment |
 +--------------+------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------+---------+
-| my_int_stage | Internal   | StageParams { storage: S3(StageS3Storage { bucket: "", path: "", credentials_aws_key_id: "", credentials_aws_secret_key: "", encryption_master_key: "" }) } | CopyOptions { on_error: None, size_limit: 0 } | FileFormatOptions { format: Csv, skip_header: 0, field_delimiter: ",", record_delimiter: "\n", compression: None } |         |
+| my_int_stage | Internal   | StageParams { storage: S3(StageS3Storage { bucket: "", path: "", credentials_aws_key_id: "", credentials_aws_secret_key: "", encryption_master_key: "" }) } | CopyOptions { on_error: None, size_limit: 0 } | FileFormatOptions { format: Parquet, skip_header: 0, field_delimiter: ",", record_delimiter: "\n", compression: None } |         |
 +--------------+------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------+---------+
 ```