datasets:
  - from: postgres:public.orders    # Database connector
    name: orders
    params:
      pg_host: localhost
      pg_db: mydb
      pg_user: reader
      pg_pass: ${secrets:PG_PASS}

  - from: s3://my-bucket/events/    # Object storage connector
    name: events
    params:
      file_format: parquet
      s3_auth: iam_role

datasets:
  - from: postgres:public.orders    # Database connector
    name: orders
    params:
      pg_host: localhost
      pg_db: mydb
      pg_user: reader
      pg_pass: ${secrets:PG_PASS}

  - from: s3://my-bucket/events/    # Object storage connector
    name: events
    params:
      file_format: parquet
      s3_auth: iam_role

/data/
  year=2024/
    month=01/
      data.parquet
    month=02/
      data.parquet

/data/
  year=2024/
    month=01/
      data.parquet
    month=02/
      data.parquet

datasets:
  - from: s3://bucket/data/sales/
    name: sales
    params:
      file_format: parquet

datasets:
  - from: s3://bucket/data/sales/
    name: sales
    params:
      file_format: parquet

datasets:
  - from: sftp://files.example.com/reports/quarterly.parquet
    name: quarterly_report

datasets:
  - from: sftp://files.example.com/reports/quarterly.parquet
    name: quarterly_report

datasets:
  - from: s3://bucket/data/
    name: partitioned_data
    params:
      file_format: parquet
      hive_partitioning_enabled: true

datasets:
  - from: s3://bucket/data/
    name: partitioned_data
    params:
      file_format: parquet
      hive_partitioning_enabled: true

SELECT * FROM partitioned_data WHERE year = '2024' AND month = '01';

SELECT * FROM partitioned_data WHERE year = '2024' AND month = '01';

datasets:
  - from: s3://bucket/data/
    name: my_data
    params:
      file_format: parquet
    metadata:
      location: enabled
      last_modified: enabled
      size: enabled

datasets:
  - from: s3://bucket/data/
    name: my_data
    params:
      file_format: parquet
    metadata:
      location: enabled
      last_modified: enabled
      size: enabled

metadata:
  location: enabled     # Only add the location column

metadata:
  location: enabled     # Only add the location column

SELECT * FROM my_data LIMIT 3;

SELECT * FROM my_data LIMIT 3;

+----+---------+------+-------+-----+----------------------+--------------------------------------------------------------+------+
| id | value   | year | month | day | last_modified        | location                                                     | size |
+----+---------+------+-------+-----+----------------------+--------------------------------------------------------------+------+
| 0  | value_0 | 2022 | 1     | 1   | 2024-10-10T05:36:59Z | s3://bucket/data/year=2022/month=1/day=1/data_0.parquet      | 2317 |
| 1  | value_1 | 2022 | 1     

+----+---------+------+-------+-----+----------------------+--------------------------------------------------------------+------+
| id | value   | year | month | day | last_modified        | location                                                     | size |
+----+---------+------+-------+-----+----------------------+--------------------------------------------------------------+------+
| 0  | value_0 | 2022 | 1     | 1   | 2024-10-10T05:36:59Z | s3://bucket/data/year=2022/month=1/day=1/data_0.parquet      | 2317 |
| 1  | value_1 | 2022 | 1     

-- Filter by file location
SELECT id, value FROM my_data
WHERE location = 's3://bucket/data/year=2022/month=1/day=1/data_0.parquet';

-- Find recently modified files
SELECT DISTINCT location, last_modified FROM my_data
WHERE last_modified > '2024-01-01T00:00:00Z';

-- Aggregate by file
SELECT location, COUNT(*) AS row_count, size
FROM my_data
GROUP BY location, size
ORDER BY location;

-- Filter by file location
SELECT id, value FROM my_data
WHERE location = 's3://bucket/data/year=2022/month=1/day=1/data_0.parquet';

-- Find recently modified files
SELECT DISTINCT location, last_modified FROM my_data
WHERE last_modified > '2024-01-01T00:00:00Z';

-- Aggregate by file
SELECT location, COUNT(*) AS row_count, size
FROM my_data
GROUP BY location, size
ORDER BY location;

>>> ls -la
total 232
drwxr-sr-x@ 22 jeadie  staff    704 30 Jul 13:12 .
drwxr-sr-x@ 18 jeadie  staff    576 30 Jul 13:12 ..
-rw-r--r--@  1 jeadie  staff   1329 15 Jan  2024 DR-000-Template.md
-rw-r--r--@  1 jeadie  staff   4966 11 Aug  2023 DR-001-Dremio-Architecture.md
-rw-r--r--@  1 jeadie  staff   2307 28 Jul  2023 DR-002-Data-Completeness.md

>>> ls -la
total 232
drwxr-sr-x@ 22 jeadie  staff    704 30 Jul 13:12 .
drwxr-sr-x@ 18 jeadie  staff    576 30 Jul 13:12 ..
-rw-r--r--@  1 jeadie  staff   1329 15 Jan  2024 DR-000-Template.md
-rw-r--r--@  1 jeadie  staff   4966 11 Aug  2023 DR-001-Dremio-Architecture.md
-rw-r--r--@  1 jeadie  staff   2307 28 Jul  2023 DR-002-Data-Completeness.md

datasets:
  - name: my_documents
    from: file:docs/decisions/
    params:
      file_format: md

datasets:
  - name: my_documents
    from: file:docs/decisions/
    params:
      file_format: md

>>> SELECT * FROM my_documents LIMIT 3
+----------------------------------------------------+--------------------------------------------------+
| location                                           | content                                          |
+----------------------------------------------------+--------------------------------------------------+
| Users/docs/decisions/DR-000-Template.md            | # DR-000: DR Template                            |
|                                                    | **Date:** <>                                     |
|                                                    | **Decision Makers:**                             |
|                                                    | - @<>                                            |
|                                                    | - @<>                                            |

>>> SELECT * FROM my_documents LIMIT 3
+----------------------------------------------------+--------------------------------------------------+
| location                                           | content                                          |
+----------------------------------------------------+--------------------------------------------------+
| Users/docs/decisions/DR-000-Template.md            | # DR-000: DR Template                            |
|                                                    | **Date:** <>                                     |
|                                                    | **Decision Makers:**                             |
|                                                    | - @<>                                            |
|                                                    | - @