import dagster as dg
import polars as pl
import datetime as dt
from HHM_dagster.lib.io_managers import IOManagerKeys
@dg.asset(
key_prefix=["issues"],
group_name="issues",
partitions_def=dg.DailyPartitionsDefinition(
start_date=dt.datetime(2024, 1, 1), timezone="Europe/Brussels", fmt="%Y-%m-%d"
),
metadata={"partition_expr": "partition_key"},
io_manager_key=IOManagerKeys.BIGQUERY_IO_MANAGER.value,
)
def my_upstream_partitioned_asset(context: dg.AssetExecutionContext) -> pl.DataFrame:
# Your asset logic here
return (
pl.DataFrame({"partition_key": [context.partition_key]})
.with_columns(pl.col("partition_key").str.to_date("%Y-%m-%d"))
.with_columns(pl.col("partition_key").alias("date"))
)
content of my resources :
IOManagerKeys.BIGQUERY_IO_MANAGER.value: PolarsBigQueryIOManager(
project=settings.gcp_project.name,
location=bigquery_zone,
gcp_credentials=settings.gcp_project.bigquery_e64,
)
When I launch the asset I get the following output metadata which confirms that the polars dataframe, before being loaded in bigquery has Date columns :
However in bigquery, the data is stored as timestamp
Why is that and how to solve this?
This is both for the partition column and the not partition column
content of my resources :
When I launch the asset I get the following output metadata which confirms that the polars dataframe, before being loaded in bigquery has Date columns :
However in bigquery, the data is stored as timestamp
Why is that and how to solve this?
This is both for the partition column and the not partition column