-
Notifications
You must be signed in to change notification settings - Fork 31
Expand file tree
/
Copy pathadvanced-validation.qmd
More file actions
93 lines (83 loc) · 3.7 KB
/
Copy pathadvanced-validation.qmd
File metadata and controls
93 lines (83 loc) · 3.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
---
title: "Advanced Validation"
description: "A validation with a comprehensive set of rules."
image: "img/advanced_validation.png"
notebook-links: false
page-navigation: false
toc: false
html-table-processing: none
---
```{python}
#| echo: false
import pointblank as pb
import polars as pl
validation = (
pb.Validate(
data=pb.load_dataset(dataset="game_revenue", tbl_type="polars"),
tbl_name="game_revenue",
label="Comprehensive validation example",
thresholds=pb.Thresholds(warning=0.10, error=0.25, critical=0.35),
)
.col_vals_regex(columns="player_id", pattern=r"^[A-Z]{12}[0-9]{3}$") # STEP 1
.col_vals_gt(columns="session_duration", value=5) # STEP 2
.col_vals_ge(columns="item_revenue", value=0.02) # STEP 3
.col_vals_in_set(columns="item_type", set=["iap", "ad"]) # STEP 4
.col_vals_in_set( # STEP 5
columns="acquisition",
set=["google", "facebook", "organic", "crosspromo", "other_campaign"]
)
.col_vals_not_in_set(columns="country", set=["Mongolia", "Germany"]) # STEP 6
.col_vals_between( # STEP 7
columns="session_duration",
left=10, right=50,
pre = lambda df: df.select(pl.median("session_duration"))
)
.rows_distinct(columns_subset=["player_id", "session_id", "time"]) # STEP 8
.row_count_match(count=2000) # STEP 9
.col_count_match(count=11) # STEP 10
.col_vals_not_null(columns=pb.starts_with("item")) # STEPS 11-13
.col_exists(columns="start_day") # STEP 14
.interrogate()
)
validation
```
```python
import pointblank as pb
import polars as pl
validation = (
pb.Validate(
data=pb.load_dataset(dataset="game_revenue", tbl_type="polars"),
tbl_name="game_revenue",
label="Comprehensive validation example",
thresholds=pb.Thresholds(warning=0.10, error=0.25, critical=0.35),
)
.col_vals_regex(columns="player_id", pattern=r"^[A-Z]{12}[0-9]{3}$") # STEP 1
.col_vals_gt(columns="session_duration", value=5) # STEP 2
.col_vals_ge(columns="item_revenue", value=0.02) # STEP 3
.col_vals_in_set(columns="item_type", set=["iap", "ad"]) # STEP 4
.col_vals_in_set( # STEP 5
columns="acquisition",
set=["google", "facebook", "organic", "crosspromo", "other_campaign"]
)
.col_vals_not_in_set(columns="country", set=["Mongolia", "Germany"]) # STEP 6
.col_vals_between( # STEP 7
columns="session_duration",
left=10, right=50,
pre = lambda df: df.select(pl.median("session_duration"))
)
.rows_distinct(columns_subset=["player_id", "session_id", "time"]) # STEP 8
.row_count_match(count=2000) # STEP 9
.col_count_match(count=11) # STEP 10
.col_vals_not_null(columns=pb.starts_with("item")) # STEPS 11-13
.col_exists(columns="start_day") # STEP 14
.interrogate()
)
validation
```
<details>
<summary>Preview of Input Table</summary>
```{python}
# | echo: false
pb.preview(pb.load_dataset(dataset="game_revenue"), n_head=10, n_tail=10)
```
</details>