forked from mlflow/recipes-examples
-
Notifications
You must be signed in to change notification settings - Fork 0
/
databricks.py
120 lines (71 loc) · 2.31 KB
/
databricks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# Databricks notebook source
# MAGIC %md
# MAGIC # MLflow Classification Recipe Databricks Notebook
# MAGIC This notebook runs the MLflow Classification Recipe on Databricks and inspects its results.
# MAGIC
# MAGIC For more information about the MLflow Classification Recipe, including usage examples,
# MAGIC see the [Classification Recipe overview documentation](https://mlflow.org/docs/latest/recipes.html#classification-recipe)
# MAGIC and the [Classification Recipe API documentation](https://mlflow.org/docs/latest/python_api/mlflow.recipes.html#module-mlflow.recipes.classification.v1.recipe).
# COMMAND ----------
# MAGIC %pip install -r ../../requirements.txt
# COMMAND ----------
# MAGIC %md ### Start with a recipe:
# COMMAND ----------
from mlflow.recipes import Recipe
r = Recipe(profile="databricks")
# COMMAND ----------
r.clean()
# COMMAND ----------
# MAGIC %md ### Inspect recipe DAG:
# COMMAND ----------
r.inspect()
# COMMAND ----------
# MAGIC %md ### Ingest the dataset:
# COMMAND ----------
r.run("ingest")
# COMMAND ----------
# MAGIC %md ### Perform some EDA on the ingested dataset
# COMMAND ----------
import matplotlib.pyplot as plt
import seaborn as sns
ingested_data = r.get_artifact("ingested_data")
dims = (3, 4)
f, axes = plt.subplots(dims[0], dims[1], figsize=(25, 15))
axis_i, axis_j = 0, 0
for col in ingested_data.columns:
if col == "is_red":
continue # Box plots cannot be used on indicator variables
sns.boxplot(
x=ingested_data["is_red"], y=ingested_data[col], ax=axes[axis_i, axis_j]
)
axis_j += 1
if axis_j == dims[1]:
axis_i += 1
axis_j = 0
# COMMAND ----------
# MAGIC %md ### Split the dataset into train, validation and test:
# COMMAND ----------
r.run("split")
# COMMAND ----------
r.run("transform")
# COMMAND ----------
# MAGIC %md ### Train the model:
# COMMAND ----------
r.run("train")
# COMMAND ----------
# MAGIC %md ### Evaluate the model:
# COMMAND ----------
r.run("evaluate")
# COMMAND ----------
# MAGIC %md ### Register the model:
# COMMAND ----------
r.run("register")
# COMMAND ----------
r.inspect("train")
# COMMAND ----------
training_data = r.get_artifact("training_data")
training_data.describe()
# COMMAND ----------
trained_model = r.get_artifact("model")
print(trained_model)
# COMMAND ----------