aset_id="<WHYLABS_DATASET_ID>"

@step(

settings={"data_validator.whylogs": whylogs_settings

def data_profiler(

dataset: pd.DataFrame,

) -> DatasetProfileView:

"""Custom data profiler step with whylogs

Args:

dataset: a Pandas DataFrame

Returns:

Whylogs profile generated for the data

"""

# validation pre-processing (e.g. dataset preparation) can take place here

data_validator = WhylogsDataValidator.get_active_data_validator()

profile = data_validator.data_profiling(

dataset,

# optionally upload the profile to WhyLabs, if WhyLabs credentials are configured

data_validator.upload_profile_view(profile)

# validation post-processing (e.g. interpret results, take actions) can happen here

return profile

Have a look at the complete list of methods and parameters available in the WhylogsDataValidator API in the SDK docs.

Call whylogs directly

You can use the whylogs library directly in your custom pipeline steps, and only leverage ZenML's capability of serializing, versioning and storing the DatasetProfileView objects in its Artifact Store. You may optionally enable whylabs logging to automatically upload the returned whylogs profile to WhyLabs, e.g.:

import pandas as pd

from whylogs.core import DatasetProfileView

import whylogs as why

from zenml import step

from zenml.integrations.whylogs.flavors.whylogs_data_validator_flavor import (

WhylogsDataValidatorSettings,

whylogs_settings = WhylogsDataValidatorSettings(

enable_whylabs=True, dataset_id="<WHYLABS_DATASET_ID>"

@step(

settings={

"data_validator.whylogs": whylogs_settings

def data_profiler(

dataset: pd.DataFrame,

) -> DatasetProfileView:

"""Custom data profiler step with whylogs

Args:

dataset: a Pandas DataFrame

Returns:

Whylogs Profile generated for the dataset

"""

# validation pre-processing (e.g. dataset preparation) can take place here

results = why.log(dataset)

profile = results.profile()

# validation post-processing (e.g. interpret results, take actions) can happen here

return profile.view()