Use case: Hair and face segmentation#
If you’re training a neural network to differentiate between a human face and human hair, you probably want to keep the identity of an actor static while adjusting their surroundings, their worn accessories, the tilt and location of their head, their facial expression, their eyes, and their hair - both head hair and facial hair.
The code on this page demonstrates how to use our SDK to build a data request with that goal in mind. The data request will generate 5 datapoints for each of 10 different actors.
When you submit the data request to our platform, each actor will be rendered 5 times, with the following variance:
Different hairstyles and colors
Different accessories
Different tilt of the head
Different location of the human head in the frame
Different expressions on the human’s face
Different eyes
Different backgrounds
Different facial hair
Each image will of course be accompanied by metadata giving you its ground truth: the actor’s optical and visual gaze vectors as well as their target of gaze in 3D space.
Tip
Click here to download the raw python code used in this sample:
This page contains an interactive Python kernel that installs our PyPi package on pageload.
!pip install datagen-tech -q
from datagen import api as datapoint_api
from datagen.api import catalog, assets
from datagen.api.catalog import attributes
from datagen.api.catalog.attributes import Gender, Ethnicity, Age
from datagen_protocol.schema.humans.human import HumanAttributes, HairAttributes
import random
# random.seed(5263)
# Getting catalogs
BACKGROUNDS = catalog.backgrounds.get()
BEARDS = catalog.beards.get()
EYES = catalog.eyes.get()
def random_sampler(options, obj=None):
"""
A helper function to randomize the attributes of an asset
:param options: the attributes that we want to randomize. They are specified in each of the functions below
:param obj: an object from our assets/attributes catalogs
:return: a float (for "RANDOM": sampling method) or the chosen/updated asset
"""
result = None
params = {}
for attribute, range_parameters in options.items():
if type(range_parameters) != tuple:
params[attribute] = options[attribute]
continue
elif range_parameters[-1] == "RANDOM":
result = random.random()
elif range_parameters[-1] == "ASSET_UNIFORM_CHOICE":
result = random.choice(range_parameters[0])
elif range_parameters[-1] == "UNIFORM":
params[attribute] = random.uniform(range_parameters[0], range_parameters[1])
elif range_parameters[-1] == "UNIFORM_CHOICE":
params[attribute] = random.choice(range_parameters[0])
elif range_parameters[-1] == "WEIGHTED_CHOICE":
params[attribute] = random.choices(range_parameters[0], weights=range_parameters[1])[0]
else:
print(f"{range_parameters[-1]} is not a supported sampling method")
raise NameError
if result:
return result
params = obj(**params)
return params
def get_single_camera(res=(512, 512), fov=(15, 15), cam_loc=assets.Point(x=0, y=-1.6, z=0.12), cam_rot=assets.Rotation(yaw=0.0, pitch=0.0, roll=0.0)):
"""
Sets the camera according to the chosen intrinsic and extrinsic parameters
:return: a camera asset
"""
return assets.Camera(
name='camera',
intrinsic_params=assets.IntrinsicParams(
projection=assets.Projection.PERSPECTIVE,
resolution_width=res[0],
resolution_height=res[1],
fov_horizontal=fov[0],
fov_vertical=fov[1],
wavelength=assets.Wavelength.VISIBLE
),
extrinsic_params=assets.ExtrinsicParams(
location=cam_loc,
rotation=cam_rot
)
)
# Randomizing functions: in each of the following functions, define the weights, the ranges and the options lists as
# indicated in each function below.
def get_random_human(gender_probs=(1 / 2, 1 / 2), age_probs=(1 / 3, 1 / 3, 1 / 3), eth_probs=(1 / 6, 1 / 6, 1 / 6, 1 / 6, 1 / 6, 1 / 6)):
"""
Gets a random ID according to the specified filters below
Edit the weights in order to change the distribution of the generated identity
:return: a human asset according to the chosen guidelines
"""
human_controls = {
"gender": ([Gender.FEMALE, Gender.MALE], list(gender_probs), "WEIGHTED_CHOICE"),
"age": ([Age.YOUNG, Age.ADULT, Age.OLD], list(age_probs), "WEIGHTED_CHOICE"),
"ethnicity": ([Ethnicity.AFRICAN, Ethnicity.HISPANIC, Ethnicity.MEDITERRANEAN, Ethnicity.NORTH_EUROPEAN,
Ethnicity.SOUTHEAST_ASIAN, Ethnicity.SOUTH_ASIAN],
list(eth_probs), "WEIGHTED_CHOICE")
}
human_attrs = random_sampler(options=human_controls, obj=HumanAttributes) # FIXME: human attrs?
random_human = random_sampler(options={"human": (catalog.humans.get(gender=human_attrs.gender, age=human_attrs.age, ethnicity=human_attrs.ethnicity), "ASSET_UNIFORM_CHOICE")})
return random_human
def set_random_head_rotation(yaw_angle_range=(-30, 30), pitch_angle_range=(-20, 20), roll_angle_range=(-10, 10)):
"""
Sets a random rotation to the head.
Edit the yaw, pitch and roll ranges.
Please refer to our documentation for more information (such as min/max ranges values).
Default values: (yaw=0, pitch=0, roll=0)
:return: The updated HeadRotation asset
"""
head_rotation_controls = {
"yaw": (yaw_angle_range[0], yaw_angle_range[1], "UNIFORM"),
"pitch": (pitch_angle_range[0], pitch_angle_range[1], "UNIFORM"),
"roll": (roll_angle_range[0], roll_angle_range[1], "UNIFORM")
}
return random_sampler(options=head_rotation_controls, obj=assets.Rotation)
def set_random_head_location(x_range=(-0.1, 0.1), y_range=(-0.0, 0.0), z_range=(-0.08, 0.08)):
"""
Sets a random location to the head.
Edit the yaw, pitch and roll ranges.
Please refer to our documentation for more information (such as min/max ranges values).
Default values: (yaw=0, pitch=0, roll=0)
:return: The updated HeadRotation asset
"""
head_location_controls = {
"x": (x_range[0], x_range[1], "UNIFORM"),
"y": (y_range[0], y_range[1], "UNIFORM"),
"z": (z_range[0], z_range[1], "UNIFORM")
}
return random_sampler(options=head_location_controls, obj=assets.Point)
def set_random_hair_color(mel_range=(0, 1), red_range=(0, 1), white_range=(0, 1), rough_range=(0.15, 0.5), ior_range=(1.4, 1.65)):
"""
Sets a random hair color to the chosen human.
Please refer to our documentation in order to learn more about how we control the hair color.
If not used, each identity has its own default hair color.
:return: The updated HairColor asset
"""
hair_color_controls = {
"melanin": (mel_range[0], mel_range[1], "UNIFORM"),
"redness": (red_range[0], red_range[1], "UNIFORM"),
"whiteness": (white_range[0], white_range[1], "UNIFORM"),
"roughness": (rough_range[0], rough_range[1], "UNIFORM"),
"index_of_refraction": (ior_range[0], ior_range[1], "UNIFORM")
}
return random_sampler(options=hair_color_controls, obj=assets.HairColor)
def set_random_expression(intensity_range=(0.1, 1)):
"""
Sets a random expressions and intensity.
Default value: NEUTRAL
:return: The updated ExpressionName asset
"""
expression_controls = {
"name": (list(assets.ExpressionName), "UNIFORM_CHOICE"),
"intensity": (intensity_range[0], intensity_range[1], "UNIFORM")
}
return random_sampler(options=expression_controls, obj=assets.Expression)
def randomize_glasses_controls(chosen_glasses, metalness_range=(0, 1), reflec_range=(0, 1), transparency_range=(0, 1)):
"""
Randomizes the glasses attributes
Please refer to our documentation in order to learn more about how we control the different glasses attributes
:param chosen_glasses: a random Glasses asset after it has been chosen
:return: The updated Glasses asset
"""
if chosen_glasses is not None:
glasses_controls = {
"id": chosen_glasses.id,
"frame_color": (list(assets.Color), "UNIFORM_CHOICE"),
"frame_metalness": (metalness_range[0], metalness_range[1], "UNIFORM"),
"lens_color": (list(assets.Color), "UNIFORM_CHOICE"),
"lens_reflectivity": (reflec_range[0], reflec_range[1], "UNIFORM"),
"lens_transparency": (transparency_range[0], transparency_range[1], "UNIFORM"),
"position": (list(assets.GlassesPosition), "UNIFORM_CHOICE")
}
chosen_glasses = random_sampler(options=glasses_controls, obj=assets.Glasses)
return chosen_glasses
def randomize_mask_controls(chosen_mask, roughness_range=(0, 1)):
"""
Randomizes the mask attributes
Please refer to our documentation in order to learn more about how we control the different mask attributes
:param chosen_mask: a Mask asset after it has been chosen
:return: The updated Mask asset
"""
if chosen_mask is not None:
mask_controls = {
"id": chosen_mask.id,
"color": (list(assets.Color), "UNIFORM_CHOICE"),
"texture": (list(assets.MaskTexture), "UNIFORM_CHOICE"),
"position": (list(assets.MaskPosition), "UNIFORM_CHOICE"),
"roughness": (roughness_range[0], roughness_range[1], "UNIFORM"),
}
chosen_mask = random_sampler(options=mask_controls, obj=assets.Mask)
return chosen_mask
def randomize_background_controls(chosen_background, rot_range=(0, 359), transparent_background="False"):
"""
Randomizes the rotation angle of the background's HDRI and defines whether the background is transparent
:param chosen_background: a Background asset after it has been chosen
:param rot_range: angle range for the HDRI rotation
:param transparent_background: choose if the background is transparent or not
:return: The updated Background asset
"""
background_controls = {
"id": chosen_background.id,
"rotation": (rot_range[0], rot_range[1], "UNIFORM"),
"transparent": transparent_background
}
return random_sampler(options=background_controls, obj=assets.Background)
def set_random_gaze_direction(x_range=(-0.5, 0.5), y_range=(-1.0, -0.85), z_range=(-0.5, 0.3)):
"""
Control the eyes gaze vector values
Please refer to our documentation for more information and examples.
Default values: x=0, y=-1, z=0
:return: The updated Gaze asset
"""
eyes_direction = {
"x": (x_range[0], x_range[1], "UNIFORM"),
"y": (y_range[0], y_range[0], "UNIFORM"),
"z": (z_range[0], z_range[0], "UNIFORM"),
}
return assets.Gaze(direction=random_sampler(options=eyes_direction, obj=assets.Vector))
def randomize_eyes_controls(eyes):
gaze_direction = set_random_gaze_direction()
if gaze_direction.direction.z >= 0:
closure = (0.0, 0.0, "UNIFORM")
else:
min_closure = -gaze_direction.direction.z
max_closure = 1.0
closure = (min_closure, max_closure, "UNIFORM")
eyes_controls = {
"id": eyes.id,
"eyelid_closure": closure,
"target_of_gaze": gaze_direction
}
return random_sampler(options=eyes_controls, obj=assets.Eyes)
def get_random_eyes(eyes_arr):
return random_sampler(options={"eyes": (eyes_arr, "ASSET_UNIFORM_CHOICE")})
def get_random_background(backgrounds_arr):
return random_sampler(options={"background": (backgrounds_arr, "ASSET_UNIFORM_CHOICE")})
def get_random_accessory(chosen_human, accessory_prob=0.0, glasses_prob=0.0):
glasses = None
mask = None
if random_sampler(options={"random": ("RANDOM",)}) < accessory_prob:
if random_sampler(options={"random": ("RANDOM",)}) < glasses_prob:
glasses = random_sampler(options={"glasses": (catalog.glasses.get(gender=chosen_human.attributes.gender), "ASSET_UNIFORM_CHOICE")})
randomize_glasses_controls(glasses)
else:
mask = random_sampler(options={"mask": (catalog.masks.get(gender=chosen_human.attributes.gender), "ASSET_UNIFORM_CHOICE")})
randomize_mask_controls(mask)
return glasses, mask
def get_random_hairstyle(chosen_human):
return random_sampler(options={"hair": (catalog.hair.get(age_group_match=chosen_human.attributes.age, gender_match=chosen_human.attributes.gender, ethnicity_match=chosen_human.attributes.ethnicity), "ASSET_UNIFORM_CHOICE")})
def get_random_beard(chosen_human):
chosen_human.head.facial_hair = random_sampler(options={"beard": (BEARDS, "ASSET_UNIFORM_CHOICE")})
chosen_human.head.facial_hair.color_settings = chosen_human.head.hair.color_settings
return chosen_human.head.facial_hair
First we will define the basic parameters of our dataset:
from datagen import api as datapoint_api
from datagen.api import catalog, assets
from datagen.api.catalog import attributes
from datagen.api.catalog.attributes import Gender, Ethnicity, Age
from datagen_protocol.schema.humans.human import HumanAttributes, HairAttributes
import random
NUMBER_OF_IDS = 10
NUMBER_OF_VARIATIONS_PER_ID = 5
ACCESSORY_PROB = 0.6 # Probability that a datapoint will include an accessory
# (glasses or mask, but not both)
GLASSES_PROB = 0.4 # When there is an accessory, mask probability is: 1 - GLASSES_PROB
BEARD_PROB = 0.7 # Probability of facial hair if ID is male and without a mask
IMAGE_RESOLUTION = (512, 512) # (width, height)
DST_JSON_PATH = 'dataset_request_face_hair_segmentation.json' # Path to where we will save the
# data request as a JSON file
Next, we will create a function that randomizes only specific parts of the scene.
Note
This block of code uses a number of functions from templates_functions.py.
def get_single_random_datapoint(chosen_human, cam):
chosen_human.head.hair = get_random_hairstyle(chosen_human)
chosen_human.head.hair.color_settings = set_random_hair_color() # Randomizes the color and light
# reflection and refraction of the hair
glasses, mask = get_random_accessory(chosen_human,
accessory_prob=ACCESSORY_PROB,
glasses_prob=GLASSES_PROB) # Either glasses, mask, or none
if mask is None and chosen_human.attributes.gender == Gender.MALE and \
random_sampler(options={"random": ("RANDOM",)}) < BEARD_PROB:
chosen_human.head.facial_hair = get_random_beard(chosen_human)
else:
human.head.facial_hair = None
chosen_human.head.rotation = set_random_head_rotation() # Chooses a random yaw, pitch, and roll
# for the tilt of the actor's head
chosen_human.head.location = set_random_head_location() # Chooses a random location
# in the scene for the actor's head (x, y, z)
background = get_random_background(BACKGROUNDS)
background = randomize_background_controls(background) # Randomizes the rotation and transparency
chosen_human.head.eyes = randomize_eyes_controls(chosen_human.head.eyes) # Chooses a random gaze direction
# and eyelid closure
chosen_human.head.expression = set_random_expression() # Chooses a random expression type and intensity
kwargs = {'human': chosen_human, 'camera': cam, 'background': background}
if mask is not None:
kwargs['mask'] = mask
if glasses is not None:
kwargs['glasses'] = glasses
random_dp = datapoint_api.create_datapoint(**kwargs)
return random_dp
Now we can run the for loop to create our request for 5 randomized images of 10 humans:
datapoints = []
camera = get_single_camera(res=IMAGE_RESOLUTION)
for i in range(NUMBER_OF_IDS):
print(f'Processing ID number {i+1} out of {NUMBER_OF_IDS}...')
human = get_random_human()
for _ in range(NUMBER_OF_VARIATIONS_PER_ID):
random_datapoint = get_single_random_datapoint(human, camera)
datapoints.append(random_datapoint)
And finally we dump the complete data request into a json file:
datapoint_api.dump(assets.DataRequest(datapoints=datapoints), path=DST_JSON_PATH)
Because this browser-based notebook runs the code in an isolated kernel, we need to take one last step to extract the JSON file for download. (This step is not necessary when you are working in your local environment.)
from IPython.display import HTML
import os, base64
def make_download_link(DST_JSON_PATH):
filename = os.path.basename(DST_JSON_PATH)
with open(DST_JSON_PATH, "rb") as f:
data = base64.b64encode(f.read()).decode()
return HTML(f'<h4>Click here to download '
f'<a href="data:application/binary;base64,{data}" '
f'download={filename}>{filename}</a>.</h4>')
make_download_link(DST_JSON_PATH)
Take this JSON file and upload it to the Datagen platform. Generation of your data will start immediately.