update post meta from array

i want to update post metas from an array if some other custom field has value i've tried this but it always updates the post meta from the second array only "pa_size", what i want to achieve is that if $color has value then it adds that attribute to the post, if it doesn't then it doesn't add that attribute and same applies on $size

//get post metas
$color = get_post_meta( $post->ID, 'color', true);
$size = get_post_meta( $post->ID, 'size', true);
//set attributes from custom fields
$colors = get_post_meta( $post->ID, 'color', false);
$sizes = get_post_meta( $post->ID, 'size', false);

$attr_color = array ( 
        'pa_color' => array (
                  'name' => 'pa_color',
                  'value' => '',
                  'position' => 1,
                  'is_visible' => 1,
                  'is_variation' => 1,
                  'is_taxonomy' => 1,
               ),
    );

    $attr_size = array ( 
         'pa_size' => array (
                  'name' => 'pa_size',
                  'value' => '',
                  'position' => 2,
                  'is_visible' => 1,
                  'is_variation' => 1,
                  'is_taxonomy' => 1,
               ),
    );
if (!empty ($color)){
wp_set_object_terms($post->ID, $colors, 'pa_color');
update_post_meta($post->ID,'_product_attributes',  $attr_color);
}
if (!empty ($size)){
wp_set_object_terms($post->ID, $sizes, 'pa_size');
update_post_meta($post->ID,'_product_attributes',  $attr_size);
}

also i've tried this way

//get post metas
$color = get_post_meta( $post->ID, 'color', true);
$size = get_post_meta( $post->ID, 'size', true);
//set attributes from custom fields
$colors = get_post_meta( $post->ID, 'color', false);
$sizes = get_post_meta( $post->ID, 'size', false);

$attr = array ( 
        'pa_color' => array (
                  'name' => 'pa_color',
                  'value' => '',
                  'position' => 1,
                  'is_visible' => 1,
                  'is_variation' => 1,
                  'is_taxonomy' => 1,
               ),
         'pa_size' => array (
                  'name' => 'pa_size',
                  'value' => '',
                  'position' => 2,
                  'is_visible' => 1,
                  'is_variation' => 1,
                  'is_taxonomy' => 1,
               ),
    );
if (!empty ($color)){
wp_set_object_terms($post->ID, $colors, 'pa_color');
update_post_meta($post->ID,'_product_attributes',  $attr['pa_color']);
}
if (!empty ($size)){
wp_set_object_terms($post->ID, $sizes, 'pa_size');
update_post_meta($post->ID,'_product_attributes',  $attr['pa_size']);
}

but this method doesn't update anything on the post