Detecting an object in an gradient background going wrong

Question

I am struggling to detect objects in an image where the background and the object have gradients applied, not only that but have transparency in the object as well, see them as holes in the object.

I've tried doing it with Sobel and more, and using GrabCut, with an background generation, and then compare the pixels from the original and the generated background with each other, where if the pixel in the original image deviates from the background pixel then that pixel is part of the object.

Results from the GrabCut system

Results from the Sobel system

The scripts:

GrabCut System:

#THE ONE USING GRABCUT
import cv2
import numpy as np
import sys
from concurrent.futures import ProcessPoolExecutor
import time

# ------------------ 1. GrabCut Segmentation ------------------
def run_grabcut(img, grabcut_iterations=5, border_margin=5):
    h, w = img.shape[:2]
    gc_mask = np.zeros((h, w), np.uint8)
    # Initialize borders as definite background
    gc_mask[:border_margin, :] = cv2.GC_BGD
    gc_mask[h-border_margin:, :] = cv2.GC_BGD
    gc_mask[:, :border_margin] = cv2.GC_BGD
    gc_mask[:, w-border_margin:] = cv2.GC_BGD
    # Everything else is set as probable foreground.
    gc_mask[border_margin:h-border_margin, border_margin:w-border_margin] = cv2.GC_PR_FGD

    bgdModel = np.zeros((1, 65), np.float64)
    fgdModel = np.zeros((1, 65), np.float64)

    try:
        cv2.grabCut(img, gc_mask, None, bgdModel, fgdModel, grabcut_iterations, cv2.GC_INIT_WITH_MASK)
    except Exception as e:
        print("ERROR: GrabCut failed:", e)
        return None, None


    fg_mask = np.where((gc_mask == cv2.GC_FGD) | (gc_mask == cv2.GC_PR_FGD), 255, 0).astype(np.uint8)
    return fg_mask, gc_mask


def generate_background_inpaint(img, fg_mask):
    
    inpainted = cv2.inpaint(img, fg_mask, inpaintRadius=3, flags=cv2.INPAINT_TELEA)
    return inpainted


def compute_final_object_mask_strict(img, background, gc_fg_mask, tol=5.0):

    # Convert both images to LAB
    lab_orig = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)
    lab_bg = cv2.cvtColor(background, cv2.COLOR_BGR2LAB)
    # Compute absolute difference per channel.
    diff = cv2.absdiff(lab_orig, lab_bg).astype(np.float32)
    # Compute Euclidean distance per pixel.
    diff_norm = np.sqrt(np.sum(diff**2, axis=2))
    # Create a mask: if difference exceeds tol, mark as object (255); else background (0).
    obj_mask = np.where(diff_norm > tol, 255, 0).astype(np.uint8)
    # Enforce GrabCut: where GrabCut says background (gc_fg_mask == 0), force object mask to 0.
    obj_mask[gc_fg_mask == 0] = 0
    return obj_mask


def process_image_strict(img, grabcut_iterations=5, tol=5.0):
    
    start_time = time.time()
    print("--- Processing Image (GrabCut + Inpaint + Strict Pixel Comparison) ---")
    
    # 1. Run GrabCut
    print("[Debug] Running GrabCut...")
    fg_mask, gc_mask = run_grabcut(img, grabcut_iterations=grabcut_iterations)
    if fg_mask is None or gc_mask is None:
        return None, None, None
    print("[Debug] GrabCut complete.")
    
    # 2. Generate Background via Inpainting.
    print("[Debug] Generating background via inpainting...")
    background = generate_background_inpaint(img, fg_mask)
    print("[Debug] Background generation complete.")
    
    # 3. Pure Pixel-by-Pixel Comparison in LAB with Tolerance.
    print(f"[Debug] Performing pixel comparison with tolerance={tol}...")
    final_mask = compute_final_object_mask_strict(img, background, fg_mask, tol=tol)
    print("[Debug] Pixel comparison complete.")
    
    total_time = time.time() - start_time
    print(f"[Debug] Total processing time: {total_time:.4f} seconds.")
    

    grabcut_disp_mask = fg_mask.copy()
    return grabcut_disp_mask, background, final_mask


def process_wrapper(args):
    img, version, tol = args
    print(f"Starting processing for image {version+1}")
    result = process_image_strict(img, tol=tol)
    print(f"Finished processing for image {version+1}")
    return result, version

def main():
    # Load images (from command-line or defaults)
    path1 = sys.argv[1] if len(sys.argv) > 1 else "test_gradient.png"
    path2 = sys.argv[2] if len(sys.argv) > 2 else "test_gradient_1.png"
    img1 = cv2.imread(path1)
    img2 = cv2.imread(path2)
    if img1 is None or img2 is None:
        print("Error: Could not load one or both images.")
        sys.exit(1)
    images = [img1, img2]


    tolerance_value = 5.0


    with ProcessPoolExecutor(max_workers=2) as executor:
        futures = {executor.submit(process_wrapper, (img, idx, tolerance_value)): idx for idx, img in enumerate(images)}
        results = [f.result() for f in futures]

    # Display results.
    for idx, (res, ver) in enumerate(results):
        if res is None:
            print(f"Skipping display for image {idx+1} due to processing error.")
            continue
        grabcut_disp_mask, generated_bg, final_mask = res
        disp_orig = cv2.resize(images[idx], (480, 480))
        disp_grabcut = cv2.resize(grabcut_disp_mask, (480, 480))
        disp_bg = cv2.resize(generated_bg, (480, 480))
        disp_final = cv2.resize(final_mask, (480, 480))
        combined = np.hstack([
            disp_orig,
            cv2.merge([disp_grabcut, disp_grabcut, disp_grabcut]),
            disp_bg,
            cv2.merge([disp_final, disp_final, disp_final])
        ])
        window_title = f"Image {idx+1} (Orig | GrabCut FG | Gen Background | Final Mask)"
        cv2.imshow(window_title, combined)
    print("Displaying results. Press any key to close.")
    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()

Sobel system:

import cv2
import numpy as np
import sys
from concurrent.futures import ProcessPoolExecutor


def get_background_constraint_mask(image):
    
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    # Compute Sobel gradients.
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)
    mag = np.sqrt(sobelx**2 + sobely**2)
    mag = np.uint8(np.clip(mag, 0, 255))
    # Hard–set threshold = 0: any nonzero gradient is an edge.
    edge_map = np.zeros_like(mag, dtype=np.uint8)
    edge_map[mag > 0] = 255
    # No morphological processing is done so that maximum sensitivity is preserved.
    inv_edge = cv2.bitwise_not(edge_map)
    h, w = inv_edge.shape
    flood_filled = inv_edge.copy()
    ff_mask = np.zeros((h+2, w+2), np.uint8)
    for j in range(w):
        if flood_filled[0, j] == 255:
            cv2.floodFill(flood_filled, ff_mask, (j, 0), 128)
        if flood_filled[h-1, j] == 255:
            cv2.floodFill(flood_filled, ff_mask, (j, h-1), 128)
    for i in range(h):
        if flood_filled[i, 0] == 255:
            cv2.floodFill(flood_filled, ff_mask, (0, i), 128)
        if flood_filled[i, w-1] == 255:
            cv2.floodFill(flood_filled, ff_mask, (w-1, i), 128)
    background_mask = np.zeros_like(flood_filled, dtype=np.uint8)
    background_mask[flood_filled == 128] = 255
    return background_mask


def generate_background_from_constraints(image, fixed_mask, max_iters=5000, tol=1e-3):
    
    H, W, C = image.shape
    if fixed_mask.shape != (H, W):
        raise ValueError("Fixed mask shape does not match image shape.")
    fixed = (fixed_mask == 255)
    fixed[0, :], fixed[H-1, :], fixed[:, 0], fixed[:, W-1] = True, True, True, True
    new_img = image.astype(np.float32).copy()
    for it in range(max_iters):
        old_img = new_img.copy()
        cardinal = (old_img[1:-1, 0:-2] + old_img[1:-1, 2:] +
                    old_img[0:-2, 1:-1] + old_img[2:, 1:-1])
        diagonal = (old_img[0:-2, 0:-2] + old_img[0:-2, 2:] +
                    old_img[2:, 0:-2] + old_img[2:, 2:])
        weighted_avg = (diagonal + 2 * cardinal) / 12.0
        free = ~fixed[1:-1, 1:-1]
        temp = old_img[1:-1, 1:-1].copy()
        temp[free] = weighted_avg[free]
        new_img[1:-1, 1:-1] = temp
        new_img[fixed] = image.astype(np.float32)[fixed]
        diff = np.linalg.norm(new_img - old_img)
        if diff < tol:
            break
    return new_img.astype(np.uint8)

def compute_final_object_mask(image, background):
    
    lab_orig = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)
    lab_bg   = cv2.cvtColor(background, cv2.COLOR_BGR2LAB)
    diff_lab = cv2.absdiff(lab_orig, lab_bg).astype(np.float32)
    diff_norm = np.sqrt(np.sum(diff_lab**2, axis=2))
    diff_norm_8u = cv2.convertScaleAbs(diff_norm)
    auto_thresh = cv2.threshold(diff_norm_8u, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[0]
    # Define weak threshold as 90% of auto_thresh:
    weak_thresh = 0.9 * auto_thresh
    strong_mask = diff_norm >= auto_thresh
    weak_mask   = diff_norm >= weak_thresh
    final_mask = np.zeros_like(diff_norm, dtype=np.uint8)
    final_mask[strong_mask] = 255
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
    prev_sum = 0
    while True:
        dilated = cv2.dilate(final_mask, kernel, iterations=1)
        new_mask = np.where((weak_mask) & (dilated > 0), 255, final_mask)
        current_sum = np.sum(new_mask)
        if current_sum == prev_sum:
            break
        final_mask = new_mask
        prev_sum = current_sum
    final_mask = cv2.morphologyEx(final_mask, cv2.MORPH_CLOSE, kernel)
    return final_mask


def process_image(img):
    
    constraint_mask = get_background_constraint_mask(img)
    background = generate_background_from_constraints(img, constraint_mask)
    final_mask = compute_final_object_mask(img, background)
    return constraint_mask, background, final_mask


def process_wrapper(args):
    img, version = args
    result = process_image(img)
    return result, version

def main():
    # Load two images: default file names.
    path1 = sys.argv[1] if len(sys.argv) > 1 else "test_gradient.png"
    path2 = sys.argv[2] if len(sys.argv) > 2 else "test_gradient_1.png"
    
    img1 = cv2.imread(path1)
    img2 = cv2.imread(path2)
    if img1 is None or img2 is None:
        print("Error: Could not load one or both images.")
        sys.exit(1)
    images = [img1, img2]  # Use images as loaded (blue gradient is original).
    
    with ProcessPoolExecutor(max_workers=2) as executor:
        futures = [executor.submit(process_wrapper, (img, idx)) for idx, img in enumerate(images)]
        results = [f.result() for f in futures]
    
    for idx, (res, ver) in enumerate(results):
        constraint_mask, background, final_mask = res
        disp_orig = cv2.resize(images[idx], (480,480))
        disp_cons = cv2.resize(constraint_mask, (480,480))
        disp_bg   = cv2.resize(background, (480,480))
        disp_final = cv2.resize(final_mask, (480,480))
        combined = np.hstack([
            disp_orig,
            cv2.merge([disp_cons, disp_cons, disp_cons]),
            disp_bg,
            cv2.merge([disp_final, disp_final, disp_final])
        ])
        cv2.imshow(f"Output Image {idx+1}", combined)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()

Original images:

I really appreciate ANY help!

Answer 1

I worked a bit in image detection alghoritms.

To find shapes in noisy images I've used canny edge detection: it's a more sophisticated edge detection algorithm that considers gradient magnitude and direction and applies hysteresis thresholding to link edges. It might be more effective, in your use case, with gradients.

import cv2
import numpy as np

img = None  # your image
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
blurred = cv2.GaussianBlur(gray, (5, 5), 0)  # Gaussian smoothing, experiment with different values, here

# add and experiment with values here also
low_threshold = 
high_threshold =  
contours = cv2.Canny(blurred, low_threshold, high_threshold)

# contours will be a binary image where white pixels represent detected edges
cv2.imshow("contours", contours)
cv2.waitKey(0)
cv2.destroyAllWindows()

The optimal low_threshold and high_threshold values for Canny depend heavily on your image (noise level, gradient strength). You'll likely need to experiment with different values (that's why I left them empty). A common heuristic is to have the high_threshold be 2 or 3 times the low_threshold, but that's not a strict rule