polishing up

.gitignore

@@ -32,3 +32,4 @@ yarn-error.log*
 .vercel
 
 /image_proc_server/__pycache__
+image_proc_server/imgproc/

components/textinput.js

@@ -13,7 +13,7 @@ function TextInput({ onTextChange }) {
   return (
     <div className={styles.textInputContainer}>
       
-      <textarea id="text-input" wrap="soft" type="text" value={text} onChange={handleTextChange} className={styles.textInput} placeholder='Describe your scene here...'/>
+      <textarea id="text-input" wrap="soft" type="text" value={text} onChange={handleTextChange} className={styles.textInput} placeholder='Describe where the item should be placed. e.g. "in the grass", or "on the moon"'/>
     </div>
   );
 }

image_proc_server/app.py

@@ -9,9 +9,6 @@ from flask_cors import CORS
 app = Flask(__name__)
 CORS(app)
 
-# authorization for removebg
-removebg_api_key = "YOUR_API_KEY"
-
 @app.route('/get_item_mask', methods=['POST'])
 def get_item_mask():
 
@@ -22,128 +19,45 @@ def get_item_mask():
     image_bytes = image_file.read()
     image = Image.open(io.BytesIO(image_bytes))
 
-    image = image.convert('RGB')
+    image = image.convert('RGBA')
+    image.save('input.png')
 
     jpeg_buffer = io.BytesIO()
-    image.save(jpeg_buffer, format='JPEG')
 
-    #print(jpeg_buffer.getvalue())
+    ai_mask_image = convert_to_black_and_white(image)
 
-    # set API endpoint and headers
+    ai_mask_image.save("output.png")
-    api_url = "https://api.remove.bg/v1.0/removebg"
+    ai_mask_image = ai_mask_image.convert('RGB')
-    headers = {"X-Api-Key": removebg_api_key}
+    ai_mask_image.save(jpeg_buffer, format="JPEG")
+    ai_mask_image_base64 = base64.b64encode(jpeg_buffer.getvalue()).decode('utf-8')
 
-    # send the request to remove.bg API
+    return {'ai_mask': ai_mask_image_base64}
-    response = requests.post(api_url, headers=headers, files={'image_file': jpeg_buffer.getvalue()})
-
-
-    # Check if the API returned a successful response
-    if response.status_code == requests.codes.ok:
-        
-        # print("response is ok")
-        # print("Success:", response.status_code, response.text)
-
-        # Save the PNG image to a file
-        with open('output.png', 'wb') as out_file:
-            out_file.write(response.content)
-
-        png_buffer = io.BytesIO()
-        ux_mask_buffer = io.BytesIO()
-
-        response_image = Image.open(io.BytesIO(response.content))
-        response_image = response_image.convert('RGBA')
-        
-        ai_mask_image = convert_to_black_and_white(response_image)
-        ai_mask_image = ai_mask_image.convert('RGB')
-        ai_mask_image.save(png_buffer, format="JPEG")
-        ai_mask_image_base64 = base64.b64encode(png_buffer.getvalue()).decode('utf-8')
-        
-        
-        ux_mask_image = convert_to_green(response_image)
-        ux_mask_image = ux_mask_image.convert('RGBA')
-        ux_mask_image.save(ux_mask_buffer, format="PNG")
-
-        ux_mask_image.save('greenoutput.png')
-
-        ux_mask_image_base64 = base64.b64encode(ux_mask_buffer.getvalue()).decode('utf-8')
-
-        return {'ai_mask': ai_mask_image_base64, 'ux_mask': ux_mask_image_base64}
-
-    else:
-        # Print an error message if the API call was not successful
-        print("Error:", response.status_code, response.text)
-        return {'error': "There was a problem with the remove bg operation"}
 
 
 def convert_to_black_and_white(image):
-
-    
     # Create a new image with the same size as the original, but with a white background
     new_image = Image.new('RGB', image.size, (255, 255, 255))
     
-    # Create a mask for the alpha channel
+    # Ensure the image has an alpha channel
-    alpha_mask = image.split()[-1]
+    if image.mode != 'RGBA':
-    
+        image = image.convert('RGBA')
-    modified_alpha_mask = modify_alpha_mask(alpha_mask)
 
-    # Paste the original image onto the new image using the alpha channel as the mask
+    # Get pixel data
-    new_image.paste(image, mask=modified_alpha_mask)
+    data = image.getdata()
 
-    new_image.save('temp_mask_image.png')
+    # Create new image data
-    
+    new_data = []
-    # Convert the new image to grayscale
+    for item in data:
-    new_image = new_image.convert('L')
+        # If the pixel is fully transparent (alpha value of 0), set it to white. Else, set it to black.
+        if item[3] == 0:
+            new_data.append((255, 255, 255, 255))
+        else:
+            new_data.append((0, 0, 0, 255))
+
+    # Update new image data
+    new_image.putdata(new_data)
+
+    # Convert the final image to grayscale for black and white effect
+    black_and_white_image = new_image.convert('L')
 
-    new_image.save('temp_mask_image_grayscale.png')
-    
-    # Convert all non-alpha pixels to black and all alpha pixels to white
-    black_and_white_image = new_image.point(lambda x: 255 if x == 255 else 0, '1')
-    
     return black_and_white_image
-
-
-def modify_alpha_mask(alpha_mask):
-    # Ensure the input image has an alpha channel
-    if alpha_mask.mode != 'L':
-        alpha_mask = alpha_mask.convert('L')
-
-    # Create a new image with modified alpha values
-    modified_alpha_mask = Image.new('L', alpha_mask.size)
-
-    # Iterate over each pixel and modify the alpha value
-    for x in range(alpha_mask.width):
-        for y in range(alpha_mask.height):
-            alpha_value = alpha_mask.getpixel((x, y))
-
-            # Check if the alpha value is semi-transparent (not fully transparent or opaque)
-            if alpha_value > 0 and alpha_value < 100:
-                alpha_value = 0  # Set to fully transparent
-
-            # Set the modified alpha value in the new image
-            modified_alpha_mask.putpixel((x, y), alpha_value)
-
-    return modified_alpha_mask
-
-
-def convert_to_green(image):
-    # Create a new image with the same size as the original, but with a transparent background
-    new_image = Image.new('RGBA', image.size, (0, 0, 0, 0))
-    
-    # Create a mask for the alpha channel
-    alpha_mask = image.split()[-1]
-    
-    # Paste the original image onto the new image using the alpha channel as the mask
-    new_image.paste(image, mask=alpha_mask)
-    
-    # Create a new image with the same size as the original, but with 0.5 alpha green color
-    green_image = Image.new('RGBA', image.size, (255, 200, 0, 180))
-
-    # Create a mask for the alpha channel
-    alpha_mask = new_image.split()[-1]
-    
-    alpha_mask = ImageOps.invert(alpha_mask)
-
-    # Paste the green image onto the new image using the alpha channel as the mask
-    new_image.paste(green_image, mask=alpha_mask)
-    
-    return new_image

image_proc_server/requirements.txt

@@ -0,0 +1,14 @@
+blinker==1.6.3
+certifi==2023.7.22
+charset-normalizer==3.3.0
+click==8.1.7
+Flask==3.0.0
+Flask-Cors==4.0.0
+idna==3.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+Pillow==10.0.1
+requests==2.31.0
+urllib3==2.0.6
+Werkzeug==3.0.0

pages/index.js

@@ -11,44 +11,15 @@ import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader';
 import {CanvasComponent} from "@/components/canvascomponent";
 
 
-/*
-links for thereejs implementation
-https://codesandbox.io/s/basic-demo-forked-rnuve?file=/src/App.js
-
-*/
-
-
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
-
 export default function Studio(){
 
-    // const imageContainer = {
-    //     position: 'absolute',
-    //     top: '0',
-    //     left: '0',
-    //     backgroundColor: 'rgb(70, 232, 83)',
-    //     height: '100%',
-    //     width: '100%',
-    // };
-
-    const [canvasSnapshotUrl, setCanvasSnapshotUrl] = useState(null)
-    const [maskImageUrl, setMaskImageUrl] = useState(null);
-    const [uxMaskImageUrl, setUxMaskImageUrl] = useState(null);
-
-    const [imageFile, setImageFile] = useState(null)
-    const [modelFile, setModelFile] = useState(null);
-    const [modelBuffer, setModelBuffer] = useState(null);
     const [gltfModel, setGltfModel] = useState(null);
-    
-    const [isImgUploadVisible, setIsImgUploadVisible] = useState(null)
-    const [isMaskVisible, setIsMaskVisible] = useState(null)
     const [isResultVisible, setIsResultVisible] = useState(null)
-    const [isFlashingProgressVisible, setIsFlashingProgressVisible] = useState(null)
+    const [isLoadingVisible, setIsLoadingVisible] = useState(null)
-
     const [prediction, setPrediction] = useState(null);
     const [error, setError] = useState(null);
-
     const [inputValue, setInputValue] = useState('');
 
     // create a canvas reference in the main state
@@ -60,12 +31,8 @@ export default function Studio(){
 
     useEffect(() => {
         console.log("loaded the page");
-
-        // define visibiltiy of the 3 layers
-        setIsImgUploadVisible(true)
-        setIsMaskVisible(true)
         setIsResultVisible(false)
-        setIsFlashingProgressVisible(false)
+        setIsLoadingVisible(false)
 
     },[]);
 
@@ -97,7 +64,7 @@ export default function Studio(){
 
     const getReplicateResults = async (image, mask) => {
 
-        setIsFlashingProgressVisible(true)
+        setIsLoadingVisible(true)
 
         let promptText = "beautiful living room"
 
@@ -111,10 +78,7 @@ export default function Studio(){
             "Content-Type": "application/json",
           },
           body: JSON.stringify({
-            //prompt: e.target.prompt.value,
+            prompt: promptText + ", photorealistic, high resolution product photography",
-            //prompt: "high resolution photography of a beige interior living room with dining chairs, around dining table, wooden floor, beige blue salmon pastel, sunlight, contrast, realistic artstation concept art, hyperdetailed, ultradetail, cinematic 8k, architectural rendering, unreal engine 5, rtx, volumetric light, cozy atmosphere",
-            //prompt: "minimalist kitchen, wooden floor, beige blue salmon pastel, sunlight, contrast, realistic artstation concept art, hyperdetailed, ultradetail, cinematic 8k, architectural rendering, unreal engine 5, rtx, volumetric light, cozy atmosphere",
-            prompt: promptText + ", creative marketing advertisement",
             negative_prompt: "blurry, painting, cartoon, abstract, ugly, deformed",
             image: image,
             mask: mask,
@@ -142,10 +106,9 @@ export default function Studio(){
           }
 
           if (prediction.status == "succeeded" && prediction.output) {
-            setIsImgUploadVisible(true)
+
-            setIsMaskVisible(true)
             setIsResultVisible(true)
-            setIsFlashingProgressVisible(false)
+            setIsLoadingVisible(false)
           }
 
           console.log({prediction})
@@ -167,8 +130,6 @@ export default function Studio(){
         }
         // Generate base64 url image for remove bg
 
-        
-
         try {
 
             const response = await fetch('http://127.0.0.1:5000/get_item_mask', {
@@ -176,26 +137,11 @@ export default function Studio(){
               body: formData
             });
     
-            // Handle response
-            // const imageBlob = await response.blob();
-            // const url = URL.createObjectURL(imageBlob);
-            // setMaskImageUrl(url)
             console.log(response)
-
             const data = await response.json();
-            //console.log(data.image)
-
             let maskBase64Url = `data:image/jpeg;base64,${data.ai_mask}`
-            let uxMaskBase64Url = `data:image/jpeg;base64,${data.ux_mask}`
 
-            setMaskImageUrl(maskBase64Url)
-            setUxMaskImageUrl(uxMaskBase64Url)
-
-            setIsMaskVisible(true)
-            setIsImgUploadVisible(true)
             setIsResultVisible(false)
-
-
             let imageURLTemp = "https://images.unsplash.com/photo-1490730141103-6cac27aaab94?ixlib=rb-4.0.3&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2940&q=80"
             
             // Generate base64 image for input image.
@@ -209,15 +155,9 @@ export default function Studio(){
                 await getReplicateResults(imageBase64Url ,maskBase64Url)
 
             };
-            
-        
         } catch (error) {
             console.error(error);
         }
-    
-        // const img = document.createElement('img');
-        // img.src = url;
-        // document.body.appendChild(img);
     }
 
     function base64ToBlob(base64Image) {
@@ -235,19 +175,10 @@ export default function Studio(){
     }
 
     function capture3DSnapshot() {
-
         const dataUrl = canvasRef.toDataURL("image/png")
-        setCanvasSnapshotUrl(dataUrl)
-
         const blob = base64ToBlob(dataUrl);
-        setImageFile(blob)
-        setIsMaskVisible(false)
-        setIsImgUploadVisible(true)
         setIsResultVisible(false)
-
-
         return blob
-         
     }
 
     //download snapshot
@@ -264,36 +195,7 @@ export default function Studio(){
 
     return(
         <div className={styles.page}>
-
-            {/* threejs container  */}
-
             <div className={styles.inputPanel}>
-                {/* <div className={styles.mainImageContainer}>
-                    {isImgUploadVisible? (
-                        <div
-                            className={styles.imageContainer}
-                            onDrop={handleDrop}
-                            onDragOver={(event) => event.preventDefault()}
-                        >
-                            {imageFile? (
-                                <div>
-                                    <Image
-                                        src={URL.createObjectURL(imageFile)}
-                                        alt="Uploaded image"
-                                        fill={true}
-                                    />
-                                </div>
-                            ) : (
-                                <div className={styles.dragAndDropText}>Drag and Drop your image here</div>
-                            )}
-
-                        </div>
-                    ): (<></>)
-                    }
-                    <CanvasComponent setCanvasRef={setCanvasRef} />
-
-                </div> */}
-
                 <div className={styles.mainImageContainer}>
                     <div
                         className={styles.imageContainer}
@@ -307,213 +209,63 @@ export default function Studio(){
                 <div><TextInput onTextChange={handleInputValueChange} /></div>
 
                 <div className={styles.buttonContainer}>
-                    {/* <div 
-                        className={styles.startNewButton}
-                        onClick={()=>setImageFile(null)}
-                    >
-                            Start New
-                    </div> */}
-
                     <div 
                         className={styles.generateButton}
                         onClick={()=>generateImages()}
                     >
                             Generate Images
                     </div>
-
-
                 </div>
-
-
             </div>
 
-
             <div className={styles.resultsPanel}>
-
+                {isResultVisible? (
-                    {/* {isImgUploadVisible? (
+                    <>
-                        <div
+                    <div>
-                            className={styles.imageContainer}
+                    {prediction.output && (
-                            onDrop={handleDrop}
+                        <div className={styles.imageResultsContainer}>
-                            onDragOver={(event) => event.preventDefault()}
+                            <div className={styles.imageWrapper}>
-                        >
-                            {imageFile? (
-                                <div>
-                                    <Image
-                                        src={URL.createObjectURL(imageFile)}
-                                        alt="Uploaded image"
-                                        fill={true}
-                                    />
-                                </div>
-                            ) : (
-                                <div className={styles.dragAndDropText}>Drag and Drop your image here. Or click 3D</div>
-                            )}
-
-
-                            {canvasSnapshotUrl? (
-                                <div>
-                                    <Image
-                                        src={canvasSnapshotUrl}
-                                        alt="Uploaded image"
-                                        fill={true}
-                                    />
-                                </div>
-                            ) : (
-                                <></>
-                            )
-                            }
-
-                        </div>
-                    ): (<></>)
-                    } */}
-
-                    {/* {canvasSnapshotUrl? (
-                        <div 
-                            className={`${styles.maskImageContainer} ${canvasSnapshotUrl ? styles.slideDown : ""}`}
-                        >    
-                            {canvasSnapshotUrl? (
                                 <Image
-                                    src={canvasSnapshotUrl? canvasSnapshotUrl : ""}
+                                fill
-                                    alt="output"
+                                src={prediction.output[0]}
-                                    fill="true"
-                                    hidden={!canvasSnapshotUrl}
-                                />
-                            ):(
-                                <div> </div>
-                            )
-                            }
-                        </div>
-                    ) : (
-                        <></>
-                    )} */}
-
-                    {/* <div 
-                        className={`${styles.maskImageContainer} ${isMaskVisible ? styles.slideDown : ""}`}
-                    >    
-                        {uxMaskImageUrl? (
-                            <Image
-                                src={uxMaskImageUrl? uxMaskImageUrl : ""}
                                 alt="output"
-                                width="400"
+                                />
-                                height="400"
+                            </div>
-                                objectFit="contain"
+                            <div className={styles.imageWrapper}>
-                                hidden={!uxMaskImageUrl}
+                                <Image
-                            />
+                                fill
-                        ):(
+                                src={prediction.output[1]}
-                            <div> </div>
+                                alt="output"
-                        )
+                                />
-                        }
+                            </div>
-                    </div> */}
+                            <div className={styles.imageWrapper}>
-
+                                <Image
-
+                                fill
-                        {/* {isFlashingProgressVisible? (
+                                src={prediction.output[2]}
-                        <div 
+                                alt="output"
-                            className={styles.flashingProgressContainer}
+                                />
-                        >    
+                            </div>
-
+                            <div className={styles.imageWrapper}>
-                        </div>
+                                <Image
-
+                                fill
-                        ):(<></>)} */}
+                                src={prediction.output[3]}
-
+                                alt="output"
-                    {isResultVisible? (
+                                />
-                        <>
-
-                        <div>
-                        {prediction.output && (
-                            <div className={styles.imageResultsContainer}>
-                                <div className={styles.imageWrapper}>
-                                    <Image
-                                    fill
-                                    src={prediction.output[0]}
-                                    alt="output"
-                                    />
-                                </div>
-                                <div className={styles.imageWrapper}>
-                                    <Image
-                                    fill
-                                    src={prediction.output[1]}
-                                    alt="output"
-                                    />
-                                </div>
-                                <div className={styles.imageWrapper}>
-                                    <Image
-                                    fill
-                                    src={prediction.output[2]}
-                                    alt="output"
-                                    />
-                                </div>
-                                <div className={styles.imageWrapper}>
-                                    <Image
-                                    fill
-                                    src={prediction.output[3]}
-                                    alt="output"
-                                    />
-                                </div>
                             </div>
-                        )}  
-
-                        </div>
-                        
-
-                    </>
-                    ) : (
-                        <></>
-                    )}
-                    {
-                        isFlashingProgressVisible?
-                        (<p>Loading...</p>):(<></>)
-                    }
-
-
-
-
-            </div>
-
-            {/* <div className={styles.contentPanel}>
-                    
-            {error && <div>{error}</div>}
-
-            {prediction && (
-            <div>
-                {prediction.output && (
-                    <div className={styles.imageResultsContainer}>
-                        <div className={styles.imageWrapper}>
-                            <Image
-                            fill
-                            src={prediction.output[0]}
-                            alt="output"
-                            />
-                        </div>
-                        <div className={styles.imageWrapper}>
-                            <Image
-                            fill
-                            src={prediction.output[1]}
-                            alt="output"
-                            />
-                        </div>
-                        <div className={styles.imageWrapper}>
-                            <Image
-                            fill
-                            src={prediction.output[2]}
-                            alt="output"
-                            />
-                        </div>
-                        <div className={styles.imageWrapper}>
-                            <Image
-                            fill
-                            src={prediction.output[3]}
-                            alt="output"
-                            />
                         </div>
+                    )}  
                     </div>
-                )}  
+                    
-                <p>status: {prediction.status}</p>
+                </>
+                ) : (
+                    <></>
+                )}
+                {
+                    isLoadingVisible?
+                    (<p>Loading...</p>):(<></>)
+                }
+
             </div>
-            )}
-
-
-
-            </div> */}
-
         </div>
     );