polishing up

This commit is contained in:
neil 2023-10-12 16:47:21 -04:00
parent 35eb117c4e
commit b2fa49a6c3
9 changed files with 90 additions and 409 deletions

1
.gitignore vendored
View File

@ -32,3 +32,4 @@ yarn-error.log*
.vercel
/image_proc_server/__pycache__
image_proc_server/imgproc/

View File

@ -13,7 +13,7 @@ function TextInput({ onTextChange }) {
return (
<div className={styles.textInputContainer}>
<textarea id="text-input" wrap="soft" type="text" value={text} onChange={handleTextChange} className={styles.textInput} placeholder='Describe your scene here...'/>
<textarea id="text-input" wrap="soft" type="text" value={text} onChange={handleTextChange} className={styles.textInput} placeholder='Describe where the item should be placed. e.g. "in the grass", or "on the moon"'/>
</div>
);
}

View File

@ -9,9 +9,6 @@ from flask_cors import CORS
app = Flask(__name__)
CORS(app)
# authorization for removebg
removebg_api_key = "YOUR_API_KEY"
@app.route('/get_item_mask', methods=['POST'])
def get_item_mask():
@ -22,128 +19,45 @@ def get_item_mask():
image_bytes = image_file.read()
image = Image.open(io.BytesIO(image_bytes))
image = image.convert('RGB')
image = image.convert('RGBA')
image.save('input.png')
jpeg_buffer = io.BytesIO()
image.save(jpeg_buffer, format='JPEG')
#print(jpeg_buffer.getvalue())
ai_mask_image = convert_to_black_and_white(image)
# set API endpoint and headers
api_url = "https://api.remove.bg/v1.0/removebg"
headers = {"X-Api-Key": removebg_api_key}
ai_mask_image.save("output.png")
ai_mask_image = ai_mask_image.convert('RGB')
ai_mask_image.save(jpeg_buffer, format="JPEG")
ai_mask_image_base64 = base64.b64encode(jpeg_buffer.getvalue()).decode('utf-8')
# send the request to remove.bg API
response = requests.post(api_url, headers=headers, files={'image_file': jpeg_buffer.getvalue()})
# Check if the API returned a successful response
if response.status_code == requests.codes.ok:
# print("response is ok")
# print("Success:", response.status_code, response.text)
# Save the PNG image to a file
with open('output.png', 'wb') as out_file:
out_file.write(response.content)
png_buffer = io.BytesIO()
ux_mask_buffer = io.BytesIO()
response_image = Image.open(io.BytesIO(response.content))
response_image = response_image.convert('RGBA')
ai_mask_image = convert_to_black_and_white(response_image)
ai_mask_image = ai_mask_image.convert('RGB')
ai_mask_image.save(png_buffer, format="JPEG")
ai_mask_image_base64 = base64.b64encode(png_buffer.getvalue()).decode('utf-8')
ux_mask_image = convert_to_green(response_image)
ux_mask_image = ux_mask_image.convert('RGBA')
ux_mask_image.save(ux_mask_buffer, format="PNG")
ux_mask_image.save('greenoutput.png')
ux_mask_image_base64 = base64.b64encode(ux_mask_buffer.getvalue()).decode('utf-8')
return {'ai_mask': ai_mask_image_base64, 'ux_mask': ux_mask_image_base64}
else:
# Print an error message if the API call was not successful
print("Error:", response.status_code, response.text)
return {'error': "There was a problem with the remove bg operation"}
return {'ai_mask': ai_mask_image_base64}
def convert_to_black_and_white(image):
# Create a new image with the same size as the original, but with a white background
new_image = Image.new('RGB', image.size, (255, 255, 255))
# Create a mask for the alpha channel
alpha_mask = image.split()[-1]
# Ensure the image has an alpha channel
if image.mode != 'RGBA':
image = image.convert('RGBA')
modified_alpha_mask = modify_alpha_mask(alpha_mask)
# Get pixel data
data = image.getdata()
# Paste the original image onto the new image using the alpha channel as the mask
new_image.paste(image, mask=modified_alpha_mask)
# Create new image data
new_data = []
for item in data:
# If the pixel is fully transparent (alpha value of 0), set it to white. Else, set it to black.
if item[3] == 0:
new_data.append((255, 255, 255, 255))
else:
new_data.append((0, 0, 0, 255))
new_image.save('temp_mask_image.png')
# Update new image data
new_image.putdata(new_data)
# Convert the new image to grayscale
new_image = new_image.convert('L')
new_image.save('temp_mask_image_grayscale.png')
# Convert all non-alpha pixels to black and all alpha pixels to white
black_and_white_image = new_image.point(lambda x: 255 if x == 255 else 0, '1')
# Convert the final image to grayscale for black and white effect
black_and_white_image = new_image.convert('L')
return black_and_white_image
def modify_alpha_mask(alpha_mask):
# Ensure the input image has an alpha channel
if alpha_mask.mode != 'L':
alpha_mask = alpha_mask.convert('L')
# Create a new image with modified alpha values
modified_alpha_mask = Image.new('L', alpha_mask.size)
# Iterate over each pixel and modify the alpha value
for x in range(alpha_mask.width):
for y in range(alpha_mask.height):
alpha_value = alpha_mask.getpixel((x, y))
# Check if the alpha value is semi-transparent (not fully transparent or opaque)
if alpha_value > 0 and alpha_value < 100:
alpha_value = 0 # Set to fully transparent
# Set the modified alpha value in the new image
modified_alpha_mask.putpixel((x, y), alpha_value)
return modified_alpha_mask
def convert_to_green(image):
# Create a new image with the same size as the original, but with a transparent background
new_image = Image.new('RGBA', image.size, (0, 0, 0, 0))
# Create a mask for the alpha channel
alpha_mask = image.split()[-1]
# Paste the original image onto the new image using the alpha channel as the mask
new_image.paste(image, mask=alpha_mask)
# Create a new image with the same size as the original, but with 0.5 alpha green color
green_image = Image.new('RGBA', image.size, (255, 200, 0, 180))
# Create a mask for the alpha channel
alpha_mask = new_image.split()[-1]
alpha_mask = ImageOps.invert(alpha_mask)
# Paste the green image onto the new image using the alpha channel as the mask
new_image.paste(green_image, mask=alpha_mask)
return new_image

Binary file not shown.

Before

Width:  |  Height:  |  Size: 68 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 KiB

View File

@ -0,0 +1,14 @@
blinker==1.6.3
certifi==2023.7.22
charset-normalizer==3.3.0
click==8.1.7
Flask==3.0.0
Flask-Cors==4.0.0
idna==3.4
itsdangerous==2.1.2
Jinja2==3.1.2
MarkupSafe==2.1.3
Pillow==10.0.1
requests==2.31.0
urllib3==2.0.6
Werkzeug==3.0.0

Binary file not shown.

Before

Width:  |  Height:  |  Size: 57 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

View File

@ -11,44 +11,15 @@ import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader';
import {CanvasComponent} from "@/components/canvascomponent";
/*
links for thereejs implementation
https://codesandbox.io/s/basic-demo-forked-rnuve?file=/src/App.js
*/
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
export default function Studio(){
// const imageContainer = {
// position: 'absolute',
// top: '0',
// left: '0',
// backgroundColor: 'rgb(70, 232, 83)',
// height: '100%',
// width: '100%',
// };
const [canvasSnapshotUrl, setCanvasSnapshotUrl] = useState(null)
const [maskImageUrl, setMaskImageUrl] = useState(null);
const [uxMaskImageUrl, setUxMaskImageUrl] = useState(null);
const [imageFile, setImageFile] = useState(null)
const [modelFile, setModelFile] = useState(null);
const [modelBuffer, setModelBuffer] = useState(null);
const [gltfModel, setGltfModel] = useState(null);
const [isImgUploadVisible, setIsImgUploadVisible] = useState(null)
const [isMaskVisible, setIsMaskVisible] = useState(null)
const [isResultVisible, setIsResultVisible] = useState(null)
const [isFlashingProgressVisible, setIsFlashingProgressVisible] = useState(null)
const [isLoadingVisible, setIsLoadingVisible] = useState(null)
const [prediction, setPrediction] = useState(null);
const [error, setError] = useState(null);
const [inputValue, setInputValue] = useState('');
// create a canvas reference in the main state
@ -60,12 +31,8 @@ export default function Studio(){
useEffect(() => {
console.log("loaded the page");
// define visibiltiy of the 3 layers
setIsImgUploadVisible(true)
setIsMaskVisible(true)
setIsResultVisible(false)
setIsFlashingProgressVisible(false)
setIsLoadingVisible(false)
},[]);
@ -97,7 +64,7 @@ export default function Studio(){
const getReplicateResults = async (image, mask) => {
setIsFlashingProgressVisible(true)
setIsLoadingVisible(true)
let promptText = "beautiful living room"
@ -111,10 +78,7 @@ export default function Studio(){
"Content-Type": "application/json",
},
body: JSON.stringify({
//prompt: e.target.prompt.value,
//prompt: "high resolution photography of a beige interior living room with dining chairs, around dining table, wooden floor, beige blue salmon pastel, sunlight, contrast, realistic artstation concept art, hyperdetailed, ultradetail, cinematic 8k, architectural rendering, unreal engine 5, rtx, volumetric light, cozy atmosphere",
//prompt: "minimalist kitchen, wooden floor, beige blue salmon pastel, sunlight, contrast, realistic artstation concept art, hyperdetailed, ultradetail, cinematic 8k, architectural rendering, unreal engine 5, rtx, volumetric light, cozy atmosphere",
prompt: promptText + ", creative marketing advertisement",
prompt: promptText + ", photorealistic, high resolution product photography",
negative_prompt: "blurry, painting, cartoon, abstract, ugly, deformed",
image: image,
mask: mask,
@ -142,10 +106,9 @@ export default function Studio(){
}
if (prediction.status == "succeeded" && prediction.output) {
setIsImgUploadVisible(true)
setIsMaskVisible(true)
setIsResultVisible(true)
setIsFlashingProgressVisible(false)
setIsLoadingVisible(false)
}
console.log({prediction})
@ -167,8 +130,6 @@ export default function Studio(){
}
// Generate base64 url image for remove bg
try {
const response = await fetch('http://127.0.0.1:5000/get_item_mask', {
@ -176,26 +137,11 @@ export default function Studio(){
body: formData
});
// Handle response
// const imageBlob = await response.blob();
// const url = URL.createObjectURL(imageBlob);
// setMaskImageUrl(url)
console.log(response)
const data = await response.json();
//console.log(data.image)
let maskBase64Url = `data:image/jpeg;base64,${data.ai_mask}`
let uxMaskBase64Url = `data:image/jpeg;base64,${data.ux_mask}`
setMaskImageUrl(maskBase64Url)
setUxMaskImageUrl(uxMaskBase64Url)
setIsMaskVisible(true)
setIsImgUploadVisible(true)
setIsResultVisible(false)
let imageURLTemp = "https://images.unsplash.com/photo-1490730141103-6cac27aaab94?ixlib=rb-4.0.3&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2940&q=80"
// Generate base64 image for input image.
@ -209,15 +155,9 @@ export default function Studio(){
await getReplicateResults(imageBase64Url ,maskBase64Url)
};
} catch (error) {
console.error(error);
}
// const img = document.createElement('img');
// img.src = url;
// document.body.appendChild(img);
}
function base64ToBlob(base64Image) {
@ -235,19 +175,10 @@ export default function Studio(){
}
function capture3DSnapshot() {
const dataUrl = canvasRef.toDataURL("image/png")
setCanvasSnapshotUrl(dataUrl)
const blob = base64ToBlob(dataUrl);
setImageFile(blob)
setIsMaskVisible(false)
setIsImgUploadVisible(true)
setIsResultVisible(false)
return blob
}
//download snapshot
@ -264,36 +195,7 @@ export default function Studio(){
return(
<div className={styles.page}>
{/* threejs container */}
<div className={styles.inputPanel}>
{/* <div className={styles.mainImageContainer}>
{isImgUploadVisible? (
<div
className={styles.imageContainer}
onDrop={handleDrop}
onDragOver={(event) => event.preventDefault()}
>
{imageFile? (
<div>
<Image
src={URL.createObjectURL(imageFile)}
alt="Uploaded image"
fill={true}
/>
</div>
) : (
<div className={styles.dragAndDropText}>Drag and Drop your image here</div>
)}
</div>
): (<></>)
}
<CanvasComponent setCanvasRef={setCanvasRef} />
</div> */}
<div className={styles.mainImageContainer}>
<div
className={styles.imageContainer}
@ -307,213 +209,63 @@ export default function Studio(){
<div><TextInput onTextChange={handleInputValueChange} /></div>
<div className={styles.buttonContainer}>
{/* <div
className={styles.startNewButton}
onClick={()=>setImageFile(null)}
>
Start New
</div> */}
<div
className={styles.generateButton}
onClick={()=>generateImages()}
>
Generate Images
</div>
</div>
</div>
<div className={styles.resultsPanel}>
{/* {isImgUploadVisible? (
<div
className={styles.imageContainer}
onDrop={handleDrop}
onDragOver={(event) => event.preventDefault()}
>
{imageFile? (
<div>
<Image
src={URL.createObjectURL(imageFile)}
alt="Uploaded image"
fill={true}
/>
</div>
) : (
<div className={styles.dragAndDropText}>Drag and Drop your image here. Or click 3D</div>
)}
{canvasSnapshotUrl? (
<div>
<Image
src={canvasSnapshotUrl}
alt="Uploaded image"
fill={true}
/>
</div>
) : (
<></>
)
}
</div>
): (<></>)
} */}
{/* {canvasSnapshotUrl? (
<div
className={`${styles.maskImageContainer} ${canvasSnapshotUrl ? styles.slideDown : ""}`}
>
{canvasSnapshotUrl? (
{isResultVisible? (
<>
<div>
{prediction.output && (
<div className={styles.imageResultsContainer}>
<div className={styles.imageWrapper}>
<Image
src={canvasSnapshotUrl? canvasSnapshotUrl : ""}
alt="output"
fill="true"
hidden={!canvasSnapshotUrl}
/>
):(
<div> </div>
)
}
</div>
) : (
<></>
)} */}
{/* <div
className={`${styles.maskImageContainer} ${isMaskVisible ? styles.slideDown : ""}`}
>
{uxMaskImageUrl? (
<Image
src={uxMaskImageUrl? uxMaskImageUrl : ""}
fill
src={prediction.output[0]}
alt="output"
width="400"
height="400"
objectFit="contain"
hidden={!uxMaskImageUrl}
/>
):(
<div> </div>
)
}
</div> */}
{/* {isFlashingProgressVisible? (
<div
className={styles.flashingProgressContainer}
>
</div>
):(<></>)} */}
{isResultVisible? (
<>
<div>
{prediction.output && (
<div className={styles.imageResultsContainer}>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[0]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[1]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[2]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[3]}
alt="output"
/>
</div>
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[1]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[2]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[3]}
alt="output"
/>
</div>
)}
</div>
</>
) : (
<></>
)}
{
isFlashingProgressVisible?
(<p>Loading...</p>):(<></>)
}
</div>
{/* <div className={styles.contentPanel}>
{error && <div>{error}</div>}
{prediction && (
<div>
{prediction.output && (
<div className={styles.imageResultsContainer}>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[0]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[1]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[2]}
alt="output"
/>
</div>
<div className={styles.imageWrapper}>
<Image
fill
src={prediction.output[3]}
alt="output"
/>
</div>
</div>
</>
) : (
<></>
)}
<p>status: {prediction.status}</p>
{
isLoadingVisible?
(<p>Loading...</p>):(<></>)
}
</div>
)}
</div> */}
</div>
);