import { useState, useEffect, useRef } from "react"; import Head from "next/head"; import Image from "next/image"; import styles from "../styles/studio.module.css"; import FormData from 'form-data'; import Dropzone from 'react-dropzone'; import { userAgentFromString } from "next/server"; import TextInput from "@/components/textinput"; import { Canvas, useFrame, useThree } from '@react-three/fiber' import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader'; import {CanvasComponent} from "@/components/canvascomponent"; /* links for thereejs implementation https://codesandbox.io/s/basic-demo-forked-rnuve?file=/src/App.js */ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); export default function Studio(){ // const imageContainer = { // position: 'absolute', // top: '0', // left: '0', // backgroundColor: 'rgb(70, 232, 83)', // height: '100%', // width: '100%', // }; const [canvasSnapshotUrl, setCanvasSnapshotUrl] = useState(null) const [maskImageUrl, setMaskImageUrl] = useState(null); const [uxMaskImageUrl, setUxMaskImageUrl] = useState(null); const [imageFile, setImageFile] = useState(null) const [modelFile, setModelFile] = useState(null); const [modelBuffer, setModelBuffer] = useState(null); const [gltfModel, setGltfModel] = useState(null); const [isImgUploadVisible, setIsImgUploadVisible] = useState(null) const [isMaskVisible, setIsMaskVisible] = useState(null) const [isResultVisible, setIsResultVisible] = useState(null) const [isFlashingProgressVisible, setIsFlashingProgressVisible] = useState(null) const [prediction, setPrediction] = useState(null); const [error, setError] = useState(null); const [inputValue, setInputValue] = useState(''); // create a canvas reference in the main state const [canvasRef, setCanvasRef] = useState(null); function handleInputValueChange(newInputValue) { setInputValue(newInputValue); } useEffect(() => { console.log("loaded the page"); // define visibiltiy of the 3 layers setIsImgUploadVisible(true) setIsMaskVisible(true) setIsResultVisible(false) setIsFlashingProgressVisible(false) },[]); const handleDrop = (event) => { event.preventDefault(); const file = event.dataTransfer.files[0]; if (file && file.name.endsWith('.glb')) { const reader = new FileReader(); reader.readAsArrayBuffer(file); reader.onload = (event) => { const arrayBuffer = event.target.result; const loader = new GLTFLoader(); loader.parse(arrayBuffer, '', (gltf) => { setGltfModel(gltf); console.log("gltf model loaded") console.log('Loaded Scene:', gltf.scene); }, (error) => { console.error('ArrayBuffer loading error:', error); }); }; } }; const getReplicateResults = async (image, mask) => { setIsFlashingProgressVisible(true) let promptText = "beautiful living room" if (inputValue) { promptText = inputValue } const response = await fetch("/api/predictions", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ //prompt: e.target.prompt.value, //prompt: "high resolution photography of a beige interior living room with dining chairs, around dining table, wooden floor, beige blue salmon pastel, sunlight, contrast, realistic artstation concept art, hyperdetailed, ultradetail, cinematic 8k, architectural rendering, unreal engine 5, rtx, volumetric light, cozy atmosphere", //prompt: "minimalist kitchen, wooden floor, beige blue salmon pastel, sunlight, contrast, realistic artstation concept art, hyperdetailed, ultradetail, cinematic 8k, architectural rendering, unreal engine 5, rtx, volumetric light, cozy atmosphere", prompt: promptText + ", creative marketing advertisement", negative_prompt: "blurry, painting, cartoon, abstract, ugly, deformed", image: image, mask: mask, num_outputs: 4, guidance_scale: 7.5, }), }); let prediction = await response.json(); if (response.status !== 201) { setError(prediction.detail); return; } setPrediction(prediction); while ( prediction.status !== "succeeded" && prediction.status !== "failed" ) { await sleep(1000); const response = await fetch("/api/predictions/" + prediction.id); prediction = await response.json(); if (response.status !== 200) { setError(prediction.detail); return; } if (prediction.status == "succeeded" && prediction.output) { setIsImgUploadVisible(true) setIsMaskVisible(true) setIsResultVisible(true) setIsFlashingProgressVisible(false) } console.log({prediction}) setPrediction(prediction); } }; async function generateImages() { console.log("Called the generate images function") // Do something with the image data URL let snapshotImage = capture3DSnapshot() const formData = new FormData(); formData.append('image', snapshotImage); if (!snapshotImage) { console.log("image file is null") } // Generate base64 url image for remove bg try { const response = await fetch('http://127.0.0.1:5000/get_item_mask', { method: 'POST', body: formData }); // Handle response // const imageBlob = await response.blob(); // const url = URL.createObjectURL(imageBlob); // setMaskImageUrl(url) console.log(response) const data = await response.json(); //console.log(data.image) let maskBase64Url = `data:image/jpeg;base64,${data.ai_mask}` let uxMaskBase64Url = `data:image/jpeg;base64,${data.ux_mask}` setMaskImageUrl(maskBase64Url) setUxMaskImageUrl(uxMaskBase64Url) setIsMaskVisible(true) setIsImgUploadVisible(true) setIsResultVisible(false) let imageURLTemp = "https://images.unsplash.com/photo-1490730141103-6cac27aaab94?ixlib=rb-4.0.3&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2940&q=80" // Generate base64 image for input image. // Filereader converts a file blob into a base64 string const reader = new FileReader(); reader.readAsDataURL(snapshotImage); reader.onload = async () => { const imageBase64Url = reader.result; // now send a request to replicate await getReplicateResults(imageBase64Url ,maskBase64Url) }; } catch (error) { console.error(error); } // const img = document.createElement('img'); // img.src = url; // document.body.appendChild(img); } function base64ToBlob(base64Image) { const parts = base64Image.split(';base64,'); const mimeType = parts[0].split(':')[1]; const byteString = atob(parts[1]); const arrayBuffer = new ArrayBuffer(byteString.length); const uint8Array = new Uint8Array(arrayBuffer); for (let i = 0; i < byteString.length; i++) { uint8Array[i] = byteString.charCodeAt(i); } return new Blob([arrayBuffer], { type: mimeType }); } function capture3DSnapshot() { const dataUrl = canvasRef.toDataURL("image/png") setCanvasSnapshotUrl(dataUrl) const blob = base64ToBlob(dataUrl); setImageFile(blob) setIsMaskVisible(false) setIsImgUploadVisible(true) setIsResultVisible(false) return blob } //download snapshot const download3DSnapshot = () => { const link = document.createElement("a"); link.setAttribute("download", "canvas.png"); link.setAttribute( "href", canvasRef.toDataURL("image/png").replace("image/png", "image/octet-stream") ); link.click(); }; return(
Loading...
):(<>>) }status: {prediction.status}