Update-video format api for reference images
This commit is contained in:
Manish Tanwar
parent
0017cc323d
commit
3602cfd73b
4 changed files with 115 additions and 50 deletions
|
|
@ -26,7 +26,9 @@
|
|||
"Bash(find:*)",
|
||||
"Bash(./setup.sh)",
|
||||
"Bash(bash:*)",
|
||||
"Bash(dos2unix:*)"
|
||||
"Bash(dos2unix:*)",
|
||||
"WebSearch",
|
||||
"WebFetch(domain:ai.google.dev)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@ class VeoVideoAPI {
|
|||
'prompt' => $prompt
|
||||
];
|
||||
|
||||
// Build parameters first
|
||||
$parameters = [
|
||||
'aspectRatio' => $aspectRatio,
|
||||
'sampleCount' => 1
|
||||
];
|
||||
|
||||
// Frame mode: first image is starting frame, optional second image is last frame for interpolation
|
||||
if (!empty($referenceImages)) {
|
||||
if (isset($referenceImages[0])) {
|
||||
|
|
@ -60,6 +66,7 @@ class VeoVideoAPI {
|
|||
$data = preg_replace('/\s+/', '', $refImg['data']);
|
||||
|
||||
if (preg_match('/^[A-Za-z0-9+\/]+={0,2}$/', $data)) {
|
||||
// Use bytesBase64Encoded format (original working format)
|
||||
$instance['image'] = [
|
||||
'bytesBase64Encoded' => $data,
|
||||
'mimeType' => $refImg['mime_type'] ?? 'image/jpeg'
|
||||
|
|
@ -68,27 +75,27 @@ class VeoVideoAPI {
|
|||
}
|
||||
}
|
||||
|
||||
// Add last frame for interpolation (when 2 images provided)
|
||||
if (count($referenceImages) >= 2 && isset($referenceImages[1])) {
|
||||
// Add last frame for interpolation (when 2 images provided AND duration is 8 seconds)
|
||||
// IMPORTANT: lastFrame feature may not be available in Gemini API yet
|
||||
// Keeping this code for when feature becomes available
|
||||
if (count($referenceImages) >= 2 && isset($referenceImages[1]) && intval($duration) === 8) {
|
||||
$lastImg = $referenceImages[1];
|
||||
$lastData = preg_replace('/\s+/', '', $lastImg['data']);
|
||||
|
||||
if (preg_match('/^[A-Za-z0-9+\/]+={0,2}$/', $lastData)) {
|
||||
$instance['lastFrame'] = [
|
||||
// Try bytesBase64Encoded format
|
||||
$parameters['lastFrame'] = [
|
||||
'bytesBase64Encoded' => $lastData,
|
||||
'mimeType' => $lastImg['mime_type'] ?? 'image/jpeg'
|
||||
];
|
||||
error_log("Added last frame for video interpolation");
|
||||
error_log("Added last frame for video interpolation (8s duration, standard model)");
|
||||
error_log("Using model: " . $this->model);
|
||||
}
|
||||
} elseif (count($referenceImages) >= 2 && intval($duration) !== 8) {
|
||||
error_log("WARNING: Second reference image provided but duration is not 8s - ignoring lastFrame");
|
||||
}
|
||||
}
|
||||
|
||||
// Build parameters
|
||||
$parameters = [
|
||||
'aspectRatio' => $aspectRatio,
|
||||
'sampleCount' => 1
|
||||
];
|
||||
|
||||
// Duration: Veo 3.1 supports 4, 6, or 8 seconds
|
||||
if (in_array(intval($duration), [4, 6, 8])) {
|
||||
$parameters['durationSeconds'] = intval($duration);
|
||||
|
|
@ -107,17 +114,20 @@ class VeoVideoAPI {
|
|||
|
||||
// Log payload structure (without full base64 data for readability)
|
||||
$logPayload = $payload;
|
||||
if (isset($logPayload['instances'][0]['image'])) {
|
||||
if (isset($logPayload['instances'][0]['image']['bytesBase64Encoded'])) {
|
||||
$logPayload['instances'][0]['image']['bytesBase64Encoded'] = '[BASE64_DATA_' . strlen($payload['instances'][0]['image']['bytesBase64Encoded']) . '_bytes]';
|
||||
}
|
||||
if (isset($logPayload['instances'][0]['lastFrame'])) {
|
||||
$logPayload['instances'][0]['lastFrame']['bytesBase64Encoded'] = '[BASE64_DATA_' . strlen($payload['instances'][0]['lastFrame']['bytesBase64Encoded']) . '_bytes]';
|
||||
if (isset($logPayload['parameters']['lastFrame']['bytesBase64Encoded'])) {
|
||||
$logPayload['parameters']['lastFrame']['bytesBase64Encoded'] = '[BASE64_DATA_' . strlen($payload['parameters']['lastFrame']['bytesBase64Encoded']) . '_bytes]';
|
||||
}
|
||||
if (isset($logPayload['instances'][0]['referenceImages'])) {
|
||||
foreach ($logPayload['instances'][0]['referenceImages'] as $i => &$refImg) {
|
||||
$refImg['image']['bytesBase64Encoded'] = '[BASE64_DATA_' . strlen($payload['instances'][0]['referenceImages'][$i]['image']['bytesBase64Encoded']) . '_bytes]';
|
||||
if (isset($refImg['image']['bytesBase64Encoded'])) {
|
||||
$refImg['image']['bytesBase64Encoded'] = '[BASE64_DATA_' . strlen($payload['instances'][0]['referenceImages'][$i]['image']['bytesBase64Encoded']) . '_bytes]';
|
||||
}
|
||||
}
|
||||
}
|
||||
error_log("Video generation request - Model: " . $this->model);
|
||||
error_log("Video generation payload structure: " . json_encode($logPayload));
|
||||
|
||||
return $this->makeRequest($payload);
|
||||
|
|
@ -177,7 +187,12 @@ class VeoVideoAPI {
|
|||
}
|
||||
|
||||
if ($errorStatus === 'INVALID_ARGUMENT') {
|
||||
throw new Exception("Invalid request format. Check your prompt and settings.");
|
||||
// Check if error is about lastFrame feature
|
||||
if (stripos($errorMessage, 'lastFrame') !== false) {
|
||||
throw new Exception("The lastFrame feature may not be available for your API key yet. Google is still rolling out Veo 3.1 features. Try using single image-to-video instead, or check Google AI Studio for feature availability. Error: " . $errorMessage);
|
||||
}
|
||||
// Include the actual error message from Google API for debugging
|
||||
throw new Exception("Invalid request format: " . $errorMessage);
|
||||
}
|
||||
|
||||
if (stripos($errorMessage, 'model') !== false && stripos($errorMessage, 'not found') !== false) {
|
||||
|
|
@ -372,6 +387,14 @@ try {
|
|||
$modelType = 'standard';
|
||||
}
|
||||
|
||||
// Force standard model when using lastFrame interpolation (2 reference images)
|
||||
// IMPORTANT: veo-3.1-fast-generate-preview does NOT support lastFrame feature
|
||||
$refCount = intval($_POST['referenceImageCount'] ?? 0);
|
||||
if ($refCount >= 2) {
|
||||
$modelType = 'standard';
|
||||
error_log("Forcing standard model for lastFrame interpolation (fast model doesn't support this feature)");
|
||||
}
|
||||
|
||||
$api = new VeoVideoAPI(GEMINI_API_KEY, $modelType);
|
||||
|
||||
// Handle generate action
|
||||
|
|
@ -557,11 +580,11 @@ try {
|
|||
$filepath = $videoDir . '/' . $filename;
|
||||
file_put_contents($filepath, $videoData);
|
||||
|
||||
// Return local URL
|
||||
// Return URL in /generated_videos/ format for frontend to handle
|
||||
echo json_encode([
|
||||
'success' => true,
|
||||
'video' => [
|
||||
'url' => ($_ENV['API_BASE_PATH'] ?? '') . '/stream_video.php?file=' . urlencode($filename),
|
||||
'url' => '/generated_videos/' . $filename,
|
||||
'filename' => $filename,
|
||||
'mime_type' => 'video/mp4'
|
||||
]
|
||||
|
|
|
|||
|
|
@ -145,6 +145,13 @@ const VideoGenTab = ({ activeProjectId, rerunData, onRerunLoaded }) => {
|
|||
setAspectRatio('16:9');
|
||||
}
|
||||
|
||||
// Auto-set duration to 8s and standard model when adding second image
|
||||
// (first+last frame interpolation requires 8s duration and standard model)
|
||||
if (referenceImages.length === 1) {
|
||||
setDuration(8);
|
||||
setModelType('standard');
|
||||
}
|
||||
|
||||
setReferenceImages(prev => [...prev, {
|
||||
data: item.data,
|
||||
mime_type: item.mimeType,
|
||||
|
|
@ -195,10 +202,20 @@ const VideoGenTab = ({ activeProjectId, rerunData, onRerunLoaded }) => {
|
|||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
setReferenceImages(prev => {
|
||||
const newLength = prev.length + 1;
|
||||
|
||||
// Auto-set aspect ratio when adding first reference image
|
||||
if (prev.length === 0) {
|
||||
setAspectRatio('16:9');
|
||||
}
|
||||
|
||||
// Auto-set duration to 8s and standard model when adding second image
|
||||
// (first+last frame interpolation requires 8s duration and standard model)
|
||||
if (newLength === 2) {
|
||||
setDuration(8);
|
||||
setModelType('standard');
|
||||
}
|
||||
|
||||
return [...prev, {
|
||||
data: reader.result.split(',')[1],
|
||||
mime_type: file.type,
|
||||
|
|
@ -266,12 +283,7 @@ const VideoGenTab = ({ activeProjectId, rerunData, onRerunLoaded }) => {
|
|||
let finalUrl = videoUrl;
|
||||
if (videoUrl.startsWith('/generated_videos/')) {
|
||||
const filename = videoUrl.replace('/generated_videos/', '');
|
||||
if (import.meta.env.DEV) {
|
||||
finalUrl = `/api/stream_video.php?file=${encodeURIComponent(filename)}`;
|
||||
} else {
|
||||
const apiUrl = import.meta.env.VITE_API_URL || 'http://localhost:5015';
|
||||
finalUrl = `${apiUrl}/stream_video.php?file=${encodeURIComponent(filename)}`;
|
||||
}
|
||||
finalUrl = getApiUrl(`stream_video.php?file=${encodeURIComponent(filename)}`);
|
||||
console.log(`Thumbnail extraction URL conversion: ${videoUrl} → ${finalUrl}`);
|
||||
}
|
||||
|
||||
|
|
@ -647,21 +659,35 @@ OUTPUT ONLY THE PROMPT - no explanations, no labels, no formatting.`;
|
|||
Model
|
||||
</label>
|
||||
<div className="flex gap-2">
|
||||
{modelOptions.map((opt) => (
|
||||
<button
|
||||
key={opt.value}
|
||||
onClick={() => setModelType(opt.value)}
|
||||
className={`flex-1 px-3 py-2 rounded-lg font-medium transition-all text-sm ${
|
||||
modelType === opt.value
|
||||
? 'bg-cinema-gold text-slate-950'
|
||||
: 'bg-slate-800 text-slate-400 hover:bg-slate-700'
|
||||
}`}
|
||||
>
|
||||
<div>{opt.label}</div>
|
||||
<div className="text-xs opacity-70">{opt.description}</div>
|
||||
</button>
|
||||
))}
|
||||
{modelOptions.map((opt) => {
|
||||
const isDisabledByInterpolation = referenceImages.length === 2 && opt.value === 'fast';
|
||||
return (
|
||||
<button
|
||||
key={opt.value}
|
||||
onClick={() => !isDisabledByInterpolation && setModelType(opt.value)}
|
||||
disabled={isDisabledByInterpolation}
|
||||
className={`flex-1 px-3 py-2 rounded-lg font-medium transition-all text-sm ${
|
||||
modelType === opt.value
|
||||
? 'bg-cinema-gold text-slate-950'
|
||||
: isDisabledByInterpolation
|
||||
? 'bg-slate-800/50 text-slate-600 cursor-not-allowed'
|
||||
: 'bg-slate-800 text-slate-400 hover:bg-slate-700'
|
||||
}`}
|
||||
>
|
||||
<div>{opt.label}</div>
|
||||
<div className="text-xs opacity-70">{opt.description}</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{referenceImages.length === 2 && (
|
||||
<div className="flex items-start gap-2 p-3 bg-blue-500/10 border border-blue-500/30 rounded-lg">
|
||||
<Info className="w-4 h-4 text-blue-400 mt-0.5 flex-shrink-0" />
|
||||
<p className="text-xs text-blue-300">
|
||||
First + Last frame interpolation requires Standard model (automatically set)
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Duration */}
|
||||
|
|
@ -671,14 +697,17 @@ OUTPUT ONLY THE PROMPT - no explanations, no labels, no formatting.`;
|
|||
</label>
|
||||
<div className="flex gap-2">
|
||||
{durationOptions.map((opt) => {
|
||||
const isDisabledByI2V = false;
|
||||
const isDisabledByInterpolation = referenceImages.length === 2 && opt.value !== 8;
|
||||
return (
|
||||
<button
|
||||
key={opt.value}
|
||||
onClick={() => setDuration(opt.value)}
|
||||
onClick={() => !isDisabledByInterpolation && setDuration(opt.value)}
|
||||
disabled={isDisabledByInterpolation}
|
||||
className={`flex-1 px-3 py-2 rounded-lg font-medium transition-all text-sm ${
|
||||
duration === opt.value
|
||||
? 'bg-cinema-gold text-slate-950'
|
||||
: isDisabledByInterpolation
|
||||
? 'bg-slate-800/50 text-slate-600 cursor-not-allowed'
|
||||
: 'bg-slate-800 text-slate-400 hover:bg-slate-700'
|
||||
}`}
|
||||
>
|
||||
|
|
@ -688,6 +717,14 @@ OUTPUT ONLY THE PROMPT - no explanations, no labels, no formatting.`;
|
|||
);
|
||||
})}
|
||||
</div>
|
||||
{referenceImages.length === 2 && (
|
||||
<div className="flex items-start gap-2 p-3 bg-blue-500/10 border border-blue-500/30 rounded-lg">
|
||||
<Info className="w-4 h-4 text-blue-400 mt-0.5 flex-shrink-0" />
|
||||
<p className="text-xs text-blue-300">
|
||||
First + Last frame interpolation requires 8-second duration (automatically set)
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Aspect Ratio */}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,17 @@ const VideoPlayer = ({ src, onFrameExtract, onSaveToProject, className = '', aut
|
|||
const videoRef = useRef(null);
|
||||
const canvasRef = useRef(null);
|
||||
|
||||
// API URL helper - uses Vite proxy in dev, direct URL in production
|
||||
const getApiUrl = (endpoint) => {
|
||||
// In development, use Vite proxy to avoid CORS
|
||||
if (import.meta.env.DEV) {
|
||||
return `/api/${endpoint}`;
|
||||
}
|
||||
// In production, use full API URL
|
||||
const apiUrl = import.meta.env.VITE_API_URL || 'http://localhost:5015';
|
||||
return `${apiUrl}/${endpoint}`;
|
||||
};
|
||||
|
||||
// Convert video source URLs based on type
|
||||
const getVideoSrc = () => {
|
||||
// Defensive check for empty/undefined src
|
||||
|
|
@ -32,16 +43,8 @@ const VideoPlayer = ({ src, onFrameExtract, onSaveToProject, className = '', aut
|
|||
// If it's a /generated_videos/ path, convert to streaming endpoint
|
||||
if (src.startsWith('/generated_videos/')) {
|
||||
const filename = src.replace('/generated_videos/', '');
|
||||
// In development, use Vite proxy
|
||||
if (import.meta.env.DEV) {
|
||||
const convertedUrl = `/api/stream_video.php?file=${encodeURIComponent(filename)}`;
|
||||
console.log(`VideoPlayer URL conversion: ${src} → ${convertedUrl}`);
|
||||
return convertedUrl;
|
||||
}
|
||||
// In production, use full API URL
|
||||
const apiUrl = import.meta.env.VITE_API_URL || 'http://localhost:5015';
|
||||
const convertedUrl = `${apiUrl}/stream_video.php?file=${encodeURIComponent(filename)}`;
|
||||
console.log(`VideoPlayer URL conversion (prod): ${src} → ${convertedUrl}`);
|
||||
const convertedUrl = getApiUrl(`stream_video.php?file=${encodeURIComponent(filename)}`);
|
||||
console.log(`VideoPlayer URL conversion: ${src} → ${convertedUrl}`);
|
||||
return convertedUrl;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue