From c994398850701fc8a6c7600c85de5dc5ebcd14a9 Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Fri, 5 Jun 2026 11:14:36 -0600 Subject: [PATCH] Rework prompts and captioning systems for ideogram to more strictly match the format provided by ideogram. --- .../captioner/Ideogram4Captioner.py | 63 +++++++++++++-- .../prompts/ideogram4_caption_prompt.py | 21 +++-- .../captioner/prompts/ideogram4_prompt.py | 26 +++--- .../prompts/ideogram4_upsample_prompt.py | 23 ++++-- ui/src/helpers/defaultSamples.ts | 81 ++++++++++++++++--- ui_scripts/upsample_ideogram4_caption.py | 54 +++++++++++-- 6 files changed, 220 insertions(+), 48 deletions(-) diff --git a/extensions_built_in/captioner/Ideogram4Captioner.py b/extensions_built_in/captioner/Ideogram4Captioner.py index 96a2c8fd..6e46048d 100644 --- a/extensions_built_in/captioner/Ideogram4Captioner.py +++ b/extensions_built_in/captioner/Ideogram4Captioner.py @@ -25,6 +25,11 @@ MIN_NEW_TOKENS = 3072 # generator was trained on, instead of ugly fractions like 1023:768. MAX_AR_DENOMINATOR = 16 +# color_palette caps: the model often ignores these, so we enforce them. +MAX_IMAGE_PALETTE = 16 # style_description.color_palette +MAX_ELEMENT_PALETTE = 5 # per-element color_palette +HEX_COLOR_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") + class Ideogram4Captioner(Qwen3VLCaptioner): def __init__(self, process_id: int, job, config: OrderedDict, **kwargs): @@ -104,22 +109,64 @@ class Ideogram4Captioner(Qwen3VLCaptioner): # stored order is [y1, x1, y2, x2] return [y1, x1, y2, x2] - def _normalize_caption(self, data: dict, aspect_ratio: str) -> dict: - """Validate/cleanup the parsed caption before storage, reordering each - bbox from model-native [x1,y1,x2,y2] to stored [y1,x1,y2,x2].""" - # Force the aspect ratio we computed; the model is told to echo it but - # we know the true value. - data["aspect_ratio"] = aspect_ratio + def _sanitize_palette(self, palette, max_len): + """Keep unique, valid hex colors in order, capped to max_len. Returns the + cleaned list, or None if nothing valid remains (drop the key).""" + if not isinstance(palette, (list, tuple)): + return None + seen = set() + out = [] + for c in palette: + if not isinstance(c, str): + continue + c = c.strip() + if not HEX_COLOR_RE.match(c): + continue + key = c.lower() + if key in seen: + continue + seen.add(key) + out.append(c) + if len(out) >= max_len: + break + return out or None + + def _normalize_caption(self, data: dict) -> dict: + """Validate/cleanup the parsed caption before storage: drop input-only + aspect_ratio, reorder bboxes to [y1,x1,y2,x2], and cap color palettes + (16 per image, 5 per element) since the model often exceeds them.""" + # aspect_ratio is input-only context, not part of the output. Drop it if + # the model echoed it anyway. + data.pop("aspect_ratio", None) + + style = data.get("style_description") + if isinstance(style, dict) and "color_palette" in style: + pal = self._sanitize_palette(style["color_palette"], MAX_IMAGE_PALETTE) + if pal is None: + style.pop("color_palette", None) + else: + style["color_palette"] = pal + decon = data.get("compositional_deconstruction", {}) elements = decon.get("elements", []) if isinstance(elements, list): for el in elements: - if isinstance(el, dict) and "bbox" in el: + if not isinstance(el, dict): + continue + if "bbox" in el: cleaned = self._convert_bbox(el["bbox"]) if cleaned is None: el.pop("bbox", None) else: el["bbox"] = cleaned + if "color_palette" in el: + pal = self._sanitize_palette( + el["color_palette"], MAX_ELEMENT_PALETTE + ) + if pal is None: + el.pop("color_palette", None) + else: + el["color_palette"] = pal return data def get_caption_for_file(self, file_path: str) -> Optional[str]: @@ -172,7 +219,7 @@ class Ideogram4Captioner(Qwen3VLCaptioner): ) return output_text - data = self._normalize_caption(data, aspect_ratio) + data = self._normalize_caption(data) # Store pretty JSON for QC/editing; the dataloader minifies at load. return json.dumps(data, ensure_ascii=False, indent=2) except Exception as e: diff --git a/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py index 9e0e59bf..3d04742a 100644 --- a/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py +++ b/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py @@ -19,16 +19,16 @@ You are CAPTIONING a real image, not imagining one. Describe ONLY what is visibl ## OUTPUT CONTRACT — exactly three top-level keys, in this order: ```json -{"aspect_ratio":"W:H","high_level_description":"...","compositional_deconstruction":{"background":"...","elements":[ ... ]}} +{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}} ``` - Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. - Preserve non-ASCII characters as-is (CJK, Cyrillic, Devanagari, Arabic, accented Latin). Never escape with `\\uNNNN`, transliterate, or replace `café` with `cafe`. - Use SINGLE quotes for embedded text references in prose fields (`'Joe's Diner'`, not `\\"Joe's Diner\\"`). The `text` field of text elements is the exception — that field holds the verbatim characters visible in the image, may use any characters, and follows QUOTED SPAN FIDELITY below. -### `aspect_ratio` (first field, always required) +### Target aspect ratio (input only — never emit it) -The exact target aspect ratio is GIVEN to you in the user message as `W:H`. Echo it VERBATIM. Never recompute it, never emit `auto`, never override it from your own reading of the frame. Every bbox you emit is normalized to THIS aspect ratio. +The user message gives the image's aspect ratio as `W:H`. Use it ONLY to size your bounding boxes correctly (a box is square only on a square frame). Do NOT emit an `aspect_ratio` key — it is not part of the output. ### `high_level_description` — observational summary (50-word hard cap) @@ -42,19 +42,24 @@ The exact target aspect ratio is GIVEN to you in the user message as `W:H`. Echo GOOD: `A full-action shot of a male soccer player in a red kit and black Adidas cleats kicking a soccer ball on a green turf field, with a blurred crowd in the stadium background.` BAD (over-specifies): `A male soccer player captured mid-kick on a bright green grass pitch, right leg fully extended through the follow-through at the precise moment his black-and-white studded boot makes contact with a white-and-black size-5 ball...` -## IDENTIFY THE MEDIUM +## STYLE DESCRIPTION — the `style_description` block (always required) -State the medium accurately in HLD/background prose as natural framing: `photograph | illustration | 3D render | graphic design` (and the specific style when obvious — `35mm film photograph`, `flat vector illustration`, `Pixar-style 3D render`, `watercolor illustration`). Read it from the image; do not impose a default. Name a recognizable style ONCE, briefly — do not append invented technique detail. +A nested object capturing the image's overall look, OBSERVED from the image (never invented). Exactly these five keys: +- `aesthetics` — the overall mood/aesthetic in a short phrase (`Cinematic, minimal, serene.` / `Bright, playful, high-energy.`). +- `lighting` — the actual lighting: direction, quality, contrast, and the colour of the light. Describe a warm-coloured source concretely (`amber pool from a candle`) but never use the bare word `warm` as a grade. +- `photo` — the medium-specific capture/render spec. Photograph → camera/film look, framing, grain, focus (`35mm film still, 16:9 framing, subtle grain, shallow depth of field`). Other media → the rendering technique (`flat vector, clean edges` / `octane 3D render, soft global illumination` / `loose watercolor on textured paper`). +- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` Read it from the image; do not impose a default. +- `color_palette` — an array of the image's DOMINANT colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. Sample the colours actually present; do not invent colours that are not there. ## ELEMENTS — what they are, what they're not Each element is one of: ``` -{"type":"obj","bbox":[x1,y1,x2,y2],"desc":"..."} -{"type":"text","bbox":[x1,y1,x2,y2],"text":"LINE ONE\\nLINE TWO","desc":"..."} +{"type":"obj","bbox":[x1,y1,x2,y2],"color_palette":["#RRGGBB"],"desc":"..."} +{"type":"text","bbox":[x1,y1,x2,y2],"color_palette":["#RRGGBB"],"text":"LINE ONE\\nLINE TWO","desc":"..."} ``` -`bbox` is optional per-element (see BBOX section below). +`bbox` and `color_palette` are both OPTIONAL per-element. `bbox`: see BBOX section below. `color_palette`: up to 5 hex strings of that element's own dominant colours — include it when the element has distinctive colours worth pinning (a red jacket, a brand logo, coloured text), omit it for colour-neutral elements. ### SINGLE SUBJECT = SINGLE ELEMENT diff --git a/extensions_built_in/captioner/prompts/ideogram4_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_prompt.py index dffc1787..5d550404 100644 --- a/extensions_built_in/captioner/prompts/ideogram4_prompt.py +++ b/extensions_built_in/captioner/prompts/ideogram4_prompt.py @@ -10,19 +10,16 @@ You convert a natural-language user idea into a structured JSON caption an image ## OUTPUT CONTRACT — exactly three top-level keys, in this order: ```json -{"aspect_ratio":"W:H","high_level_description":"...","compositional_deconstruction":{"background":"...","elements":[ ... ]}} +{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}} ``` - Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. - Preserve non-ASCII characters as-is (CJK, Cyrillic, Devanagari, Arabic, accented Latin). Never escape with `\uNNNN`, transliterate, or replace `café` with `cafe`. - Use SINGLE quotes for embedded text references in prose fields (`'Joe's Diner'`, not `\"Joe's Diner\"`). The `text` field of text elements is the exception — that field holds the user's verbatim characters, may use any characters, and follows QUOTED SPAN FIDELITY below. -### `aspect_ratio` (first field, always required) +### Target aspect ratio (input only — never emit it) -A string in `W:H` form with positive integers (`1:1`, `16:9`, `9:16`, `4:5`, `3:1`, `2:3`, etc.). -- If the user message gives a concrete `W:H`, echo it verbatim. -- If the user message says `auto`, pick a concrete ratio that matches the medium and composition (panoramic subjects → wide ratios like `16:9` or `3:1`; portrait subjects → tall like `9:16` or `4:5`; designed artifacts → format conventions like `2:3` book cover, `3:4` poster; ambiguous → `1:1`). NEVER emit the literal string `auto`. -- The aspect ratio you commit to drives every bbox decision. Pick it first. +The user message gives a target aspect ratio as `W:H` (or `auto`). Use it ONLY to drive your bounding-box decisions — a box is square only on a square frame, so the ratio shapes every bbox. Do NOT emit an `aspect_ratio` key; it is not part of the output. ### `high_level_description` — observational summary (50-word hard cap) @@ -36,15 +33,26 @@ A string in `W:H` form with positive integers (`1:1`, `16:9`, `9:16`, `4:5`, `3: GOOD: `A full-action shot of a male soccer player in a red kit and black Adidas cleats kicking a soccer ball on a green turf field, with a blurred crowd in the stadium background.` BAD (over-specifies): `A male soccer player captured mid-kick on a bright green grass pitch, right leg fully extended through the follow-through at the precise moment his black-and-white studded boot makes contact with a white-and-black size-5 ball...` +### `style_description` — the global look block (always required) + +A nested object with exactly these five keys: +- `aesthetics` — overall mood/aesthetic in a short phrase (`Cinematic, minimal, serene.`). +- `lighting` — direction, quality, contrast, and colour of the light. Describe a warm-coloured source concretely (`amber sun low at the horizon`); never use the bare word `warm` as a grade. +- `photo` — the medium-specific capture/render spec. Photograph → camera/film look, framing, grain, focus (`35mm motion-picture film still, 16:9 framing, subtle grain`). Other media → the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor on textured paper`). +- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` +- `color_palette` — an array of the dominant colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. This conditions the image's colours directly, so commit to the actual hexes you intend. + +Name a recognized style ONCE here (see PLANNING → Style commitment); do not append invented technique detail on top of a well-known style name. + ## ELEMENTS — what they are, what they're not Each element is one of: ``` -{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"..."} -{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"..."} +{"type":"obj","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"desc":"..."} +{"type":"text","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"text":"LINE ONE\nLINE TWO","desc":"..."} ``` -`bbox` is optional per-element (see BBOX section below). +`bbox` and `color_palette` are both OPTIONAL per-element. `bbox`: see BBOX section below. `color_palette`: up to 5 hex strings steering that element's own dominant colours — include it when the element has a distinctive colour (a red jacket, a brand logo, coloured text), omit it otherwise. ### SINGLE SUBJECT = SINGLE ELEMENT diff --git a/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py index c8f8d51b..31a0176a 100644 --- a/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py +++ b/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py @@ -17,29 +17,40 @@ You convert a user prompt into a structured JSON caption an image renderer can c ## OUTPUT CONTRACT — exactly three top-level keys, in this order: ```json -{"aspect_ratio":"W:H","high_level_description":"...","compositional_deconstruction":{"background":"...","elements":[ ... ]}} +{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}} ``` - Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. - Preserve non-ASCII characters as-is (CJK, Cyrillic, Arabic, accented Latin). Never escape them as unicode code-point sequences or transliterate. - Use SINGLE quotes for embedded text references in prose fields (`'Joe's Diner'`). The `text` field is the exception — it holds verbatim characters. -### `aspect_ratio` (first field) +### Target aspect ratio (input only — never emit it) -The target ratio is given. Echo it VERBATIM. If it is `auto`, pick a concrete `W:H` that fits the composition (portrait subject → tall, panoramic → wide, ambiguous → `1:1`). Never emit `auto`. +The user message gives a target aspect ratio as `W:H` (or `auto`). Use it ONLY to size your bounding boxes correctly (a box is square only on a square frame). Do NOT emit an `aspect_ratio` key — it is not part of the output. ### `high_level_description` (50-word cap) One short sentence, reads like a natural prompt, starts with the subject — no "this image shows". Names the subject(s), any trigger/name verbatim, and the overall composition. Don't enumerate fine detail. +## STYLE DESCRIPTION — the `style_description` block (always required) + +A nested object with exactly these five keys, filled FROM the prompt: +- `aesthetics` — the overall mood/aesthetic in a short phrase. +- `lighting` — the lighting (direction, quality, colour). Describe a warm-coloured source concretely; never use the bare word `warm` as a grade. +- `photo` — the medium-specific capture/render spec (photograph → camera/film look, framing, grain, focus; other media → the rendering technique). +- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` +- `color_palette` — an array of dominant colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. + +Respect FIDELITY: if the prompt NAMES a style, medium, artist, or look, put it in these fields BY NAME (e.g. `medium`/`photo`/`aesthetics`) and do NOT invent its characteristics. Pull lighting and colours from what the prompt states. In faithful mode, only commit to a value the prompt implies, keeping the rest minimal; in creative mode you may infer fitting style values — but never elaborate a named style and never override what the user gave. + ## ELEMENTS Each element is one of: ``` -{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"..."} -{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"..."} +{"type":"obj","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"desc":"..."} +{"type":"text","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"text":"LINE ONE\nLINE TWO","desc":"..."} ``` -`bbox` is optional per element (see BBOX). +`bbox` and `color_palette` are both OPTIONAL per element. `bbox`: see BBOX. `color_palette`: up to 5 hex strings of that element's dominant colours — include it when the prompt gives the element a distinctive colour (a red jacket, coloured text), otherwise omit. - **One coherent subject = ONE element.** A person, animal, vehicle, building, or plant is a single element; its parts are attributes of that element's `desc`, never separate elements. Multiple distinct subjects = multiple elements (one each). - **`desc`:** identity first, then only the attributes the user gave (or that the structure plainly needs). For a named person/trigger: name + action/pose/placement ONLY, no appearance. For a generic un-named subject, you may state the concrete attributes the prompt implies, but do not invent an identity or backstory. diff --git a/ui/src/helpers/defaultSamples.ts b/ui/src/helpers/defaultSamples.ts index bd65941a..52fa72d6 100644 --- a/ui/src/helpers/defaultSamples.ts +++ b/ui/src/helpers/defaultSamples.ts @@ -412,8 +412,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A 35mm film photograph of a red-haired woman in a green jacket playing chess at an outdoor park table, mid-move over a wooden board, while a fiery explosion erupts from a building in the distant background.", + "style_description": { + "aesthetics": "Cinematic, tense, candid realism.", + "lighting": "Overcast afternoon daylight, soft and low-contrast, cool-neutral white balance.", + "photo": "35mm film still, subtle grain, natural depth of field.", + "medium": "Photograph.", + "color_palette": ["#8C9B82", "#B7402A", "#5A5F57", "#9AA7AE", "#D98A3D"] + }, "compositional_deconstruction": { "background": "An urban public park on an overcast afternoon under a pale grey-blue sky, cool-neutral white balance. A grassy lawn with scattered fallen leaves stretches behind the foreground table, bordered by a paved walking path and a row of bare-branched trees. In the far distance, a multi-story stone building erupts in a large orange-and-yellow fireball with a thick black smoke plume rising and rolling outward, sending a faint haze across the upper sky. The blast is out of focus and far off, framed between the tree trunks.", "elements": [ @@ -455,8 +461,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A 35mm film photograph of a woman in a grey beanie holding a coffee cup while sitting at a wooden cafe table by a window, with a blurred cafe interior behind her.", + "style_description": { + "aesthetics": "Cozy, relaxed, intimate.", + "lighting": "Soft diffused window daylight, cool-neutral white balance, low contrast.", + "photo": "35mm film still, shallow depth of field, subtle grain.", + "medium": "Photograph.", + "color_palette": ["#9A5A3E", "#E0D2BA", "#7C7872", "#B07C45", "#33312D"] + }, "compositional_deconstruction": { "background": "Interior of a small cafe shot in natural daylight with cool-neutral white balance. A large window occupies the left portion of the frame, soft diffused daylight falling across the scene. Exposed brick wall in warm reddish-brown tones runs along the back, partly out of focus. A wooden shelf mounted on the back wall holds a row of white ceramic mugs and a small potted trailing plant. Pendant lights with matte black shades hang from the ceiling, slightly blurred. The floor is wide-plank weathered oak. Distant blurred tables and chairs recede into the soft-focus background on the right side.", "elements": [ @@ -519,8 +531,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A fish-eye lens photograph of a horse DJing behind turntables at a packed night club, holding a martini glass, surrounded by laser lights and drifting smoke-machine haze on a glowing dance floor.", + "style_description": { + "aesthetics": "High-energy, surreal, neon nightlife.", + "lighting": "Dim club lighting with magenta and cyan washes and crisscrossing green and magenta laser beams cutting through haze.", + "photo": "Fish-eye lens with strong barrel distortion, deep shadow contrast.", + "medium": "Photograph.", + "color_palette": ["#0B0B12", "#D81E8F", "#1FB6C9", "#37C46A", "#6A4A2E"] + }, "compositional_deconstruction": { "background": "Interior of a dark night club shot through a fish-eye lens with strong barrel distortion bowing the edges of the frame. Black walls and low ceiling studded with mounted laser-light fixtures throwing crisscrossing green and magenta beams that cut through thick drifting haze from a smoke machine. Ambient lighting is dim with cool magenta and cyan washes pooling across a glossy black dance floor that reflects fragmented colored beams. A blurred simplified crowd of clubgoers fills the mid-distance, hands raised, rendered as dark silhouettes against the colored glow.", "elements": [ @@ -567,13 +585,20 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { } ] } +} `, }, { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A 35mm film photograph of a smiling man proudly showing off his graphic t-shirt on a sandy beach, with a great white shark leaping out of the ocean in the background.", + "style_description": { + "aesthetics": "Bright, playful, candid.", + "lighting": "Bright overcast daylight, soft and shadowless, cool-neutral white balance.", + "photo": "35mm film still, natural depth of field, subtle grain.", + "medium": "Photograph.", + "color_palette": ["#C9B68C", "#2E6B7A", "#9FB7BE", "#1B3A5C", "#E7E2D6"] + }, "compositional_deconstruction": { "background": "Sandy beach scene under a bright overcast sky with cool-neutral white balance. Pale tan sand stretches across the lower portion, slightly damp and packed near the waterline with scattered footprints. Behind the man, the open ocean fills the midground, deep blue-green with choppy whitecaps and rolling waves breaking toward the shore. The horizon line sits high in the frame where the sea meets a hazy pale sky with thin diffuse clouds. Soft even daylight, no harsh shadows, accurate natural color.", "elements": [ @@ -616,8 +641,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A brown grizzly bear standing upright on its hind legs, lifting a wooden log onto a half-built log cabin in a snow-covered mountain clearing, with snowy pine forest and peaks behind, rendered as a 35mm film photograph.", + "style_description": { + "aesthetics": "Serene, rugged, wintry.", + "lighting": "Pale overcast winter daylight, even and shadowless, cool-neutral white balance.", + "photo": "35mm film still, subtle grain, soft natural focus.", + "medium": "Photograph.", + "color_palette": ["#E8EDF0", "#6B4A30", "#3C5240", "#9AA6AD", "#C8A877"] + }, "compositional_deconstruction": { "background": "Snow-covered alpine clearing under a pale overcast winter sky with soft diffused daylight and cool-neutral white balance. Thick fresh snow blankets the ground, undisturbed except around the build site. A dense forest of snow-laden evergreen pines fills the midground, their branches drooping under powder. Jagged grey-and-white granite mountain peaks rise across the distant horizon, partly veiled in light haze. Faint snowflakes drift through the air. The light is even and shadowless across the scene.", "elements": [ @@ -669,8 +700,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A punk rocker woman mid-performance on a concert stage, playing an electric guitar and singing into a microphone, with laser lights cutting through haze in a 35mm concert photograph.", + "style_description": { + "aesthetics": "Gritty, energetic, high-contrast.", + "lighting": "Dark stage lit by green and magenta laser beams through haze, deep shadow contrast, cool-neutral white balance.", + "photo": "35mm concert photograph, subtle grain, deep contrast.", + "medium": "Photograph.", + "color_palette": ["#0C0C10", "#37C46A", "#D81E8F", "#C9C9C9", "#5A4633"] + }, "compositional_deconstruction": { "background": "A dark concert stage shell with a black back wall and exposed steel truss rigging overhead holding stage fixtures. Green and magenta laser beams fan out across the upper space, cutting through a light haze that fills the air and scatters the beams into visible shafts. The stage floor is matte black with scuffed gaffer-tape marks. Distant blurred crowd silhouettes fill the lower foreground edge, lit faintly by stage spill. 35mm concert photograph with cool-neutral white balance and deep shadow contrast.", "elements": [ @@ -733,8 +770,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A 35mm film photograph of a bearded hipster man assembling a wooden chair on a workbench in a cluttered woodworking shop, surrounded by hand tools and lumber.", + "style_description": { + "aesthetics": "Rustic, focused, artisanal.", + "lighting": "Diffused overcast daylight from a high window, cool-neutral white balance, low contrast.", + "photo": "35mm film still, subtle grain, natural depth of field.", + "medium": "Photograph.", + "color_palette": ["#8A5A3C", "#6E4327", "#9A9488", "#4A5340", "#C7B299"] + }, "compositional_deconstruction": { "background": "Interior of a small woodworking workshop with weathered exposed-brick walls on the left and unfinished plywood-panel walls on the right. Sawdust-dusted concrete floor. A pegboard mounted on the rear wall holds rows of hanging hand tools. A single industrial window high on the left wall lets in diffused overcast daylight with a cool-neutral white balance. Fine sawdust haze drifts in the air. Coils of wood shavings and scattered offcuts rest near the wall base. Shot on 35mm film.", "elements": [ @@ -797,8 +840,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A studio fashion photograph of a man in a medium shot modeling a casual outfit against a seamless white backdrop, lit with even studio lighting.", + "style_description": { + "aesthetics": "Clean, minimal, editorial.", + "lighting": "Even diffused studio softbox lighting, neutral white balance, shadowless.", + "photo": "Studio fashion photograph, sharp focus, seamless white cyclorama.", + "medium": "Photograph.", + "color_palette": ["#F2F2F0", "#9A9CA0", "#2A2F3C", "#5B5E66", "#D8D8D6"] + }, "compositional_deconstruction": { "background": "Seamless white studio backdrop, smoothly lit with even diffused studio lighting from soft boxes on both sides, producing a clean bright cyclorama with no visible seams, corners, or shadows behind the subject. Neutral white balance.", "elements": [ @@ -820,8 +869,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A 35mm film photograph of a man standing on a city sidewalk holding a white cardboard sign reading 'this is a sign', shot at eye-level with neutral daylight.", + "style_description": { + "aesthetics": "Plain, candid, documentary.", + "lighting": "Overcast daylight, soft and even, cool-neutral white balance.", + "photo": "35mm film still, eye-level, subtle grain.", + "medium": "Photograph.", + "color_palette": ["#9AA0A4", "#7C4A38", "#3A3D44", "#1E3A66", "#E8E6E0"] + }, "compositional_deconstruction": { "background": "An urban sidewalk scene under overcast daylight with cool-neutral white balance. A grey concrete pavement runs along the bottom, bordered by the brick facade of a low storefront building with large plate-glass windows. A few out-of-focus pedestrians and a parked dark sedan sit in the blurred mid-distance. Pale grey sky visible above the rooflines.", "elements": [ @@ -864,8 +919,14 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { { prompt: ` { - "aspect_ratio": "1:1", "high_level_description": "A 35mm film photograph of a muscular bulldog in a worn leather jacket standing beside a battered motorcycle in a post-apocalyptic desert, gripping a sawed-off shotgun, with a hazy ruined skyline on the horizon.", + "style_description": { + "aesthetics": "Rugged, cinematic, post-apocalyptic.", + "lighting": "Pale dust-choked daylight softened by airborne grit, cool-neutral white balance, low contrast.", + "photo": "35mm film still, subtle grain, hazy distance.", + "medium": "Photograph.", + "color_palette": ["#C2A878", "#6B4A2E", "#3A352E", "#9A8A6C", "#B5562A"] + }, "compositional_deconstruction": { "background": "A sun-scorched post-apocalyptic desert under a pale dust-choked sky, cool-neutral white balance with a thin haze of airborne grit softening the light. Cracked sandy hardpan stretches to a distant horizon where the silhouettes of half-collapsed buildings, a leaning radio tower, and rusted girders rise out of the heat shimmer. Scattered scrub brush and faint tire tracks mark the packed dirt, and a thin band of overcast cloud sits low over the ruined skyline.", "elements": [ diff --git a/ui_scripts/upsample_ideogram4_caption.py b/ui_scripts/upsample_ideogram4_caption.py index 273f099b..7f601888 100644 --- a/ui_scripts/upsample_ideogram4_caption.py +++ b/ui_scripts/upsample_ideogram4_caption.py @@ -127,21 +127,61 @@ def sanitize_bbox(bbox): return [y1, x1, y2, x2] -def sanitize_caption(data: dict, aspect_ratio: str) -> dict: - """Light cleanup: force a concrete aspect ratio (never echo 'auto') and clean - each bbox. Leaves prose untouched.""" - if aspect_ratio.lower() != "auto": - data["aspect_ratio"] = aspect_ratio +HEX_COLOR_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") + + +def sanitize_palette(palette, max_len): + """Keep unique, valid hex colors in order, capped to max_len. Returns the + cleaned list, or None if nothing valid remains (drop the key).""" + if not isinstance(palette, (list, tuple)): + return None + seen = set() + out = [] + for c in palette: + if not isinstance(c, str): + continue + c = c.strip() + if not HEX_COLOR_RE.match(c): + continue + key = c.lower() + if key in seen: + continue + seen.add(key) + out.append(c) + if len(out) >= max_len: + break + return out or None + + +def sanitize_caption(data: dict) -> dict: + """Light cleanup: drop any aspect_ratio key (input-only context, not output), + clean each bbox, and cap color palettes (16 per image, 5 per element).""" + data.pop("aspect_ratio", None) + style = data.get("style_description") + if isinstance(style, dict) and "color_palette" in style: + pal = sanitize_palette(style["color_palette"], 16) + if pal is None: + style.pop("color_palette", None) + else: + style["color_palette"] = pal decon = data.get("compositional_deconstruction", {}) elements = decon.get("elements", []) if isinstance(elements, list): for el in elements: - if isinstance(el, dict) and "bbox" in el: + if not isinstance(el, dict): + continue + if "bbox" in el: cleaned = sanitize_bbox(el["bbox"]) if cleaned is None: el.pop("bbox", None) else: el["bbox"] = cleaned + if "color_palette" in el: + pal = sanitize_palette(el["color_palette"], 5) + if pal is None: + el.pop("color_palette", None) + else: + el["color_palette"] = pal return data @@ -184,7 +224,7 @@ def upsample_one( log("Failed to parse JSON from model output. Raw output follows:") log(output_text) return None - return sanitize_caption(data, aspect_ratio) + return sanitize_caption(data) def normalize_item(item, default_aspect_ratio):