Merged in develop (pull request #23)

Develop
This commit is contained in:
Nick Viljoen 2026-05-17 20:18:23 +00:00
commit 7b0e22060e
9 changed files with 1538 additions and 20 deletions

3
.gitignore vendored
View file

@ -86,3 +86,6 @@ config.env
backend/config.env
config/development.env
config/production.env
# Local test fixtures (real HP Source Messaging files; not for commit)
backend/tests/fixtures/

View file

@ -8,6 +8,7 @@ import sys
import json
import base64
import collections
import html
import importlib
import traceback
import re
@ -858,6 +859,8 @@ def get_client_from_profile(profile_id):
return 'amazon'
elif profile_lower.startswith('boots'):
return 'boots'
elif profile_lower.startswith('hp_'):
return 'hp'
elif profile_lower.startswith(('dow_jones', 'dj_', 'marketwatch', 'mw_', 'wsj')):
return 'dow_jones'
else:
@ -973,6 +976,12 @@ def generate_html_content(report_data, filename, file_path=None):
json_data = check_data.get('json_data', {})
response_text = ""
# Structured findings (e.g. hp_copy_review) render as a table
# instead of the default response-text block. If absent, falls
# back to the existing text rendering below.
findings = (json_data or {}).get('findings') if isinstance(json_data, dict) else None
findings_html = _render_findings_table(findings) if findings is not None else None
# Try to extract detailed analysis from JSON data
if json_data:
# Look for common detailed fields in the JSON
@ -1099,12 +1108,12 @@ def generate_html_content(report_data, filename, file_path=None):
</div>
<div class="analysis-section">
<h4>Analysis Details:</h4>
<div class="response-text">{response_text.replace(chr(10), '<br>')}</div>
{f'<div class="response-text">{html.escape(json_data.get("summary", "") or "") if isinstance(json_data, dict) else ""}</div>{findings_html}' if findings_html is not None else f'<div class="response-text">{response_text.replace(chr(10), "<br>")}</div>'}
</div>
</div>
</div>
"""
# Get summary score result
overall_score = report_data['summary']['overall_score']
overall_result, overall_color = get_score_result(overall_score/10) # Normalize to 0-10 scale
@ -1163,6 +1172,16 @@ def generate_html_content(report_data, filename, file_path=None):
.json-toggle {{ cursor: pointer; color: #FFC407; text-decoration: underline; padding: 15px; text-align: center; font-weight: bold; }}
.json-view {{ display: none; margin-top: 20px; }}
.json-view pre {{ background-color: #2d3748; color: #e2e8f0; padding: 20px; border-radius: 10px; overflow-x: auto; font-size: 0.9em; }}
.findings-table {{ width: 100%; border-collapse: collapse; margin-top: 12px; font-size: 0.92em; }}
.findings-table th {{ background: #f1f3f5; color: #495057; text-align: left; padding: 8px 10px; border-bottom: 2px solid #dee2e6; font-weight: 600; }}
.findings-table td {{ padding: 8px 10px; border-bottom: 1px solid #e9ecef; vertical-align: top; word-break: break-word; }}
.findings-table tr:last-child td {{ border-bottom: none; }}
.findings-table code {{ background: #f8f9fa; padding: 2px 5px; border-radius: 4px; font-family: 'SFMono-Regular', Consolas, Menlo, monospace; font-size: 0.9em; color: #c7254e; }}
.priority-pill {{ display: inline-block; padding: 3px 8px; border-radius: 10px; color: white; font-weight: 600; font-size: 0.78em; letter-spacing: 0.03em; }}
.priority-high {{ background-color: #dc3545; }}
.priority-medium {{ background-color: #fd7e14; }}
.priority-low {{ background-color: #28a745; }}
.muted {{ color: #6c757d; font-size: 0.9em; }}
</style>
</head>
<body>
@ -1256,6 +1275,45 @@ def generate_html_response(report_data, filename, save_to_file=False, session_id
else:
return Response(html_content, mimetype='text/html')
def _render_findings_table(findings):
"""Render an hp_copy_review-style findings array as an HTML table.
Each finding dict is expected to carry: priority (high|medium|low),
category, quote, issue, suggested_fix, source_reference. All string
fields are HTML-escaped before interpolation. An empty/None findings
list renders a friendly "clean copy" note instead of an empty table.
"""
if not findings:
return '<p class="muted">No findings — copy is clean.</p>'
rows = []
for f in findings:
priority = (f.get('priority') or 'low').lower()
pri_class = {
'high': 'priority-high',
'medium': 'priority-medium',
'low': 'priority-low',
}.get(priority, 'priority-low')
quote_raw = (f.get('quote') or '')[:200]
rows.append(
'<tr>'
f'<td><span class="priority-pill {pri_class}">{html.escape(priority.upper())}</span></td>'
f'<td><code>{html.escape(f.get("category", "") or "")}</code></td>'
f'<td><code>{html.escape(quote_raw)}</code></td>'
f'<td>{html.escape(f.get("issue", "") or "")}</td>'
f'<td>{html.escape(f.get("suggested_fix", "") or "")}</td>'
f'<td class="muted">{html.escape(f.get("source_reference", "") or "")}</td>'
'</tr>'
)
return (
'<table class="findings-table"><thead><tr>'
'<th>Priority</th><th>Category</th><th>Quote</th>'
'<th>Issue</th><th>Suggested fix</th><th>Source</th>'
'</tr></thead><tbody>'
+ ''.join(rows) +
'</tbody></table>'
)
def _render_technical_section_html(report):
"""Render the technical pre-flight report as an HTML block. Empty string if no report."""
if not report or report.get('kind') in (None, 'unknown'):
@ -1340,7 +1398,14 @@ def generate_comprehensive_html_report(analysis_result, filename, file_path=None
score_color = '#28a745' if score >= 6 else '#dc3545'
response = result.get('response', 'No response available')
display_name = check_name.replace('_', ' ').replace(chr(32).join([w.capitalize() for w in check_name.split('_')]), check_name.replace('_', ' ').title())
# Structured findings (e.g. hp_copy_review) render as a table
# instead of the default response-text block. Fallback to the
# existing response rendering when 'findings' is absent.
json_data = result.get('json_data') if isinstance(result, dict) else None
findings = json_data.get('findings') if isinstance(json_data, dict) else None
findings_html = _render_findings_table(findings) if findings is not None else None
# Remove JSON blocks for cleaner display and handle empty responses
response = re.sub(r'```json.*?```', '', response, flags=re.DOTALL).strip()
if not response:
@ -1367,7 +1432,7 @@ def generate_comprehensive_html_report(analysis_result, filename, file_path=None
</div>
<div class="analysis-section">
<h4>Analysis Details:</h4>
<div class="response-text">{response.replace(chr(10), '<br>')}</div>
{f'<div class="response-text">{html.escape(json_data.get("summary", "") or "") if isinstance(json_data, dict) else ""}</div>{findings_html}' if findings_html is not None else f'<div class="response-text">{response.replace(chr(10), "<br>")}</div>'}
</div>
</div>
</div>
@ -1421,6 +1486,16 @@ def generate_comprehensive_html_report(analysis_result, filename, file_path=None
.check-metadata {{ background: #f8f9fa; padding: 15px; border-radius: 8px; margin-bottom: 15px; }}
.analysis-section h4 {{ color: #495057; margin-bottom: 10px; }}
.response-text {{ background: #f8f9fa; padding: 15px; border-radius: 8px; line-height: 1.6; font-family: 'Montserrat', Georgia, serif; }}
.findings-table {{ width: 100%; border-collapse: collapse; margin-top: 12px; font-size: 0.92em; }}
.findings-table th {{ background: #f1f3f5; color: #495057; text-align: left; padding: 8px 10px; border-bottom: 2px solid #dee2e6; font-weight: 600; }}
.findings-table td {{ padding: 8px 10px; border-bottom: 1px solid #e9ecef; vertical-align: top; word-break: break-word; }}
.findings-table tr:last-child td {{ border-bottom: none; }}
.findings-table code {{ background: #f8f9fa; padding: 2px 5px; border-radius: 4px; font-family: 'SFMono-Regular', Consolas, Menlo, monospace; font-size: 0.9em; color: #c7254e; }}
.priority-pill {{ display: inline-block; padding: 3px 8px; border-radius: 10px; color: white; font-weight: 600; font-size: 0.78em; letter-spacing: 0.03em; }}
.priority-high {{ background-color: #dc3545; }}
.priority-medium {{ background-color: #fd7e14; }}
.priority-low {{ background-color: #28a745; }}
.muted {{ color: #6c757d; font-size: 0.9em; }}
</style>
</head>
<body>
@ -1632,6 +1707,15 @@ def get_reference_asset_content(reference_asset_id):
reference_content += f"\nLocalization Matrix: Contains {', '.join(loc_messages)} "
reference_content += f"for {len(loc_countries)} markets ({', '.join(loc_countries[:10])}).\n"
reference_content += "Expected copy will be cross-referenced with the media plan during analysis.\n"
elif file_record.get('summary_path'):
# Source-messaging Excel (HP and similar) — inject the Gemini-generated Markdown summary
try:
with open(file_record['summary_path'], 'r', encoding='utf-8') as f:
summary = f.read()
reference_content += f"\nSource Messaging Summary (extracted from {original_filename}):\n{summary}\n"
except Exception as e:
print(f"Failed to read source-messaging summary at {file_record['summary_path']}: {e}")
reference_content += f"\nReference file ({file_ext}) uploaded but summary unreadable.\n"
else:
reference_content += f"\nReference file ({file_ext}) uploaded as reference.\n"
else:
@ -4832,15 +4916,15 @@ def upload_brand_guideline():
).start()
file_record['processing_status'] = 'processing'
# Trigger localization matrix parsing for Excel files
# Trigger Excel processing: try localization matrix first (existing
# clients), fall back to Source Messaging summary (HP and similar).
elif file_record.get('file_type') in ('.xlsx', '.xls'):
import threading
def _process_localization_bg(fid, spath, fdir):
def _process_excel_bg(fid, spath, fdir):
try:
from localization_processor import parse_localization_matrix
parsed = parse_localization_matrix(spath)
if parsed:
# Save parsed JSON
json_path = os.path.join(fdir, f"{fid}_localization.json")
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(parsed, f, indent=2, ensure_ascii=False)
@ -4855,25 +4939,34 @@ def upload_brand_guideline():
print(f"Localization matrix parsing complete for {fid}: "
f"{len(parsed.get('messages', {}))} messages, "
f"{len(parsed.get('countries', []))} countries")
else:
brand_db.update_file_record(fid, {
'processed': True,
'processed_at': datetime.now().isoformat(),
'asset_type': 'excel_file',
})
print(f"Excel file {fid} is not a localization matrix, stored as-is")
return
# Not a localization matrix — process as Source Messaging
# (HP-style structured Markdown summary via Gemini).
from excel_processor import process_excel_file
summary_text, summary_path = process_excel_file(spath, fid)
brand_db.update_file_record(fid, {
'processed': True,
'processed_at': datetime.now().isoformat(),
'summary_path': summary_path,
'summary_length': len(summary_text),
'cover_image_path': None,
'asset_type': 'source_messaging',
})
print(f"Source-messaging summary complete for {fid}: "
f"{len(summary_text)} chars")
except Exception as e:
print(f"Localization matrix parsing failed for {fid}: {e}")
print(f"Excel processing failed for {fid}: {e}")
brand_db.update_file_record(fid, {
'processed': 'error',
'processing_error': str(e)
'processing_error': str(e),
})
threading.Thread(
target=_process_localization_bg,
target=_process_excel_bg,
args=(file_record['id'], file_record['stored_path'],
str(brand_db.files_dir)),
daemon=True
daemon=True,
).start()
file_record['processing_status'] = 'processing'

View file

@ -63,9 +63,10 @@ CLIENT_PROFILES = {
},
'hp': {
'name': 'HP',
'profiles': ['static_general', 'video_general'],
'profiles': ['hp_copy_review', 'static_general', 'video_general'],
'display_name': 'HP',
'description': 'Demo client — scope pending'
'description': 'HP marketing copy QC graded against canonical Source Messaging',
'default_profile': 'hp_copy_review',
},
'ferrero': {
'name': 'Ferrero',

162
backend/excel_processor.py Normal file
View file

@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Excel reference-asset processor for HP Source Messaging files.
Mirrors pdf_processor.py: openpyxl extracts raw cell content from every
sheet, Gemini 2.5 Pro summarises the result into structured Markdown
under brand_guidelines/files/{file_id}_summary.md. The hp_copy_review
check pulls that Markdown into its prompt at QC time.
Public surface:
process_excel_file(file_path, file_id) -> (summary_text, summary_path)
Behaviour mirrors pdf_processor.summarize_brand_guidelines: on Gemini
failure we write a degraded summary containing the raw extraction so
the reference asset stays usable downstream. The function does not
raise failures are logged and surfaced via the degraded payload.
"""
import os
from typing import Tuple
from openpyxl import load_workbook
BRAND_GUIDELINES_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'brand_guidelines', 'files'
)
# Cap raw extraction at ~50K chars to keep the summary prompt bounded.
# A 30-row, 12-column workbook is ~10-15K chars in practice; this leaves
# headroom for HP's larger source files without blowing the prompt budget.
_RAW_EXTRACTION_CAP = 50_000
_SYSTEM_PROMPT = """You're processing an HP Source Messaging Excel into a structured Markdown reference. Output these sections exactly, in this order:
## Product / Variant
(brand, product line, variant if any e.g. "HP OmniDesk Mini — Core")
## Key Selling Points (KSPs)
For each KSP: heading, value proposition, supporting body copy, message-length variants (ultra-short / short / medium / long if present in the source).
## Disclaimers / Footnotes
Numbered list, exact wording, what claim each footnote anchors to.
## Approved Brand and Product Names
Exact spellings, including trademark glyphs (, ®, ©).
## Variant Notes / Watch-outs
Anything explicitly marked variant-specific (e.g. "Mainstream only", "Core only", "must not appear in entry tier").
## Verboten Phrasing
Any explicitly disallowed or deprecated phrasing called out in the source.
Be exhaustive but concise. Quote exactly where the source is explicit. If a section has no content in this source, write 'None specified' under it do not omit the section heading."""
def process_excel_file(file_path: str, file_id: str) -> Tuple[str, str]:
"""Extract + summarise an HP Source Messaging Excel.
Args:
file_path: Path to the .xlsx file on disk.
file_id: Stable identifier used for the output filename.
Returns:
Tuple of (summary_text, summary_path). Summary is written to
BRAND_GUIDELINES_DIR/{file_id}_summary.md.
Never raises. On Gemini failure, writes a degraded summary that
embeds the raw extraction so the reference asset stays usable.
"""
try:
raw_text = _extract_workbook_text(file_path)
except Exception as e:
print(f" Excel extraction failed for {file_id}: {type(e).__name__}: {e}")
summary = (
f"# {os.path.basename(file_path)} (degraded — extraction failed)\n\n"
f"openpyxl extraction failed: {type(e).__name__}: {e}\n"
)
raw_text = ''
else:
try:
summary = _summarise_with_gemini(raw_text, os.path.basename(file_path))
except Exception as e:
print(f" Gemini summarisation failed for {file_id}: {type(e).__name__}: {e}")
summary = (
f"# {os.path.basename(file_path)} (degraded — summary failed)\n\n"
f"Gemini summarisation failed: {type(e).__name__}: {e}\n\n"
f"## Raw extraction\n\n```\n{raw_text}\n```\n"
)
os.makedirs(BRAND_GUIDELINES_DIR, exist_ok=True)
summary_path = os.path.join(BRAND_GUIDELINES_DIR, f"{file_id}_summary.md")
with open(summary_path, 'w', encoding='utf-8') as f:
f.write(summary)
return summary, summary_path
def _extract_workbook_text(file_path: str) -> str:
"""Read every sheet, dump as 'Sheet: <name>\\n<tab-aligned rows>\\n\\n'.
Empty rows are skipped. Output is capped at _RAW_EXTRACTION_CAP chars;
when exceeded, a truncation marker is appended and the rest is dropped.
"""
wb = load_workbook(file_path, data_only=True, read_only=True)
try:
parts = []
total_chars = 0
for sheet in wb.worksheets:
header = f"Sheet: {sheet.title}\n"
parts.append(header)
total_chars += len(header)
for row in sheet.iter_rows(values_only=True):
if not any((c is not None and str(c).strip()) for c in row):
continue
line = '\t'.join(('' if c is None else str(c)) for c in row)
parts.append(line + '\n')
total_chars += len(line) + 1
if total_chars >= _RAW_EXTRACTION_CAP:
parts.append(
f"\n[truncated — exceeded {_RAW_EXTRACTION_CAP}-char cap]\n"
)
return ''.join(parts)
parts.append('\n')
total_chars += 1
return ''.join(parts)
finally:
wb.close()
def _summarise_with_gemini(raw_text: str, source_filename: str) -> str:
"""Send the extracted workbook text to Gemini 2.5 Pro for summarisation.
Mirrors pdf_processor.summarize_brand_guidelines: uses
google.generativeai directly with MODEL_VERSIONS.gemini_vision
(currently gemini-2.5-pro). Raises on any failure; the caller
converts failures into a degraded summary.
"""
import google.generativeai as genai
from llm_config import MODEL_VERSIONS
api_key = os.getenv("GOOGLE_API_KEY")
if not api_key:
raise RuntimeError("GOOGLE_API_KEY not configured")
genai.configure(api_key=api_key)
model = genai.GenerativeModel(MODEL_VERSIONS.gemini_vision)
prompt = (
f"{_SYSTEM_PROMPT}\n\n"
f"Source filename: {source_filename}\n\n"
f"Raw cell content:\n\n```\n{raw_text}\n```"
)
response = model.generate_content(prompt)
# Mirror pdf_processor's safety-block handling: surface a useful error.
if not getattr(response, 'parts', None):
feedback = getattr(response, 'prompt_feedback', 'No specific feedback provided.')
raise RuntimeError(
f"Gemini response blocked or empty. Feedback: {feedback}"
)
return response.text

View file

@ -0,0 +1,14 @@
{
"name": "HP Copy Review",
"description": "Marketing copy graded against canonical HP Source Messaging",
"mode": "asset",
"visibility": "client_specific",
"visible_to_clients": ["hp"],
"checks": {
"hp_copy_review": {
"weight": 10.0,
"llm": "Gemini",
"enabled": true
}
}
}

View file

@ -0,0 +1,179 @@
"""HP Copy Review — single-call LLM grader against canonical Source Messaging.
This check compares all visible copy on an HP marketing asset (claims,
headlines, body, disclaimers, footnotes, spec call-outs, brand mentions)
against the canonical Source Messaging summaries attached as reference
assets (.xlsx Markdown summary via excel_processor).
It returns a structured JSON object with a 0-10 score, a one-paragraph
summary, and a `findings` array (priority / category / quote / issue /
suggested_fix / source_reference). Empty findings on a clean asset is a
valid result (score 9-10). When no Source Messaging is attached, the
LLM is instructed to return score 0 with an explanatory message rather
than grade blind.
Reference assets and media-plan context (including `language`) are
injected by `process_single_check` in `api_server.py` this module
exposes only the static prompt template. A standalone `build_prompt()`
helper is provided for unit-style smoke tests and for any future caller
that wants to assemble the full prompt outside the production path.
"""
import os
import sys
from typing import Iterable, Mapping, Optional, Sequence, Tuple
# Add parent directory to path so we can import shared template
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from visual_qc_apps.flask_app_template import FlaskAppTemplate
# --- Canonical prompt template ------------------------------------------------
#
# The reference-asset summary block ("CANONICAL SOURCE MESSAGING") is
# prepended by `process_single_check` in `api_server.py` via
# `get_reference_asset_content()`. Likewise the media-plan context block
# ("=== MEDIA PLAN CONTEXT ===" with `- Language: <value>`) is appended
# by `process_single_check`. We embed instructions that *reference* both
# blocks so the LLM knows where to look.
HP_COPY_REVIEW_PROMPT = """You are a copy reviewer for HP marketing materials. Your job is to compare the marketing asset against the canonical Source Messaging that has been attached as a reference asset, and report every copy discrepancy as a structured finding.
WHAT YOU WILL BE GIVEN:
1. One or more canonical Source Messaging summaries, attached above as REFERENCE ASSET GUIDELINES. Each Source Messaging file (e.g. `messi_core.xlsx`, `messi_mainstream.xlsx`) has been pre-summarised into Markdown and is the single source of truth for product claims, KSPs, disclaimers, spec call-outs, variant naming, and approved tone.
2. A media-plan context block (appended below the prompt) which may include `- Language: <value>` and `- Country: <value>`. Treat the language value as the PRODUCT LANGUAGE the asset should be using (e.g. "UK English", "US English", "French (France)").
3. The marketing asset image itself.
WHAT TO DO:
For every claim, headline, body line, disclaimer, footnote, spec call-out, and brand mention visible on the asset, evaluate it against the canonical Source Messaging. Flag:
- Wording that disagrees with an approved KSP or claim.
- Missing or incorrect mandatory disclaimers / legal footnotes / asterisked notes.
- Spec call-outs that contradict the canonical spec (wrong number, wrong unit, wrong product variant).
- Variant / product-name errors (e.g. "OmniDesk Mini" vs "OmniDesk Mini Core").
- Tone / phrasing drift from the approved brand voice described in the source.
- Brand-name misuse (HP, sub-brand capitalisation, trademark glyph misuse).
- Language / locale mismatch against the media-plan PRODUCT LANGUAGE (e.g. "color" appearing in a UK English asset, or French copy on an asset specified as US English).
OUTPUT return ONE JSON object, and nothing else (no prose, no markdown fences outside the JSON code block). The shape:
```json
{
"score": <number 0-10>,
"summary": "<one-paragraph headline finding>",
"findings": [
{
"priority": "high" | "medium" | "low",
"category": "ksp" | "disclaimer" | "spec" | "variant" | "tone" | "brand-name" | "language" | "other",
"quote": "<exact quote from the asset>",
"issue": "<what's wrong>",
"suggested_fix": "<what it should say, citing the canonical source>",
"source_reference": "<where in the source messaging this comes from, e.g. file name + section heading>"
}
]
}
```
RULES:
- If no Source Messaging reference asset is attached (i.e. there is no "REFERENCE ASSET GUIDELINES" block above describing canonical HP messaging), return EXACTLY:
{"score": 0, "summary": "No HP Source Messaging reference was attached — cannot grade copy without a canonical source.", "findings": []}
Do not attempt to grade copy from prior knowledge.
- High-priority findings (factually-wrong claims, missing mandatory disclaimers, wrong product variant, wrong language) weight the score most heavily. A single high-priority finding should typically pull the score below 6.
- Medium-priority findings are wording drift that changes nuance but not meaning, or missing optional supporting copy.
- Low-priority findings are tone / style nits.
- An empty `findings` array is a valid and expected result for a clean asset in that case score 9 or 10 and write a short, positive summary.
- The `quote` field must be the EXACT visible text from the asset, including punctuation. If you can read it, quote it.
- `source_reference` should make it easy for a reviewer to verify the finding name the Source Messaging file and the section/heading you matched against.
- Return ONLY the JSON object inside a single ```json ... ``` code block. No surrounding prose, no explanations outside the JSON.
"""
def build_prompt(
reference_summaries: Optional[Sequence[Tuple[str, str]]] = None,
media_plan_row: Optional[Mapping[str, str]] = None,
base_prompt: str = HP_COPY_REVIEW_PROMPT,
) -> str:
"""Assemble a fully-rendered HP copy-review prompt for testing / inspection.
In production, `process_single_check` (api_server.py) does this
assembly itself: it prepends `get_reference_asset_content(...)` and
appends `build_media_plan_context(...)`. This helper mirrors that
flow so we can smoke-test the prompt assembly without running the
full server, and so callers that want to render the exact prompt
text for logging / debugging have a single entry point.
Args:
reference_summaries: List of (filename, markdown_summary) tuples,
one per attached Source Messaging .xlsx. Each summary is
already a Markdown string produced by `excel_processor`.
None or [] means "no canonical source attached" in that
case we still build the prompt but omit the canonical block,
and the LLM will fall back to the score-0 rule.
media_plan_row: Mapping with optional `language`, `country`,
`placement`, etc. Only `language` and `country` are
rendered into the prompt here; the production flow uses
`build_media_plan_context` and includes more fields.
base_prompt: Override for the canonical prompt template (used
in tests where we want to inject a shorter stub).
Returns:
The fully-assembled prompt string, with the canonical source
messaging block (if any) prepended, the media-plan language /
country line(s) appended, and the base template in between.
"""
parts = []
# 1. Canonical source messaging block — mirrors the shape of
# `get_reference_asset_content` so the LLM sees a consistent
# "REFERENCE ASSET GUIDELINES" heading whether it's running in
# production or via this helper.
if reference_summaries:
ref_lines = ["\n\n=== REFERENCE ASSET GUIDELINES ===",
"CANONICAL SOURCE MESSAGING:"]
for filename, summary in reference_summaries:
ref_lines.append(f"\n--- File: {filename} ---\n{summary}")
ref_lines.append("=== END REFERENCE ASSET GUIDELINES ===\n")
parts.append("\n".join(ref_lines))
# 2. The static prompt template itself.
parts.append(base_prompt)
# 3. Media-plan context (language / country). Production appends
# the full `build_media_plan_context` block; here we render just
# the language + country fields, which is what Step 5.6 asserts.
if media_plan_row:
mp_lines = ["\n=== MEDIA PLAN CONTEXT ==="]
if media_plan_row.get('language'):
mp_lines.append(f"- Language: {media_plan_row['language']}")
if media_plan_row.get('country'):
mp_lines.append(f"- Country: {media_plan_row['country']}")
mp_lines.append("=== END MEDIA PLAN CONTEXT ===")
parts.append("\n".join(mp_lines))
return "\n".join(parts)
class HpCopyReviewApp(FlaskAppTemplate):
"""HP Copy Review — single-call LLM copy grader against Source Messaging.
Subclasses `FlaskAppTemplate` so the check is auto-discovered by
`load_qc_apps()` in `api_server.py`. The class instance exposes
`self.prompt` (the canonical template plus the standard scoring
instructions appended by the template base class).
Reference asset summaries and media-plan context are injected at
runtime by `process_single_check` this class does NOT call Gemini
directly. Response parsing is handled by
`extract_json_from_response` / `extract_score_from_result` in
api_server.py, which will lift `score`, `summary`, and `findings`
out of the JSON code block returned by the LLM.
"""
def __init__(self):
super().__init__(__name__, HP_COPY_REVIEW_PROMPT)
# Allow running this check standalone for ad-hoc testing
if __name__ == "__main__":
app_instance = HpCopyReviewApp()
app_instance.run()

View file

@ -0,0 +1,786 @@
# HP Onboarding — Cycle 1 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` (recommended) or `superpowers:executing-plans` to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Implement the `hp_copy_review` check and its supporting infrastructure per `docs/superpowers/specs/2026-05-17-hp-cycle-1-onboarding-design.md`, replacing the deprecated `hp-copy` PHP/Make.com POC.
**Architecture:** New `excel_processor.py` mirrors `pdf_processor.py` to convert HP Source Messaging Excels into structured Markdown summaries at upload time. A single new `hp_copy_review` QC check assembles those summaries + media-plan language metadata + the asset image into one Gemini prompt and returns a structured findings list. HP gets a real client config entry, a dedicated profile, and routing for `.xlsx` uploads through the existing `/api/brand_guidelines` endpoint.
**Tech Stack:**
- openpyxl 3.x (existing dep, used by `media_plan_processor.py`)
- Gemini 2.5 Pro via `llm_config.py` (existing)
- Existing reference-asset / brand-guidelines flow
- Existing media-plan processor
- No new external dependencies
**Branch:** `feature/hp-cycle-1-onboarding` from `develop`.
**Testing posture:** This project does not use pytest. Verification matches `backend/scripts/test-system.sh`: `py_compile`, import checks, profile-load tests, and real-asset smoke runs on the dev server. Inline `python3 -c "..."` snippets stand in for unit tests where helpful.
---
## File Structure
**New files:**
- `backend/excel_processor.py` — Excel ingestion + Gemini summarisation
- `backend/profiles/hp_copy_review.json` — new profile
- `backend/visual_qc_apps/hp_copy_review/app.py` — new QC check
- `backend/visual_qc_apps/hp_copy_review/__init__.py` — empty module marker
**Modified files:**
- `backend/client_config.py` — HP entry promoted from placeholder
- `backend/api_server.py``.xlsx` dispatch on `/api/brand_guidelines` POST + findings-table rendering in both HTML generators
- `backend/media_plan_processor.py``language` column extraction + metadata surfacing
- `CLAUDE.md` — HP row updated from "_scope pending_" to the new doc reference (small)
**Test fixtures (placed manually on disk, not committed):**
- `backend/tests/fixtures/hp/messi_core_source_messaging.xlsx`
- `backend/tests/fixtures/hp/messi_mainstream_source_messaging.xlsx`
- `backend/tests/fixtures/hp/gaston_source_messaging.xlsx`
The user-provided originals live at `/Users/nickviljoen/Desktop/AI_QC_Bitbucket/hp/recieved_docs/excel/` — those get *copied* (not symlinked) into `backend/tests/fixtures/hp/` for repeatable local verification. The directory is gitignored.
---
### Task 1: Excel processor module
Implement `excel_processor.py` mirroring `pdf_processor.py`. This is the most foundational change and the largest single module of new code.
**Files:**
- Create: `backend/excel_processor.py`
- Create: `backend/tests/fixtures/hp/` (gitignored)
- Modify: `.gitignore` (add `backend/tests/fixtures/`)
- [ ] **Step 1.1: Set up the fixtures directory**
```bash
mkdir -p backend/tests/fixtures/hp
cp '/Users/nickviljoen/Desktop/AI_QC_Bitbucket/hp/recieved_docs/excel/26C2 Messi Core HP OmniDesk Mini Desktop PC Source Messaging 04-10 (1).xlsx' backend/tests/fixtures/hp/messi_core.xlsx
cp '/Users/nickviljoen/Desktop/AI_QC_Bitbucket/hp/recieved_docs/excel/26C2 Messi Mainstream HP OmniDesk Mini Desktop PC Source Messaging 04-10 (1).xlsx' backend/tests/fixtures/hp/messi_mainstream.xlsx
cp '/Users/nickviljoen/Desktop/AI_QC_Bitbucket/hp/recieved_docs/excel/HP AluminiumBook Source Messaging - Gaston 05-06.xlsx' backend/tests/fixtures/hp/gaston.xlsx
ls backend/tests/fixtures/hp/
```
Expected: three `.xlsx` files listed.
- [ ] **Step 1.2: Add gitignore rule for fixtures**
Add to `.gitignore` near the existing legacy-env block:
```
# Local test fixtures (real HP Source Messaging files; not for commit)
backend/tests/fixtures/
```
- [ ] **Step 1.3: Read `pdf_processor.py` as the pattern source**
```bash
wc -l backend/pdf_processor.py
```
Read the file end-to-end. Identify: public surface (`process_pdf_file`), helper for raw extraction, helper for LLM summarisation, file path conventions (`brand_guidelines/files/{file_id}_summary.txt`), error handling shape, retry pattern, return tuple `(summary_text, summary_path)`.
- [ ] **Step 1.4: Create `excel_processor.py` skeleton**
Create `backend/excel_processor.py` with:
```python
"""Excel reference-asset processor for HP Source Messaging files.
Mirrors pdf_processor.py: openpyxl extracts raw cell content from
every sheet, Gemini summarises the result into structured Markdown
under brand_guidelines/files/{file_id}_summary.md. The check
hp_copy_review pulls that Markdown into its prompt at QC time.
"""
import os
from typing import Tuple
from openpyxl import load_workbook
from llm_config import call_gemini_text # adjust to actual export name
BRAND_GUIDELINES_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'brand_guidelines', 'files'
)
# Cap raw extraction at ~50K chars to keep the summary prompt bounded.
# A 30-row, 12-column workbook is ~10-15K chars in practice; this leaves
# headroom for HP's larger source files without blowing the prompt budget.
_RAW_EXTRACTION_CAP = 50_000
def process_excel_file(file_path: str, file_id: str) -> Tuple[str, str]:
"""Extract + summarise an HP Source Messaging Excel.
Returns (summary_text, summary_path). Saves the summary as
{file_id}_summary.md under BRAND_GUIDELINES_DIR. Never raises —
on failure, writes a degraded summary containing the raw extraction
so the reference asset is still usable, and returns that.
"""
raw_text = _extract_workbook_text(file_path)
try:
summary = _summarise_with_gemini(raw_text, os.path.basename(file_path))
except Exception as e:
summary = (
f"# {os.path.basename(file_path)} (degraded — summary failed)\n\n"
f"Gemini summarisation failed: {type(e).__name__}: {e}\n\n"
f"## Raw extraction\n\n```\n{raw_text}\n```\n"
)
os.makedirs(BRAND_GUIDELINES_DIR, exist_ok=True)
summary_path = os.path.join(BRAND_GUIDELINES_DIR, f"{file_id}_summary.md")
with open(summary_path, 'w', encoding='utf-8') as f:
f.write(summary)
return summary, summary_path
```
- [ ] **Step 1.5: Implement `_extract_workbook_text`**
Append:
```python
def _extract_workbook_text(file_path: str) -> str:
"""Read every sheet, dump as 'Sheet: <name>\\n<tab-aligned rows>\\n\\n'."""
wb = load_workbook(file_path, data_only=True, read_only=True)
parts = []
total_chars = 0
for sheet in wb.worksheets:
parts.append(f"Sheet: {sheet.title}\n")
for row in sheet.iter_rows(values_only=True):
# Skip rows where every cell is None/empty
if not any((c is not None and str(c).strip()) for c in row):
continue
line = '\t'.join(('' if c is None else str(c)) for c in row)
parts.append(line + '\n')
total_chars += len(line) + 1
if total_chars >= _RAW_EXTRACTION_CAP:
parts.append(f"\n[truncated — exceeded {_RAW_EXTRACTION_CAP}-char cap]\n")
return ''.join(parts)
parts.append('\n')
wb.close()
return ''.join(parts)
```
- [ ] **Step 1.6: Implement `_summarise_with_gemini`**
Append:
```python
_SYSTEM_PROMPT = """You're processing an HP Source Messaging Excel into a structured Markdown reference. Output these sections exactly, in this order:
## Product / Variant
(brand, product line, variant if any — e.g. "HP OmniDesk Mini — Core")
## Key Selling Points (KSPs)
For each KSP: heading, value proposition, supporting body copy, message-length variants (ultra-short / short / medium / long if present in the source).
## Disclaimers / Footnotes
Numbered list, exact wording, what claim each footnote anchors to.
## Approved Brand and Product Names
Exact spellings, including trademark glyphs (™, ®, ©).
## Variant Notes / Watch-outs
Anything explicitly marked variant-specific (e.g. "Mainstream only", "Core only", "must not appear in entry tier").
## Verboten Phrasing
Any explicitly disallowed or deprecated phrasing called out in the source.
Be exhaustive but concise. Quote exactly where the source is explicit. If a section has no content in this source, write 'None specified' under it — do not omit the section heading."""
def _summarise_with_gemini(raw_text: str, source_filename: str) -> str:
user_prompt = (
f"Source filename: {source_filename}\n\n"
f"Raw cell content:\n\n```\n{raw_text}\n```"
)
# call_gemini_text is the existing text-only Gemini wrapper in llm_config.
# If the actual export name differs, adjust in Step 1.7 verification.
return call_gemini_text(
system_prompt=_SYSTEM_PROMPT,
user_prompt=user_prompt,
model='gemini-2.5-pro',
)
```
- [ ] **Step 1.7: Verify llm_config exports a usable text-only Gemini wrapper**
```bash
grep -nE "def (call_gemini|gemini_text|generate.*gemini)" backend/llm_config.py | head -20
```
If `call_gemini_text` doesn't exist under that name, find the closest analogue (look at how `pdf_processor.py` calls Gemini) and update the import + call site in `excel_processor.py` accordingly.
- [ ] **Step 1.8: Syntax + import verification**
```bash
cd backend && python3 -m py_compile excel_processor.py && python3 -c "import excel_processor; print('OK', excel_processor.BRAND_GUIDELINES_DIR)"
```
Expected: `OK <path>/brand_guidelines/files`
- [ ] **Step 1.9: Run the processor against the Messi-Core fixture**
```bash
cd backend && python3 -c "
import os, sys
sys.path.insert(0, '.')
from excel_processor import process_excel_file
summary, path = process_excel_file('tests/fixtures/hp/messi_core.xlsx', 'test-messi-core')
print('summary_path:', path)
print('summary_len:', len(summary))
print('first 800 chars:')
print(summary[:800])
"
```
Expected: summary is 15004000 chars, contains `## Key Selling Points`, `## Disclaimers`, `## Approved Brand and Product Names`, and at least one KSP-level content snippet referencing "OmniDesk" or "Mini".
- [ ] **Step 1.10: Commit Task 1**
```bash
git add backend/excel_processor.py .gitignore
git commit -m "feat(excel-processor): add openpyxl + Gemini summary pipeline for HP Source Messaging
Mirrors pdf_processor.py — public process_excel_file() reads any HP
Source Messaging Excel, extracts cells via openpyxl (skipping empty
rows, capped at 50K chars), and summarises into structured Markdown
via Gemini 2.5 Pro. Output saved as brand_guidelines/files/{file_id}_summary.md.
On Gemini failure the processor writes a degraded summary containing
the raw extraction so the reference asset stays usable. Test fixtures
(real HP Excels) live under backend/tests/fixtures/hp/ and are gitignored."
```
---
### Task 2: `.xlsx` dispatch on the reference asset upload endpoint
Wire `excel_processor.process_excel_file` into the `/api/brand_guidelines` POST handler at `backend/api_server.py:4771` so `.xlsx` uploads route correctly.
**Files:**
- Modify: `backend/api_server.py` (around the existing `/api/brand_guidelines` POST handler near line 4771)
- [ ] **Step 2.1: Read the existing handler to find the PDF dispatch**
```bash
sed -n '4760,4900p' backend/api_server.py
```
Identify: where the extension is checked, where `pdf_processor.process_pdf_file` is called, and what's returned to the client.
- [ ] **Step 2.2: Add the `.xlsx` branch**
Edit the POST handler to dispatch by extension. The exact change depends on the existing code shape — pattern is:
- Where the handler currently checks for `.pdf` and calls `pdf_processor.process_pdf_file(...)`, add an `elif filename.lower().endswith('.xlsx')` branch that imports `excel_processor` and calls `excel_processor.process_excel_file(...)` with the same arg signature.
- The DB record / response shape should be identical to the PDF path — same `file_id`, same `status`, same return JSON.
- Cover image: PDF has one; Excel doesn't. If the DB record assigns a `cover_path`, set it to `None` for Excels.
- [ ] **Step 2.3: Syntax + import verification**
```bash
cd backend && python3 -m py_compile api_server.py && python3 -c "import api_server; print('api_server OK')"
```
- [ ] **Step 2.4: Commit Task 2**
```bash
git add backend/api_server.py
git commit -m "feat(brand-guidelines): route .xlsx uploads to excel_processor
The /api/brand_guidelines POST handler now dispatches by extension:
.pdf → pdf_processor.process_pdf_file (existing), .xlsx →
excel_processor.process_excel_file (new). Same DB record shape;
cover image is null for Excel since there's no first-page analogue."
```
---
### Task 3: Media plan `language` column
Add `language` to the media-plan column extraction and surface it into the prompt context.
**Files:**
- Modify: `backend/media_plan_processor.py`
- [ ] **Step 3.1: Locate the column-extraction logic**
```bash
grep -n -E "country|placement|vendor|dimensions" backend/media_plan_processor.py | head -10
```
These are the existing matched-row metadata fields. The `language` field will live alongside them.
- [ ] **Step 3.2: Add `language` to the case-insensitive header match list**
Edit the column-mapping section to recognise `Language` / `language` / `LANGUAGE` headers and store the value in the matched-row dict under the key `language`.
- [ ] **Step 3.3: Surface `language` in the prompt context block**
Locate where the matched-row dict is rendered as text injected into check prompts (the function that returns the "media plan context" string used by `process_single_check`). Add a line:
```python
if row.get('language'):
lines.append(f"Language: {row['language']}")
```
— preserving the existing structure (no line if absent).
- [ ] **Step 3.4: Syntax + import verification**
```bash
cd backend && python3 -m py_compile media_plan_processor.py && python3 -c "import media_plan_processor; print('OK')"
```
- [ ] **Step 3.5: Quick functional test with a synthetic plan**
```bash
cd backend && python3 -c "
# Mock test: build a minimal row dict with a language field and confirm the
# prompt-context formatter emits 'Language: <value>'. Exact function name to
# locate during Step 3.3 — adjust below.
from media_plan_processor import format_matched_row_for_prompt # adjust if named differently
row = {'country': 'UK', 'language': 'UK English', 'placement': 'eTail tile'}
print(format_matched_row_for_prompt(row))
"
```
Expected: output includes a line `Language: UK English`.
- [ ] **Step 3.6: Commit Task 3**
```bash
git add backend/media_plan_processor.py
git commit -m "feat(media-plan): extract and surface 'language' column
Adds case-insensitive 'language' header recognition to the media-plan
column mapper. When present in a matched row, the value flows into
the prompt context block as 'Language: <value>'. Absent → no line
(graceful no-op for clients whose plans don't include the field).
Enables multilingual support for hp_copy_review (Cycle 1) and any
future check that wants to reason about asset language."
```
---
### Task 4: HP client config + profile
Promote HP from placeholder. Create the `hp_copy_review` profile JSON. Ensure the profile loader picks it up.
**Files:**
- Modify: `backend/client_config.py`
- Create: `backend/profiles/hp_copy_review.json`
- [ ] **Step 4.1: Update the HP entry in `CLIENT_PROFILES`**
Edit `backend/client_config.py`. Replace the existing `'hp'` entry with:
```python
'hp': {
'name': 'HP',
'profiles': ['hp_copy_review', 'static_general', 'video_general'],
'display_name': 'HP',
'description': 'HP marketing copy QC graded against canonical Source Messaging',
'default_profile': 'hp_copy_review',
},
```
- [ ] **Step 4.2: Create the profile JSON**
Create `backend/profiles/hp_copy_review.json`:
```json
{
"name": "HP Copy Review",
"description": "Marketing copy graded against canonical HP Source Messaging",
"mode": "asset",
"visibility": "client_specific",
"visible_to_clients": ["hp"],
"checks": {
"hp_copy_review": {
"weight": 10.0,
"llm": "gemini",
"enabled": true
}
}
}
```
- [ ] **Step 4.3: Verify client config**
```bash
cd backend && python3 -c "
from client_config import get_client_profiles, get_default_profile
print('profiles:', get_client_profiles('hp'))
print('default:', get_default_profile('hp'))
"
```
Expected:
```
profiles: ['hp_copy_review', 'static_general', 'video_general']
default: hp_copy_review
```
- [ ] **Step 4.4: Verify profile load**
```bash
cd backend && python3 -c "
from profile_config import get_profile
p = get_profile('hp_copy_review')
print('name:', p.name)
print('mode:', getattr(p, 'mode', 'asset'))
print('enabled checks:', p.get_enabled_checks())
print('strict_grade:', getattr(p, 'strict_grade', False))
"
```
Expected: profile loads, mode is `asset`, enabled_checks lists `['hp_copy_review']`. (The check itself doesn't exist yet → may emit a "Loaded profile" line but the check loader fails for `hp_copy_review`; that's expected at this task boundary.)
- [ ] **Step 4.5: Commit Task 4**
```bash
git add backend/client_config.py backend/profiles/hp_copy_review.json
git commit -m "feat(hp): promote HP client + add hp_copy_review profile
HP is no longer a placeholder. The client gets a new hp_copy_review
profile (single weighted check, client-specific visibility) as its
default, plus the generic static_general and video_general profiles
it already had visibility into."
```
---
### Task 5: `hp_copy_review` check module
The actual QC check — single LLM call per asset.
**Files:**
- Create: `backend/visual_qc_apps/hp_copy_review/__init__.py` (empty)
- Create: `backend/visual_qc_apps/hp_copy_review/app.py`
- [ ] **Step 5.1: Read `flask_app_template.py` and a comparable real check**
```bash
ls backend/flask_app_template.py 2>/dev/null && wc -l backend/flask_app_template.py
ls backend/visual_qc_apps/boots_tandc_wording/app.py && wc -l backend/visual_qc_apps/boots_tandc_wording/app.py
```
Read both. The boots_tandc_wording check is the closest analogue (copy-against-reference, image input, structured findings output). Use it as the implementation pattern.
- [ ] **Step 5.2: Create the directory + empty `__init__.py`**
```bash
mkdir -p backend/visual_qc_apps/hp_copy_review
touch backend/visual_qc_apps/hp_copy_review/__init__.py
```
- [ ] **Step 5.3: Create `app.py` with the standard check skeleton**
Copy the structure from `boots_tandc_wording/app.py` (Flask blueprint pattern, `run_check(...)` or equivalent entry point, the reference-asset summary injection, the media-plan context injection). Adapt the prompt to:
```
You are a copy reviewer for HP marketing materials. Compare the
marketing asset against the canonical Source Messaging provided.
PRODUCT LANGUAGE: <from media plan, or "not specified">
CANONICAL SOURCE MESSAGING:
<one or more Markdown summaries from attached Excel reference assets,
concatenated, each preceded by a header like "--- File: messi_core.xlsx ---">
MARKETING ASSET:
[image]
For every claim, headline, body line, disclaimer, footnote, spec
call-out, and brand mention visible on the asset, evaluate against
the canonical source. Output a JSON object with this shape:
{
"score": <number 0-10>,
"summary": "<one-paragraph headline finding>",
"findings": [
{
"priority": "high" | "medium" | "low",
"category": "ksp" | "disclaimer" | "spec" | "variant" | "tone" | "brand-name" | "language" | "other",
"quote": "<exact quote from the asset>",
"issue": "<what's wrong>",
"suggested_fix": "<what it should say, citing the canonical source>",
"source_reference": "<where in source messaging this comes from>"
}
]
}
Rules:
- If no Source Messaging is attached, return {"score": 0, "summary": "No HP Source Messaging reference was attached — cannot grade copy without a canonical source.", "findings": []}
- High-priority findings weight the score most heavily
- Empty findings (clean asset) is a valid result; score 9-10
- Return ONLY the JSON object, no surrounding prose
```
- [ ] **Step 5.4: Implement response parsing**
The check function must parse the LLM's JSON response. Handle:
- Valid JSON with the expected shape → extract `score`, `summary`, `findings` and return them in the standard check result shape (`{'score': ..., 'response': ..., 'findings': ...}` — match the existing checks' return shape so the report renderer can pick up `findings` later).
- Malformed JSON → score 0, response = raw LLM text, findings = `[]`, summary = "Failed to parse check output".
- The `findings` array gets attached to the check result dict so the report renderer in Task 6 can detect it.
- [ ] **Step 5.5: Syntax + import + profile load verification**
```bash
cd backend && python3 -m py_compile visual_qc_apps/hp_copy_review/app.py && python3 -c "
from profile_config import get_profile
from app_discovery import discover_qc_apps # or the actual loader path
apps = discover_qc_apps()
print('hp_copy_review in apps:', 'hp_copy_review' in apps)
p = get_profile('hp_copy_review')
print('profile enabled checks:', p.get_enabled_checks())
"
```
Expected: `hp_copy_review in apps: True`, profile lists it as enabled.
- [ ] **Step 5.6: Dry-run prompt-assembly test (no LLM call)**
```bash
cd backend && python3 -c "
# Smoke test: instantiate the check, call its prompt-assembly helper
# (without invoking Gemini) with mock reference summaries and a mock
# media-plan row including language='UK English'. Confirm output prompt
# contains 'Language: UK English', 'CANONICAL SOURCE MESSAGING', and
# the findings-format instructions.
from visual_qc_apps.hp_copy_review.app import build_prompt # adjust if named differently
prompt = build_prompt(
reference_summaries=[('messi_core.xlsx', '## Product\nHP OmniDesk Mini Core')],
media_plan_row={'language': 'UK English', 'country': 'UK'},
)
assert 'Language: UK English' in prompt, 'language missing from prompt'
assert 'CANONICAL SOURCE MESSAGING' in prompt
assert 'findings' in prompt
print('prompt assembly OK')
"
```
- [ ] **Step 5.7: Commit Task 5**
```bash
git add backend/visual_qc_apps/hp_copy_review/
git commit -m "feat(hp_copy_review): single-check LLM grader against Source Messaging
Single Gemini call per asset. Prompt assembles attached Source
Messaging summaries + media-plan language context + the asset image.
Returns structured JSON with score, summary, and a findings array
(priority, category, quote, issue, suggested fix, source reference).
Empty findings = clean asset; missing reference → score 0 with a
clear message rather than running blind."
```
---
### Task 6: Findings-table rendering in both HTML report generators
Both HTML generators need a small case to render `findings` as a table.
**Files:**
- Modify: `backend/api_server.py` (`generate_html_content` and `generate_comprehensive_html_report` — see [[feedback_multi_html_generators]])
- [ ] **Step 6.1: Locate both generators**
```bash
grep -n "def generate_html_content\|def generate_comprehensive_html_report" backend/api_server.py
```
Expected: two function definitions, both render check results to HTML.
- [ ] **Step 6.2: Identify where each renders a per-check response**
In each generator, find the section that renders the per-check `response` text (often inside an expandable `<details>` block). The new case goes *before* that fallback: if the check's result dict contains a `findings` array, render the table; else fall back to the text response.
- [ ] **Step 6.3: Implement a shared helper `_render_findings_table(findings)`**
Add near the existing CSS/render helpers in `api_server.py`:
```python
def _render_findings_table(findings):
"""Render an hp_copy_review-style findings array as an HTML table."""
if not findings:
return '<p class="muted">No findings — copy is clean.</p>'
rows = []
for f in findings:
priority = f.get('priority', 'low')
pri_class = {'high': 'score-bad', 'medium': 'score-ok', 'low': 'score-good'}.get(priority, 'muted')
rows.append(
f'<tr>'
f'<td><span class="score-pill {pri_class}">{priority.upper()}</span></td>'
f'<td><code>{f.get("category", "")}</code></td>'
f'<td><code>{(f.get("quote") or "")[:200]}</code></td>'
f'<td>{f.get("issue", "")}</td>'
f'<td>{f.get("suggested_fix", "")}</td>'
f'<td class="muted">{f.get("source_reference", "")}</td>'
f'</tr>'
)
return (
'<table class="findings-table"><thead><tr>'
'<th>Priority</th><th>Category</th><th>Quote</th>'
'<th>Issue</th><th>Suggested fix</th><th>Source</th>'
'</tr></thead><tbody>'
+ ''.join(rows) + '</tbody></table>'
)
```
- [ ] **Step 6.4: Wire the helper into both generators**
In each generator, where it renders a check's response block, add (in pseudocode):
```python
findings = check_result.get('findings')
if findings is not None:
body_html += _render_findings_table(findings)
else:
body_html += render_response_text(check_result.get('response', ''))
```
Match the exact variable names and HTML scaffolding used by each generator.
- [ ] **Step 6.5: Syntax verification + manual HTML inspection**
```bash
cd backend && python3 -m py_compile api_server.py && python3 -c "
from api_server import _render_findings_table
html = _render_findings_table([
{'priority': 'high', 'category': 'disclaimer', 'quote': 'must be linked to a boots.com account', 'issue': 'Wrong account type', 'suggested_fix': '...linked to an Advantage Card account...', 'source_reference': 'Messi Core T&Cs row 18'},
{'priority': 'low', 'category': 'tone', 'quote': 'a tiny powerhouse', 'issue': 'Not approved phrasing', 'suggested_fix': 'Use \"compact and capable\"', 'source_reference': 'KSP 1'},
])
with open('/tmp/findings_preview.html', 'w') as f:
f.write('<!DOCTYPE html><html><head><style>table{border-collapse:collapse}td,th{border:1px solid #ddd;padding:6px}</style></head><body>' + html + '</body></html>')
print('wrote /tmp/findings_preview.html')
"
open /tmp/findings_preview.html
```
Eye-check: table renders, priority pills coloured correctly, quote in monospace.
- [ ] **Step 6.6: Commit Task 6**
```bash
git add backend/api_server.py
git commit -m "feat(report): render hp_copy_review findings as a structured table
Both HTML report generators (generate_html_content and
generate_comprehensive_html_report) get a small case: when a check
result has a 'findings' array, render it as a priority-coloured
table with quote/issue/suggested-fix/source columns instead of the
default response-text block. Fallback to text rendering when
findings is absent — every existing check is unaffected."
```
---
### Task 7: Dev smoke test + deployment
End-to-end verification on the dev server with real assets and real LLM calls.
- [ ] **Step 7.1: Run the full pre-session checklist**
```bash
cd backend && python3 -c "
from profile_config import get_profile
for p in ['general_check','static_general','unilever_key_visual','unilever_packaging','diageo_key_visual','diageo_packaging','loreal_static','amazon_static','boots_static','boots_ppack','inclusive_accessibility','video_general','axa_policy_document','axa_policy_document_diff','axa_accessibility','hp_copy_review']:
prof = get_profile(p)
print(f'OK {prof.name} ({len(prof.get_enabled_checks())} checks)')
"
cd .. && python3 -m py_compile backend/**/*.py
python3 -c "
import sys; sys.path.insert(0, 'backend')
import api_server, llm_config, profile_config, jwt_validator, auth_middleware
print('all imports OK')
"
```
Expected: every profile (including new `hp_copy_review`) loads; all syntax + imports green.
- [ ] **Step 7.2: Push the feature branch**
```bash
git push -u origin feature/hp-cycle-1-onboarding
```
- [ ] **Step 7.3: Open PR `feature/hp-cycle-1-onboarding → develop` via Bitbucket**
URL: `https://bitbucket.org/zlalani/ai_qc/pull-requests/new?source=feature/hp-cycle-1-onboarding&t=1`. Destination = `develop`. Title: "feat(hp): cycle 1 — hp_copy_review check + excel processor + language field". Body links to the spec.
- [ ] **Step 7.4: Merge PR, then deploy to dev**
SSH to `optical-production-dev`:
```bash
cd /opt/ai_qc
backend/scripts/deploy.sh dev
sudo journalctl -u ai-qc -n 30 --no-pager
```
Confirm clean deploy + service healthy.
- [ ] **Step 7.5: Manually upload Source Messaging fixtures to dev**
Via the UI at `optical-dev.oliver.solutions/ai_qc/`:
1. Sign in (admin).
2. Settings → Reference Assets (for client `hp`).
3. Upload `messi_core.xlsx`, `messi_mainstream.xlsx`, `gaston.xlsx` (from the original locations under `~/Desktop/AI_QC_Bitbucket/hp/recieved_docs/excel/`).
4. Watch the status badge — each should flip to `ready` within 60s. If degraded, inspect the saved `_summary.md` to see what failed.
- [ ] **Step 7.6: Run an HP marketing asset through `hp_copy_review`**
1. From the HP team, get a real Messi or Gaston marketing image (PNG/JPG).
2. Open a QC session as client `hp`, profile `hp_copy_review`.
3. Attach the relevant Source Messaging reference (e.g. `messi_core` for a Core-targeted asset).
4. (Optional) Upload a media plan with a `language` column populated so the prompt picks it up.
5. Run the QC.
6. Inspect the report: confirm findings table renders, priority pills coloured correctly, quotes are real text from the asset.
If output structure is wrong (e.g. LLM returns prose instead of JSON), iterate the prompt — small follow-up PRs against `develop`.
- [ ] **Step 7.7: PR `develop → main` and tag**
Once HP-side smoke testing confirms the output is useful:
```bash
# (laptop) sync local develop, open PR via Bitbucket UI:
# https://bitbucket.org/zlalani/ai_qc/pull-requests/new?source=develop&dest=main&t=1
```
After merge:
```bash
git fetch origin
git tag -a v1.4.0 origin/main -m "v1.4.0 — HP onboarding cycle 1 (hp_copy_review check + excel processor + media-plan language field)"
git push origin v1.4.0
git rev-parse v1.4.0^{commit}; git rev-parse origin/main # should match
```
- [ ] **Step 7.8: Deploy v1.4.0 to prod**
SSH to `optical-production`:
```bash
cd /opt/ai_qc
backend/scripts/deploy.sh prod v1.4.0
sudo journalctl -u ai-qc -n 30 --no-pager
```
No env-file backup dance needed — env files are now permanently gitignored (since v1.3.2).
- [ ] **Step 7.9: Upload Source Messaging files to prod**
Repeat Step 7.5 against the prod UI (`optical-prod.oliver.solutions/ai_qc/`). Source Messaging files are *per-server* — they live in `brand_guidelines/files/` on disk and don't sync between dev and prod.
- [ ] **Step 7.10: Hand off to HP team**
Confirm HP has access (via per-user client access — `Nick.Viljoen@oliver.agency` adds the HP team's email(s)). Walk them through:
1. Where to upload Source Messaging files (Settings → Reference Assets).
2. How to run a QC (select hp_copy_review, attach the right reference).
3. What feedback to send back (findings missed, findings wrong, output format suggestions).
Collect first-week feedback before opening Cycle 2 (Word/PPT processor).

View file

@ -0,0 +1,280 @@
# HP Onboarding — Cycle 1: `hp_copy_review` Check
**Goal:** Onboard HP onto the AI QC platform with a Source-Messaging-grounded copy review check, replacing the existing `hp-copy` PHP/Make.com POC tool.
**Architecture:** Single new QC check `hp_copy_review` grades an HP marketing asset's on-asset copy against canonical Source Messaging Excel files uploaded as reference assets. A new `excel_processor.py` mirrors `pdf_processor.py`: openpyxl extracts raw cell content at upload time, Gemini summarises into structured Markdown, saved alongside the file under `brand_guidelines/files/`. At QC time the check prompt assembles the Markdown summary(s) + media-plan language metadata + the asset image and returns a structured findings list. HP gets a real client config entry plus the generic profiles it already has visibility into.
**Tech stack:** openpyxl 3.x (already a project dep — used by `media_plan_processor.py`), existing `llm_config.py` Gemini integration, existing brand-guidelines flow, existing media-plan processor. **No new external dependencies.**
**Status:** Cycle 1 of 3 in HP onboarding. Cycles 2 (Word/PPT ingestion) and 3 (Box file picker) are independent and ship later. This cycle is independently shippable.
---
## Context
HP's existing `hp-copy` is a PHP UI wrapping a Make.com webhook (opaque). The PM raised seven concerns; Dave's decision is to deprecate the POC and migrate HP onto AI QC. Of the seven concerns:
- **Solved natively by AI QC today:** stability, configurable rule sets, accuracy (LLM + reference assets eliminate the false-positives-on-brand-names class of bugs because the canonical source list comes from the Excels), bulk processing (local upload supports multi-file out of the box).
- **Cycle 1 (this spec) addresses:** the HP-specific check, the Source-Messaging Excel ingestion pipeline, and multilingual via a media-plan `language` field.
- **Other cycles:** Word/PPT support (Cycle 2), Box file picker (Cycle 3).
The user-visible flow Day 1 after this cycle ships:
1. HP user uploads Source Messaging `.xlsx` files (Messi-Core, Messi-Mainstream, Gaston) once via Settings → Reference Assets.
2. HP user uploads marketing asset(s) via local upload — same UX as Boots/AXA/LOREAL.
3. HP user selects the `hp_copy_review` profile and attaches the relevant Source Messaging reference(s).
4. The check returns a structured findings table matching the Messi Copy Review document format (priority, quote, issue, suggested fix, source citation).
## Scope
### In scope (this cycle)
1. **HP client config** promoted from `_scope pending_` to a real entry with `hp_copy_review` as the default profile.
2. **`hp_copy_review` profile JSON** — single weighted check, client-specific visibility.
3. **`hp_copy_review` QC check** at `backend/visual_qc_apps/hp_copy_review/app.py`.
4. **`backend/excel_processor.py`** — new module mirroring `pdf_processor.py`. openpyxl extraction → Gemini summary → Markdown saved as `{file_id}_summary.md`.
5. **Reference-asset upload routing**`.xlsx` uploads route to `excel_processor.process_excel_file`. Existing endpoints (`POST /api/brand_guidelines`, `GET /api/brand_guidelines/<id>/status`, `POST .../reprocess`) work without modification beyond the dispatch line.
6. **Media plan `language` field** — free-form text column; surfaced in matched-row metadata; included in the check prompt when present; absent → graceful no-op.
7. **Report rendering** — small case in the two HTML report generators so the findings JSON renders as a priority-coloured table instead of a wall of text.
8. **Unit + smoke tests** as listed under Testing.
### Out of scope (other cycles or deferred)
- Word / PPT ingestion as reference assets — Cycle 2.
- Box file picker UI — Cycle 3.
- HP master brand guidelines reference — HP hasn't provided one yet.
- Briefs (`.pptx`) as reference assets — depends on Cycle 2.
- Multi-language Source Messaging variants — HP currently has English-only files. If they later provide Spanish / Dutch versions, no code change is needed; they upload as separate reference assets.
- Strict-grade enforcement — the HP Copy Review is a nuanced priority-tiered (High / Medium / Low) review, not pass/fail. Standard 0100 weighted scoring.
- Replacing or modifying the existing `hp-copy` PHP tool. We leave it running; HP migrates traffic at their own pace.
---
## Components
### `backend/client_config.py` — HP entry
Promote HP from placeholder to a real entry. Add `hp_copy_review` to the profile list, set as default:
```python
'hp': {
'name': 'HP',
'profiles': ['hp_copy_review', 'static_general', 'video_general'],
'display_name': 'HP',
'description': 'HP marketing copy QC graded against canonical Source Messaging',
'default_profile': 'hp_copy_review',
},
```
`box_folder_id` / `box_reports_folder_id` deferred to Cycle 3.
### `backend/profiles/hp_copy_review.json` — new profile
```json
{
"name": "HP Copy Review",
"description": "Marketing copy graded against canonical HP Source Messaging",
"mode": "asset",
"visibility": "client_specific",
"visible_to_clients": ["hp"],
"checks": {
"hp_copy_review": {
"weight": 10.0,
"llm": "gemini",
"enabled": true
}
}
}
```
Total weight = 10.0 → scoring uses the `weighted_score × 10` path, max 100. Single check carries the whole score. No `strict_grade`.
### `backend/visual_qc_apps/hp_copy_review/app.py` — new check
Standard QC app module following `flask_app_template.py`. Single Gemini call. Returns: `score` (010), `summary` (one-paragraph headline), and `findings` (JSON list).
**Prompt structure** (starting point — expect tuning during smoke testing):
```
You are a copy reviewer for HP marketing materials. Compare the
marketing asset against the canonical Source Messaging provided.
PRODUCT LANGUAGE: <from media plan, or "not specified">
CANONICAL SOURCE MESSAGING:
<one or more Markdown summaries from attached Excel reference assets,
concatenated with a `---` separator and a file-name header>
MARKETING ASSET:
<image>
For every claim, headline, body line, disclaimer, footnote, spec
call-out, and brand mention visible on the asset, evaluate against
the canonical source. Output a structured findings array:
[
{
"priority": "high" | "medium" | "low",
"category": "ksp" | "disclaimer" | "spec" | "variant" |
"tone" | "brand-name" | "language" | "other",
"quote": "<exact quote from the asset>",
"issue": "<what's wrong>",
"suggested_fix": "<what it should say, citing the canonical source>",
"source_reference": "<where in source messaging this comes from,
e.g. 'Core sheet row 12 KSP 3'>"
},
...
]
Then provide a score from 010 reflecting overall copy quality
(10 = no issues, 0 = severe and pervasive issues). Score should
weight high-priority issues most heavily.
If no Source Messaging is attached, return score 0 with a clear
summary explaining that no canonical source was provided.
```
**Empty-findings case** (clean asset): valid result — score 910, `findings: []`, summary "no issues identified".
**No-reference-attached case**: check returns score 0 with the explanatory message, rather than running blind against an empty source.
### `backend/excel_processor.py` — new module
Mirrors `pdf_processor.py`. Public surface:
- `process_excel_file(file_path, file_id) -> tuple[str, str]` — reads `.xlsx`, returns `(summary_text, summary_path)`. Saves `{file_id}_summary.md` under `brand_guidelines/files/`.
Internal helpers:
- `_extract_workbook_text(path) -> str` — openpyxl, iterates all sheets, dumps as `"Sheet: <name>\n<row-by-row tab-aligned cell values>\n\n"`. Skips empty rows. Caps at a reasonable cell budget (e.g. 50K chars) to bound prompt size.
- `_summarise_with_gemini(raw_text, source_filename) -> str` — Gemini 2.5 Pro call with HP-tuned system prompt (below) producing a structured Markdown summary, ~15003000 words.
**Summary prompt** (Excel-specific):
```
You're processing an HP Source Messaging Excel into a structured
Markdown reference. Output these sections:
## Product / Variant
(brand, product line, variant if any — e.g. "HP OmniDesk Mini — Core")
## Key Selling Points (KSPs)
For each KSP: heading, value proposition, supporting body copy,
message-length variants (ultra-short / short / medium / long if
present in the source).
## Disclaimers / Footnotes
Numbered list, exact wording, what claim each footnote anchors to.
## Approved Brand and Product Names
Exact spellings, including trademark glyphs (™, ®, ©).
## Variant Notes / Watch-outs
Anything explicitly marked variant-specific (e.g. "Mainstream only",
"Core only", "must not appear in entry tier").
## Verboten Phrasing
Any explicitly disallowed or deprecated phrasing called out in the source.
Be exhaustive but concise. Quote exactly where the source is explicit.
```
No cover image (Excel has no analogous concept). The reference-asset DB record schema already permits a null `cover_path`.
### `backend/media_plan_processor.py``language` column
When parsing media-plan Excel sheets, extract `language` (case-insensitive header match: `language`, `Language`, `LANGUAGE`) into the matched-row metadata dict. The existing media-plan-context block injected into prompts gains a `Language: <value>` line when the field is present; if absent, the line is omitted entirely (graceful no-op for clients whose media plans don't include language).
### `api_server.py` — reference asset upload routing
Existing `/api/brand_guidelines` POST routes `.pdf``pdf_processor.process_pdf_file`. Extend the dispatch: `.xlsx``excel_processor.process_excel_file`. Reuse the existing DB-record shape and the existing `GET .../<id>/status` and `POST .../<id>/reprocess` endpoints unchanged — they're agnostic to processor type.
### Report rendering — findings table
Per the [[feedback_multi_html_generators]] memory, there are two HTML generators (`generate_html_content` and `generate_comprehensive_html_report`). Both need a small case for `hp_copy_review`: when the check response contains a `findings` array, render as a table with columns for **Priority** (red/amber/green pill), **Category** (pill), **Quote** (monospace), **Issue**, **Suggested fix**, **Source**. Falls back to the existing plain-text response renderer if `findings` is absent (e.g. malformed LLM response).
---
## Data Flow
**Reference asset upload (one-time per Source Messaging file):**
1. HP user uploads `.xlsx` via Settings → Reference Assets.
2. `api_server.py` routes by extension to `excel_processor.process_excel_file`.
3. openpyxl extracts raw cell content from all sheets.
4. Gemini summarises into structured Markdown via the HP-specific summary prompt.
5. Summary saved at `brand_guidelines/files/{file_id}_summary.md`.
6. DB record updated; status flips to `ready`.
**QC run (per analysis):**
1. HP user uploads marketing asset (image).
2. Selects `hp_copy_review` profile.
3. Selects one or more Source Messaging reference assets (Core / Mainstream / Gaston as applicable).
4. (Optional) The asset's filename matches a media plan row containing a `language` value.
5. `process_single_check` for `hp_copy_review` assembles the prompt: system instructions + concatenated Markdown summaries + media-plan context (with language if present) + asset image.
6. Single Gemini call returns score + summary + findings JSON.
7. Report renderer presents findings as a Messi-Review-style table.
---
## Error Handling
- **Excel parse failure** (corrupt file, password-protected, etc.) — processor returns an error; DB status = `failed`; user sees the error in the reference-assets list. No app crash.
- **Gemini summarisation failure at upload** — retry once with exponential backoff; if still failing, save the raw extraction as the summary and mark status = `degraded`. The check can still use a degraded summary (lower fidelity) rather than blocking.
- **Check-time LLM failure or malformed findings JSON** — existing `process_single_check` exception handling captures and records a score-0 result with the error in the response. Standard pattern, no new surface.
- **Empty findings** (clean asset) — valid result; score 910, `findings: []`, summary "no issues identified".
- **No reference asset attached** — check returns score 0 with a clear message ("No HP Source Messaging reference selected — attach a Source Messaging Excel to compare against"). Doesn't run blind.
- **Excel processing concurrency** — uploads are independent files; `pdf_processor.py` already handles concurrent uploads safely (per-file_id artefact paths). Same pattern applies.
---
## Testing
Tests run against the project's existing pytest setup. Real Source Messaging Excels live under `tests/fixtures/hp/` (copied from the user-provided originals).
- **Unit tests**`excel_processor`:
- Happy path: Messi-Core / Messi-Mainstream / Gaston Excels each yield a non-empty `.md` summary containing the expected section headers (`## Key Selling Points`, `## Disclaimers / Footnotes`, etc.) and at least one KSP-level content snippet.
- Corrupt file: error returned, no crash.
- Empty workbook: graceful degradation with a sensible message.
- **Unit tests**`hp_copy_review/app.py`:
- Prompt assembly: given mock reference summaries and a mock media-plan row with `language: "UK English"`, assert the assembled prompt contains the language line, the source-messaging block delimiter, and the findings-format instructions.
- Response parsing: given a known Gemini-shape JSON response (fixture), assert findings list extracted correctly with all six fields per finding.
- Empty references: score 0 + the explanatory message.
- **Integration smoke test**: end-to-end with a real Messi asset (sample PNG of an OmniDesk eTail tile) + the Messi-Core Source Messaging reference attached. Assert the check runs to completion, returns a valid score, returns at least one finding (the Messi Copy Review found 34 — Gemini should surface at least 3 in the deterministic ones).
- **Profile load** in the pre-session checklist: add `hp_copy_review` to the loader test.
---
## Deployment
Code-only changes — no infrastructure work, no requirements changes (openpyxl already installed).
1. PR `feature/hp-cycle-1-onboarding → develop`. Deploy to dev via `deploy.sh dev`.
2. **One-time data step on dev:** HP team (or Nick on their behalf) uploads the three Source Messaging Excel files (Messi-Core, Messi-Mainstream, Gaston-v2) via the UI. These land in `brand_guidelines/files/` on dev only — uploads are not synced between dev and prod; the prod uploads happen separately.
3. Dev smoke test: run an HP marketing image through `hp_copy_review` with the Messi-Core reference attached. Verify output structure mirrors the Messi Copy Review doc.
4. PR `develop → main`. Tag `v1.4.0` (minor — new client capability). Deploy to prod via `deploy.sh prod v1.4.0`.
5. HP team uploads Source Messaging files on prod, runs first real QC, provides feedback. Prompt tuning iterations are post-deploy LLM-prompt changes — small follow-up PRs as needed, no spec changes.
---
## Definition of Done
- `hp_copy_review` profile loads cleanly (pre-session checklist passes with the new profile in the loader script).
- `client_config.get_client_profiles('hp')` returns `['hp_copy_review', 'static_general', 'video_general']`.
- `client_config.get_default_profile('hp')` returns `'hp_copy_review'`.
- Uploading a Source Messaging `.xlsx` produces a non-empty `_summary.md` within 60s of upload.
- Running `hp_copy_review` on a known Messi asset with the Messi-Core reference attached returns findings overlapping with at least 3 of the 34 issues in the HP-provided Messi Copy Review doc (rough qualitative bar — Gemini scoring varies run-to-run, but the major issues should be detected).
- Report renders the findings as a structured table, not free-text.
- Media plan parsing extracts `language` when present; the check prompt includes a `Language:` line in that case.
- Standard pre-session checklist all green on develop tip.
---
## Deferred decisions (worth surfacing at follow-up)
- **Strict-grade for HP?** Not in V1. If HP wants any High-priority finding to force overall Fail, add `strict_grade: true` to the profile and extend the scoring path (small retrofit).
- **HP master brand guidelines** — none today. Whenever HP provides a master brand guide PDF (colour palette, logo usage, typography), it can be attached as an additional reference asset alongside Source Messaging. No code change.
- **Prompt template tuning** — the templates above are starting points. Live HP usage will surface what to refine. Iterate via small prompt-only PRs.
- **Non-English Source Messaging** — if HP later provides Spanish / Dutch versions, they upload as separate reference assets and select the relevant one(s) per QC run. Works without code change.
- **Findings-output schema versioning** — if HP wants additional fields per finding (e.g. screenshot crop region, suggested approval routing), add to the JSON shape and bump renderer.
- **Briefs as reference assets** — depends on Cycle 2 (Word/PPT ingestion). Once that lands, HP can attach Gaston/Messi `.pptx` briefs alongside the Excel sources.