Over 25 years of quality engineering teaches you one very harsh lesson: never trust a green checkmark.
With automated accessibility test frameworks like speca11y, axe-core, and pa11y across hundreds of target sites, there is a hidden trap that silently corrupts datasets. It isn’t a flaw in the accessibility rules themselves - it is how headless browsers handle the messy reality of the network.
If you feed an automated scanner a raw list of URLs without a strict “pre-flight” gatekeeper, you are almost certainly collecting ghost data.
The Redirect Mirage#
Imagine you feed a scanner http://company.com. The server responds with a 301 redirect to https://www.company.com/en-uk/.
Many automated tools will execute against the first response they receive. The result? The scanner analyzes a blank HTML document containing nothing but a meta-refresh tag, proudly reporting “0 Accessibility Errors Found!”
You haven’t validated a web application; you just certified a redirect header as WCAG compliant.
The Zombie 404s and Parked Pages#
What happens when a URL points to a dead project or a parked domain?
The headless browser faithfully loads the registrar’s placeholder page. Because domain registrars use incredibly simple, lightweight HTML for parking pages, the accessibility scanner often passes them with flying colors. Your analytics dashboard is now polluted with perfect scores from domains that do not even have a website.
The Pre-Flight Solution#
Heavy automated testing requires a strict network gatekeeper. Before firing up memory-heavy headless browsers, a lightweight Node.js pipeline must validate the routing:
Resolve the Final Destination: Intercept network requests and follow all redirects to ensure the scanner only receives the final, live DOM URL.
Scrub the Zombies: Parse the initial text payload for known parking keywords (e.g., “domain for sale”, “default nginx page”) and immediately drop them from the queue.
Isolate the Failures: Route 404s and unreachable servers into a diagnostic log, keeping the primary testing queue completely pure.
We cannot automate digital accessibility effectively if our tools are spending compute cycles auditing empty rooms.
Pre-flight check#
# --- PRE-FLIGHT CHECKER CONFIGURATION ---
PREFLIGHT_KEYWORDS = {
"parked": [
'domain for sale', 'buy this domain', 'this domain is registered',
'domain name is reserved', 'domeinnaam gereserveerd', 'is for sale', 'parked free',
'welcome to nginx', 'default server page', 'apache2 ubuntu default page',
'cgi-sys/defaultwebpage.cgi', 'this account has been suspended'
],
"waf_challenge": [
'attention required! | cloudflare',
'please wait while your request is being verified',
'enable javascript and cookies to continue',
'checking your browser before accessing'
],
"soft_404": [
'<title>404', '<title>page not found', '<title>not found'
]
}
async def check_url(url: str, client: httpx.AsyncClient, semaphore: asyncio.Semaphore) -> tuple[str, str | None]:
if not url.startswith(("http://", "https://")):
url = f"https://{url}"
async with semaphore:
try:
response = await client.get(
url,
follow_redirects=True,
timeout=10.0,
headers={
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
)
response.raise_for_status()
content_type = response.headers.get("content-type", "").lower()
if content_type and "text/html" not in content_type and "text/plain" not in content_type:
return url, f"Invalid Content-Type: {content_type}"
text = response.text.lower()
for kw in PREFLIGHT_KEYWORDS["parked"]:
if kw in text: return url, "Parked, Default Page, or Suspended"
for kw in PREFLIGHT_KEYWORDS["waf_challenge"]:
if kw in text: return url, "Cloudflare/WAF Challenge Page"
for kw in PREFLIGHT_KEYWORDS["soft_404"]:
if kw in text: return url, "Soft 404 (Page Not Found)"
final_url = str(response.url)
meta_match = re.search(r'<meta[^>]+http-equiv=["\']?refresh["\']?[^>]+content=["\']?\d+;\s*url=([^"\'>]+)["\']?', text, re.IGNORECASE)
if meta_match:
refresh_url = meta_match.group(1).replace("&", "&")
final_url = str(response.url.join(refresh_url))
return final_url, None
except httpx.TimeoutException:
return url, "Connection Timed Out (10s)"
except httpx.RequestError as e:
return url, f"Network Error: {type(e).__name__}"
except httpx.HTTPStatusError as e:
return url, f"HTTP {e.response.status_code}"
except Exception as e:
return url, f"Error: {str(e)}"
async def sanitize_urls(urls: list[str]) -> tuple[list[str], list[str]]:
valid_urls = []
failed_logs = []
# Limit concurrency to 15 to prevent exhausting network sockets
semaphore = asyncio.Semaphore(15)
# verify=False ensures we still evaluate sites with self-signed test certs
async with httpx.AsyncClient(verify=False) as client:
tasks = [check_url(url, client, semaphore) for url in urls]
results = await asyncio.gather(*tasks)
for final_url, error in results:
if error:
failed_logs.append(f"{final_url} - {error}")
else:
valid_urls.append(final_url)
# Deduplicate while preserving order
unique_valid = list(dict.fromkeys(valid_urls))
return unique_valid, failed_logs