{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://spectrai-initiative.github.io/bio-benchmark-atlas/schema/registry.schema.json", "title": "BioBench Atlas Registry Entity", "oneOf": [ {"$ref": "#/$defs/Benchmark"}, {"$ref": "#/$defs/Work"}, {"$ref": "#/$defs/Model"}, {"$ref": "#/$defs/EvaluationRun"}, {"$ref": "#/$defs/BenchmarkUse"} ], "$defs": { "Id": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}, "Verification": { "type": "object", "additionalProperties": false, "required": ["status", "last_verified"], "properties": { "status": {"enum": ["draft", "verified", "deprecated"]}, "last_verified": {"type": "string", "format": "date"}, "notes": {"type": ["string", "null"]} } }, "Confidence": {"enum": ["high", "medium", "low"]}, "Locator": { "type": "object", "additionalProperties": false, "required": ["type", "value"], "properties": { "type": {"enum": ["page", "section", "figure", "table", "repository-path", "release", "dataset-card", "web-anchor", "other"]}, "value": {"type": "string", "minLength": 1}, "note": {"type": ["string", "null"]}, "document_page": {"type": ["integer", "null"], "minimum": 1}, "printed_page": {"type": ["string", "null"]}, "source_fragment_sha256": {"type": ["string", "null"], "pattern": "^[a-f0-9]{64}$"} } }, "Evidence": { "type": "object", "additionalProperties": false, "required": ["locator", "supports"], "properties": { "work_id": {"$ref": "#/$defs/Id"}, "id": {"$ref": "#/$defs/Id"}, "source_type": {"enum": ["work", "resource"]}, "source_id": {"$ref": "#/$defs/Id"}, "accessed_date": {"type": "string", "format": "date"}, "locator": {"anyOf": [{"type": "string", "minLength": 1}, {"$ref": "#/$defs/Locator"}]}, "supports": {"type": "array", "minItems": 1, "items": {"type": "string"}} }, "anyOf": [ {"required": ["work_id"]}, {"required": ["id", "source_type", "source_id", "accessed_date"]} ] }, "ResourcePin": { "type": "object", "additionalProperties": false, "required": ["kind", "value"], "properties": { "kind": {"enum": ["commit", "tag", "release", "version", "snapshot"]}, "value": {"type": "string", "minLength": 1}, "url": {"type": ["string", "null"], "format": "uri"} } }, "Resource": { "type": "object", "additionalProperties": false, "required": ["type", "url"], "properties": { "id": {"$ref": "#/$defs/Id"}, "type": {"enum": ["paper", "website", "repository", "dataset", "grader", "leaderboard", "documentation"]}, "url": {"type": "string", "format": "uri"}, "license": {"type": ["string", "null"]}, "access_notes": {"type": ["string", "null"]}, "last_checked": {"type": "string", "format": "date"}, "pin": {"anyOf": [{"$ref": "#/$defs/ResourcePin"}, {"type": "null"}]} } }, "ReportedValue": { "type": "object", "additionalProperties": false, "required": ["value", "reporting_status"], "properties": { "value": {}, "reporting_status": {"enum": ["reported", "not_reported", "not_applicable"]}, "notes": {"type": ["string", "null"]} } }, "TaskCounts": { "type": "object", "additionalProperties": false, "required": ["total", "basis", "subsets"], "properties": { "total": {"type": ["integer", "null"], "minimum": 0}, "basis": {"type": "string"}, "reporting_status": {"enum": ["reported", "not_reported"]}, "subsets": {"type": "array", "items": { "type": "object", "additionalProperties": false, "required": ["id", "label", "count", "basis", "exclusive", "exhaustive", "reporting_status"], "properties": { "id": {"$ref": "#/$defs/Id"}, "label": {"type": "string"}, "count": {"type": ["integer", "null"], "minimum": 0}, "basis": {"type": "string"}, "exclusive": {"type": "boolean"}, "exhaustive": {"type": "boolean"}, "partition_group": {"anyOf": [{"$ref": "#/$defs/Id"}, {"type": "null"}]}, "reporting_status": {"enum": ["reported", "not_reported"]}, "notes": {"type": ["string", "null"]} } }} } }, "BenchmarkVersion": { "type": "object", "additionalProperties": false, "required": ["id", "label", "status", "release_date", "as_of", "task_counts", "formal_tracks", "notes", "evidence_ids"], "properties": { "id": {"$ref": "#/$defs/Id"}, "label": {"type": "string", "minLength": 1}, "status": {"enum": ["current", "active", "superseded", "rolling"]}, "release_date": {"type": ["string", "null"], "format": "date"}, "as_of": {"type": ["string", "null"], "format": "date"}, "task_counts": {"$ref": "#/$defs/TaskCounts"}, "formal_tracks": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "notes": {"type": ["string", "null"]}, "evidence_ids": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}} } }, "Audit": { "type": "object", "additionalProperties": false, "required": ["status", "audited_date", "unresolved_fields"], "properties": { "status": {"enum": ["legacy", "audited", "audited-with-caveats"]}, "audited_date": {"type": ["string", "null"], "format": "date"}, "unresolved_fields": {"type": "integer", "minimum": 0}, "notes": {"type": ["string", "null"]} } }, "FieldStatus": { "type": "object", "additionalProperties": false, "required": ["path", "status", "confidence", "reason", "evidence_ids"], "properties": { "path": {"type": "string", "pattern": "^/"}, "status": {"enum": ["provisional", "conflicted"]}, "confidence": {"$ref": "#/$defs/Confidence"}, "reason": {"type": "string", "minLength": 1}, "evidence_ids": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}} } }, "ScientificTaskCoverage": { "type": "object", "additionalProperties": false, "required": ["task_type_id", "coverage", "mapping_method", "confidence", "count", "count_unit", "count_basis", "count_ref", "reporting_status", "evidence_ids", "notes"], "properties": { "task_type_id": {"$ref": "#/$defs/Id"}, "coverage": {"enum": ["explicitly-in-scope", "observed", "not-in-scope"]}, "mapping_method": {"enum": ["official-taxonomy", "official-track", "artifact-derived"]}, "confidence": {"$ref": "#/$defs/Confidence"}, "count": {"type": ["integer", "null"], "minimum": 0}, "count_unit": {"enum": ["tasks", "questions", "examples", "assays", "targets", "systems", "problems", "records", "episodes", "tracks", "other"]}, "count_basis": {"type": "string", "minLength": 1}, "count_ref": {"type": ["string", "null"], "pattern": "^/"}, "reporting_status": {"enum": ["reported", "not_reported"]}, "evidence_ids": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "notes": {"type": ["string", "null"]} } }, "ScientificTaskClassification": { "type": "object", "additionalProperties": false, "required": ["status", "benchmark_version", "as_of", "notes", "entries"], "properties": { "status": {"enum": ["complete", "partial", "unclassified"]}, "benchmark_version": {"type": "string", "minLength": 1}, "as_of": {"type": ["string", "null"], "format": "date"}, "notes": {"type": ["string", "null"]}, "entries": {"type": "array", "items": {"$ref": "#/$defs/ScientificTaskCoverage"}} } }, "Benchmark": { "type": "object", "additionalProperties": false, "required": ["entity_type", "id", "name", "aliases", "summary", "kind", "parent_id", "organizations", "release_date", "latest_version", "domains", "capabilities", "modalities", "task_formats", "task_counts", "coverage_notes", "scientific_task_classification", "access", "resources", "implementations", "verification", "evidence"], "properties": { "entity_type": {"const": "benchmark"}, "id": {"$ref": "#/$defs/Id"}, "name": {"type": "string", "minLength": 1}, "aliases": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, "summary": {"type": "string", "minLength": 20}, "kind": {"enum": ["suite", "track", "dataset", "competition", "agentic-eval"]}, "parent_id": {"anyOf": [{"$ref": "#/$defs/Id"}, {"type": "null"}]}, "organizations": {"type": "array", "minItems": 1, "items": {"type": "string"}}, "release_date": {"type": "string", "format": "date"}, "latest_version": {"type": "string"}, "domains": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "capabilities": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "modalities": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "task_formats": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string"}}, "task_counts": {"$ref": "#/$defs/TaskCounts"}, "coverage_notes": {"type": "array", "items": { "type": "object", "additionalProperties": false, "required": ["tag", "coverage", "count", "reporting_status", "notes"], "properties": { "tag": {"$ref": "#/$defs/Id"}, "coverage": {"enum": ["explicitly-in-scope", "observed", "not-in-scope", "unknown"]}, "count": {"type": ["integer", "null"], "minimum": 0}, "reporting_status": {"enum": ["reported", "not_reported"]}, "notes": {"type": ["string", "null"]} } }}, "scientific_task_classification": {"$ref": "#/$defs/ScientificTaskClassification"}, "access": { "type": "object", "additionalProperties": false, "required": ["level", "tasks", "artifacts", "grader", "license", "biosafety_notes"], "properties": { "level": {"$ref": "#/$defs/Id"}, "tasks": {"type": "string"}, "artifacts": {"type": "string"}, "grader": {"type": "string"}, "license": {"type": ["string", "null"]}, "biosafety_notes": {"type": ["string", "null"]} } }, "resources": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/Resource"}}, "implementations": {"type": "array", "items": { "type": "object", "additionalProperties": false, "required": ["framework", "status", "url", "commit"], "properties": {"framework": {"type": "string"}, "status": {"enum": ["official", "community", "local-only", "not-available"]}, "url": {"type": ["string", "null"], "format": "uri"}, "commit": {"type": ["string", "null"]}, "notes": {"type": ["string", "null"]}} }}, "versions": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/BenchmarkVersion"}}, "audit": {"$ref": "#/$defs/Audit"}, "field_status": {"type": "array", "items": {"$ref": "#/$defs/FieldStatus"}}, "verification": {"$ref": "#/$defs/Verification"}, "evidence": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/Evidence"}} }, "allOf": [{ "if": {"properties": {"audit": {"properties": {"status": {"enum": ["audited", "audited-with-caveats"]}}}}, "required": ["audit"]}, "then": {"required": ["versions", "field_status"]} }] }, "Work": { "type": "object", "additionalProperties": false, "required": ["entity_type", "id", "title", "authors", "organizations", "work_type", "source_class", "publication_date", "canonical_url", "doi", "arxiv", "status", "source_versions", "current_version_id", "verification"], "properties": { "entity_type": {"const": "work"}, "id": {"$ref": "#/$defs/Id"}, "title": {"type": "string"}, "authors": {"type": "array", "items": {"type": "string"}}, "organizations": {"type": "array", "items": {"type": "string"}}, "work_type": {"enum": ["paper", "preprint", "system-card", "official-release", "technical-report", "competition-site"]}, "source_class": {"enum": ["benchmark_creator", "official_model_provider", "independent_reproduction"]}, "publication_date": {"type": "string", "format": "date"}, "canonical_url": {"type": "string", "format": "uri"}, "doi": {"type": ["string", "null"]}, "arxiv": {"type": ["string", "null"]}, "status": {"enum": ["published", "preprint", "living", "archived"]}, "source_versions": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/WorkSourceVersion"}}, "current_version_id": {"$ref": "#/$defs/Id"}, "review_provenance": {"$ref": "#/$defs/ReviewProvenance"}, "verification": {"$ref": "#/$defs/Verification"} } }, "WorkSourceVersion": { "type": "object", "additionalProperties": false, "required": ["id", "label", "status", "publication_date", "canonical_url", "doi", "arxiv"], "properties": { "id": {"$ref": "#/$defs/Id"}, "label": {"type": "string", "minLength": 1}, "status": {"enum": ["current", "superseded", "version-of-record"]}, "publication_date": {"type": "string", "format": "date"}, "canonical_url": {"type": "string", "format": "uri"}, "doi": {"type": ["string", "null"]}, "arxiv": {"type": ["string", "null"]}, "source_access": {"enum": ["open-url", "submitted-pdf", "metadata-only"]}, "content_sha256": {"type": ["string", "null"], "pattern": "^[a-f0-9]{64}$"}, "content_type": {"type": ["string", "null"]}, "retrieved_at": {"type": ["string", "null"], "format": "date-time"} } }, "ReviewProvenance": { "type": "object", "additionalProperties": false, "required": ["method", "pipeline_version", "prompt_version", "source_version_id", "extractor_model_requested", "extractor_model_resolved", "verifier_model_requested", "verifier_model_resolved", "generated_at"], "properties": { "method": {"enum": ["automated-double-pass", "local-codex-double-pass"]}, "execution_surface": {"enum": ["github-actions-responses-api", "local-codex-cli"]}, "pipeline_version": {"type": "string", "minLength": 1}, "prompt_version": {"type": "string", "minLength": 1}, "source_version_id": {"$ref": "#/$defs/Id"}, "extractor_model_requested": {"type": "string", "minLength": 1}, "extractor_model_resolved": {"type": ["string", "null"], "minLength": 1}, "verifier_model_requested": {"type": "string", "minLength": 1}, "verifier_model_resolved": {"type": ["string", "null"], "minLength": 1}, "model_resolution_status": {"enum": ["reported", "not-reported"]}, "codex_cli_version": {"type": "string", "minLength": 1}, "local_run_id": {"type": "string", "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"}, "generated_at": {"type": "string", "format": "date-time"} }, "allOf": [{ "if": {"properties": {"method": {"const": "local-codex-double-pass"}}, "required": ["method"]}, "then": { "required": ["execution_surface", "model_resolution_status", "codex_cli_version", "local_run_id"], "properties": {"execution_surface": {"const": "local-codex-cli"}} } }] }, "Model": { "type": "object", "additionalProperties": false, "required": ["entity_type", "id", "name", "provider", "release_date", "version_string", "version_status", "aliases", "verification"], "properties": { "entity_type": {"const": "model"}, "id": {"$ref": "#/$defs/Id"}, "name": {"type": "string"}, "provider": {"type": "string"}, "release_date": {"type": ["string", "null"], "format": "date"}, "version_string": {"type": ["string", "null"]}, "version_status": {"enum": ["reported", "not_reported"]}, "aliases": {"type": "array", "items": {"type": "string"}}, "verification": {"$ref": "#/$defs/Verification"} } }, "EvaluationRun": { "type": "object", "additionalProperties": false, "required": ["entity_type", "id", "work_id", "work_version_id", "benchmark_id", "benchmark_version", "scope", "protocol", "metrics", "results", "comparability_group", "verification", "evidence"], "properties": { "entity_type": {"const": "evaluation_run"}, "id": {"$ref": "#/$defs/Id"}, "work_id": {"$ref": "#/$defs/Id"}, "benchmark_id": {"$ref": "#/$defs/Id"}, "work_version_id": {"$ref": "#/$defs/Id"}, "benchmark_version": {"type": ["string", "null"]}, "model_ids": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "scope": { "type": "object", "additionalProperties": false, "required": ["type", "n", "subset_id", "filter", "selection", "reporting_status"], "properties": {"type": {"enum": ["full", "subset", "track", "unknown"]}, "n": {"type": ["integer", "null"], "minimum": 0}, "subset_id": {"anyOf": [{"$ref": "#/$defs/Id"}, {"type": "null"}]}, "filter": {"type": ["string", "null"]}, "selection": {"enum": ["formal-subset", "filtered", "sampled", "hand-selected", "truncated", null]}, "reporting_status": {"enum": ["reported", "not_reported"]}} }, "protocol": { "type": "object", "additionalProperties": false, "required": ["shots", "turns", "system_prompt_public", "reasoning", "tools", "token_budget", "time_budget", "temperature", "seed", "repeats", "grader", "statistical", "contamination"], "properties": { "shots": {"$ref": "#/$defs/ReportedValue"}, "turns": {"$ref": "#/$defs/ReportedValue"}, "system_prompt_public": {"$ref": "#/$defs/ReportedValue"}, "reasoning": {"$ref": "#/$defs/ReportedValue"}, "tools": { "type": "object", "additionalProperties": false, "required": ["browser", "internet", "databases", "code_execution", "container", "external_tools"], "properties": {"browser": {"$ref": "#/$defs/ReportedValue"}, "internet": {"$ref": "#/$defs/ReportedValue"}, "databases": {"$ref": "#/$defs/ReportedValue"}, "code_execution": {"$ref": "#/$defs/ReportedValue"}, "container": {"$ref": "#/$defs/ReportedValue"}, "external_tools": {"$ref": "#/$defs/ReportedValue"}} }, "token_budget": {"$ref": "#/$defs/ReportedValue"}, "time_budget": {"$ref": "#/$defs/ReportedValue"}, "temperature": {"$ref": "#/$defs/ReportedValue"}, "seed": {"$ref": "#/$defs/ReportedValue"}, "repeats": {"$ref": "#/$defs/ReportedValue"}, "grader": {"type": "object", "additionalProperties": false, "required": ["type", "model", "human_review", "reporting_status"], "properties": {"type": {"type": ["string", "null"]}, "model": {"type": ["string", "null"]}, "human_review": {"type": ["boolean", "null"]}, "reporting_status": {"enum": ["reported", "not_reported"]}}}, "statistical": {"$ref": "#/$defs/ReportedValue"}, "contamination": {"$ref": "#/$defs/ReportedValue"} } }, "metrics": {"type": "array", "minItems": 1, "items": { "type": "object", "additionalProperties": false, "required": ["metric_id", "source_label", "unit", "range", "higher_is_better", "aggregation", "pass_threshold", "tolerance", "kind", "baseline_model_id"], "properties": {"metric_id": {"$ref": "#/$defs/Id"}, "source_label": {"type": "string"}, "unit": {"type": "string"}, "range": {"type": ["array", "null"], "minItems": 2, "maxItems": 2, "items": {"type": "number"}}, "higher_is_better": {"type": "boolean"}, "aggregation": {"type": ["string", "null"]}, "pass_threshold": {"type": ["number", "null"]}, "tolerance": {"type": ["string", "null"]}, "kind": {"enum": ["absolute", "delta"]}, "baseline_model_id": {"anyOf": [{"$ref": "#/$defs/Id"}, {"type": "null"}]}} }}, "results": {"type": "array", "items": { "type": "object", "additionalProperties": false, "required": ["model_id", "metric_id", "value", "ci_low", "ci_high", "n", "notes"], "properties": {"model_id": {"$ref": "#/$defs/Id"}, "metric_id": {"$ref": "#/$defs/Id"}, "value": {"type": "number"}, "ci_low": {"type": ["number", "null"]}, "ci_high": {"type": ["number", "null"]}, "n": {"type": ["integer", "null"], "minimum": 0}, "notes": {"type": ["string", "null"]}, "status": {"enum": ["verified", "provisional", "conflicted"]}, "confidence": {"$ref": "#/$defs/Confidence"}, "evidence_ids": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}} }}, "comparability_group": {"$ref": "#/$defs/Id"}, "verification": {"$ref": "#/$defs/Verification"}, "evidence": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/Evidence"}} } }, "BenchmarkUseScope": { "type": "object", "additionalProperties": false, "required": ["type", "subset_kind", "n", "subset_id", "selection", "selection_method", "reporting_status"], "properties": { "type": {"enum": ["full", "subset", "track", "unknown"]}, "subset_kind": {"enum": ["formal-subset", "paper-specific", "not-reported", "not-applicable"]}, "n": {"type": ["integer", "null"], "minimum": 0}, "subset_id": {"anyOf": [{"$ref": "#/$defs/Id"}, {"type": "null"}]}, "selection": {"enum": ["formal-subset", "filtered", "sampled", "hand-selected", "truncated", null]}, "selection_method": {"type": ["string", "null"]}, "reporting_status": {"enum": ["reported", "not_reported", "not_applicable"]} } }, "BenchmarkUse": { "type": "object", "additionalProperties": false, "required": ["entity_type", "id", "work_id", "work_version_id", "benchmark_id", "benchmark_version", "relation_type", "status", "model_ids", "scope", "metric_labels", "evaluation_run_ids", "reporting_gaps", "notes", "verification", "evidence"], "properties": { "entity_type": {"const": "benchmark_use"}, "id": {"$ref": "#/$defs/Id"}, "work_id": {"$ref": "#/$defs/Id"}, "work_version_id": {"$ref": "#/$defs/Id"}, "benchmark_id": {"$ref": "#/$defs/Id"}, "benchmark_version": {"type": ["string", "null"]}, "relation_type": {"enum": ["benchmark-creation", "evaluation", "training", "fine-tuning", "validation", "model-selection", "external-result-summary"]}, "status": {"enum": ["normalized", "partial", "non-evaluation"]}, "model_ids": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "scope": {"$ref": "#/$defs/BenchmarkUseScope"}, "metric_labels": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, "evaluation_run_ids": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/Id"}}, "reporting_gaps": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, "notes": {"type": ["string", "null"]}, "verification": {"$ref": "#/$defs/Verification"}, "evidence": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/Evidence"}} } } } }