{
  "single-agent": {
    "summary": {
      "method": "single-agent",
      "dataset": "MultiGov-30-v2",
      "n_questions": 10,
      "avg_em": 0.0,
      "avg_f1": 0.0,
      "avg_latency_ms": 7205.441279988736,
      "avg_cost_usd": 0.0012975500000000002,
      "avg_tokens": 12975.5,
      "avg_rounds": 1.0,
      "avg_nodes": 1.0,
      "total_cost_usd": 0.012975500000000001,
      "total_latency_s": 72.05441279988736,
      "total_shacl_pass": 0,
      "total_shacl_fail": 0,
      "total_constraint_propagations": 0,
      "total_observer_redirects": 0,
      "total_ontology_route_filtered": 0
    },
    "questions": [
      {
        "question_id": "MG-A01",
        "question": "What AI practices are completely prohibited under the EU AI Act, and what are the penalties?",
        "gold_answer": "Article 5 prohibits: subliminal manipulation, exploitation of vulnerabilities, social scoring by public authorities, real-time remote biometric identification in public spaces (with limited exceptions for law enforcement), untargeted scraping for facial recognition databases, emotion recognition in workplace and education, and biometric categorisation using sensitive characteristics. Under Article 99, violations carry fines up to EUR 35 million or 7% of worldwide annual turnover.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7150.851999875158,
        "cost_usd": 0.0012977000000000002,
        "total_tokens": 12977,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A02",
        "question": "What is the complete compliance chain a provider must follow to place a high-risk AI system on the EU market?",
        "gold_answer": "The provider must follow this compliance chain: (1) Implement risk management system (Art. 9); (2) Ensure data governance for training data (Art. 10); (3) Draw up technical documentation (Art. 11); (4) Ensure record-keeping and logging (Art. 12); (5) Provide transparency and instructions for use (Art. 13); (6) Design for human oversight (Art. 14); (7) Achieve accuracy, robustness and cybersecurity (Art. 15); (8) Establish quality management system (Art. 17); (9) Complete conformity assessment (Art. 43); (10) Affix CE marking (Art. 48); (11) Register in EU database (Art. 49). Only after all steps can the system be lawfully placed on the market.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7306.522499769926,
        "cost_usd": 0.0012981,
        "total_tokens": 12981,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A03",
        "question": "What are the obligations of deployers of high-risk AI systems regarding human oversight?",
        "gold_answer": "Under Article 26, deployers must assign human oversight to natural persons with necessary competence, training and authority. Under Article 14, the system must be designed to enable individuals to: fully understand system capacities and limitations, monitor operation, remain aware of automation bias, correctly interpret outputs, decide not to use the system, override or reverse outputs, and intervene through a stop button. For biometric identification (Annex III point 1(a)), at least two natural persons must verify and confirm any action.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7204.224800225347,
        "cost_usd": 0.0012974000000000002,
        "total_tokens": 12974,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A04",
        "question": "What specific requirements does the EU AI Act impose on general-purpose AI models with systemic risk?",
        "gold_answer": "Under Article 51, a GPAI model is classified as having systemic risk when training computation exceeds 10^25 FLOPs. Article 55 requires such providers to: perform model evaluations with standardised protocols including adversarial testing; assess and mitigate systemic risks at Union level; keep track of and report serious incidents to the AI Office; ensure adequate cybersecurity protection. Article 52 requires all GPAI providers to maintain technical documentation, provide information to downstream providers, and have a copyright policy.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7226.228599902242,
        "cost_usd": 0.0012977000000000002,
        "total_tokens": 12977,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A05",
        "question": "What are the GDPR principles for processing personal data and how do they apply to AI training?",
        "gold_answer": "GDPR Article 5 requires personal data to be processed with: lawfulness, fairness and transparency; purpose limitation (collected for specified purposes only); data minimisation (adequate, relevant, limited to necessary); accuracy (kept up to date); storage limitation (not kept longer than necessary); integrity and confidentiality (appropriate security). For AI training data under Article 10 of the AI Act, these principles mean training data must comply with data minimisation and purpose limitation when processing personal data.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7200.1249003224075,
        "cost_usd": 0.0012978,
        "total_tokens": 12978,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A06",
        "question": "What transparency obligations exist for AI systems that generate deepfakes or synthetic content?",
        "gold_answer": "Article 50 requires providers of AI systems generating synthetic audio, image, video or text content to ensure outputs are marked in a machine-readable format as artificially generated or manipulated. Deployers of AI systems generating deep fakes must disclose that content has been artificially generated or manipulated. For AI systems interacting directly with persons, providers must design them so persons are informed they are interacting with an AI, unless obvious from context. Exception: AI used to detect or prevent criminal offences.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7180.742399767041,
        "cost_usd": 0.0012974000000000002,
        "total_tokens": 12974,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A07",
        "question": "How does DORA define ICT risk management requirements for financial entities?",
        "gold_answer": "DORA Articles 5-16 require financial entities to have an internal governance and control framework ensuring effective and prudent management of ICT risk. The framework must include strategies, policies, procedures, ICT protocols and tools to protect all information assets and ICT assets including software, hardware and servers. Financial entities must identify, classify and adequately document all ICT-supported business functions.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7209.265400189906,
        "cost_usd": 0.0012972,
        "total_tokens": 12972,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A08",
        "question": "What incident notification timelines does NIS2 require for essential entities?",
        "gold_answer": "NIS2 Article 23 requires essential entities to notify CSIRT or competent authority: (a) within 24 hours \u2014 early warning indicating whether incident is suspected of being caused by unlawful or malicious acts; (b) within 72 hours \u2014 incident notification with assessment of severity and impact; (c) upon request \u2014 intermediate report; (d) within one month \u2014 final report with detailed description, root cause, and mitigation measures.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7195.15909999609,
        "cost_usd": 0.0012971000000000002,
        "total_tokens": 12971,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A09",
        "question": "What high-risk AI use cases does Annex III define for employment and workers management?",
        "gold_answer": "Annex III point 4 classifies as high-risk: AI systems for recruitment or selection of natural persons, particularly for placing targeted job advertisements, screening or filtering applications, evaluating candidates in interviews or tests. Also: AI for making decisions affecting terms of work-related relationships, promotion or termination, task allocation based on individual behaviour or personal traits, and monitoring and evaluating performance and behaviour of persons in work relationships.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7187.147899996489,
        "cost_usd": 0.0012975,
        "total_tokens": 12975,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A10",
        "question": "What is the role and scope of AI regulatory sandboxes under the EU AI Act?",
        "gold_answer": "Articles 57-58 require Member States to ensure at least one AI regulatory sandbox is operational at national level by August 2026. Sandboxes provide a controlled environment for development, testing and validation of innovative AI systems under strict oversight. They include real-world testing conditions. Personal data may be processed in sandboxes for developing AI in the public interest, with specific safeguards. SMEs and startups have priority access. Sandbox results are considered by market surveillance authorities.",
        "predicted_answer": "",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 7194.145199842751,
        "cost_usd": 0.0012976,
        "total_tokens": 12976,
        "convergence_rounds": 1,
        "active_nodes": 1,
        "method": "single-agent",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      }
    ]
  },
  "cognigraph-pcst-v2": {
    "summary": {
      "method": "cognigraph-pcst-v2",
      "dataset": "MultiGov-30-v2",
      "n_questions": 10,
      "avg_em": 0.0,
      "avg_f1": 0.0,
      "avg_latency_ms": 0.0,
      "avg_cost_usd": 0.0,
      "avg_tokens": 0.0,
      "avg_rounds": 0.0,
      "avg_nodes": 0.0,
      "total_cost_usd": 0.0,
      "total_latency_s": 0.0,
      "total_shacl_pass": 0,
      "total_shacl_fail": 0,
      "total_constraint_propagations": 0,
      "total_observer_redirects": 0,
      "total_ontology_route_filtered": 0
    },
    "questions": [
      {
        "question_id": "MG-A01",
        "question": "What AI practices are completely prohibited under the EU AI Act, and what are the penalties?",
        "gold_answer": "Article 5 prohibits: subliminal manipulation, exploitation of vulnerabilities, social scoring by public authorities, real-time remote biometric identification in public spaces (with limited exceptions for law enforcement), untargeted scraping for facial recognition databases, emotion recognition in workplace and education, and biometric categorisation using sensitive characteristics. Under Article 99, violations carry fines up to EUR 35 million or 7% of worldwide annual turnover.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A02",
        "question": "What is the complete compliance chain a provider must follow to place a high-risk AI system on the EU market?",
        "gold_answer": "The provider must follow this compliance chain: (1) Implement risk management system (Art. 9); (2) Ensure data governance for training data (Art. 10); (3) Draw up technical documentation (Art. 11); (4) Ensure record-keeping and logging (Art. 12); (5) Provide transparency and instructions for use (Art. 13); (6) Design for human oversight (Art. 14); (7) Achieve accuracy, robustness and cybersecurity (Art. 15); (8) Establish quality management system (Art. 17); (9) Complete conformity assessment (Art. 43); (10) Affix CE marking (Art. 48); (11) Register in EU database (Art. 49). Only after all steps can the system be lawfully placed on the market.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A03",
        "question": "What are the obligations of deployers of high-risk AI systems regarding human oversight?",
        "gold_answer": "Under Article 26, deployers must assign human oversight to natural persons with necessary competence, training and authority. Under Article 14, the system must be designed to enable individuals to: fully understand system capacities and limitations, monitor operation, remain aware of automation bias, correctly interpret outputs, decide not to use the system, override or reverse outputs, and intervene through a stop button. For biometric identification (Annex III point 1(a)), at least two natural persons must verify and confirm any action.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A04",
        "question": "What specific requirements does the EU AI Act impose on general-purpose AI models with systemic risk?",
        "gold_answer": "Under Article 51, a GPAI model is classified as having systemic risk when training computation exceeds 10^25 FLOPs. Article 55 requires such providers to: perform model evaluations with standardised protocols including adversarial testing; assess and mitigate systemic risks at Union level; keep track of and report serious incidents to the AI Office; ensure adequate cybersecurity protection. Article 52 requires all GPAI providers to maintain technical documentation, provide information to downstream providers, and have a copyright policy.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A05",
        "question": "What are the GDPR principles for processing personal data and how do they apply to AI training?",
        "gold_answer": "GDPR Article 5 requires personal data to be processed with: lawfulness, fairness and transparency; purpose limitation (collected for specified purposes only); data minimisation (adequate, relevant, limited to necessary); accuracy (kept up to date); storage limitation (not kept longer than necessary); integrity and confidentiality (appropriate security). For AI training data under Article 10 of the AI Act, these principles mean training data must comply with data minimisation and purpose limitation when processing personal data.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A06",
        "question": "What transparency obligations exist for AI systems that generate deepfakes or synthetic content?",
        "gold_answer": "Article 50 requires providers of AI systems generating synthetic audio, image, video or text content to ensure outputs are marked in a machine-readable format as artificially generated or manipulated. Deployers of AI systems generating deep fakes must disclose that content has been artificially generated or manipulated. For AI systems interacting directly with persons, providers must design them so persons are informed they are interacting with an AI, unless obvious from context. Exception: AI used to detect or prevent criminal offences.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A07",
        "question": "How does DORA define ICT risk management requirements for financial entities?",
        "gold_answer": "DORA Articles 5-16 require financial entities to have an internal governance and control framework ensuring effective and prudent management of ICT risk. The framework must include strategies, policies, procedures, ICT protocols and tools to protect all information assets and ICT assets including software, hardware and servers. Financial entities must identify, classify and adequately document all ICT-supported business functions.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A08",
        "question": "What incident notification timelines does NIS2 require for essential entities?",
        "gold_answer": "NIS2 Article 23 requires essential entities to notify CSIRT or competent authority: (a) within 24 hours \u2014 early warning indicating whether incident is suspected of being caused by unlawful or malicious acts; (b) within 72 hours \u2014 incident notification with assessment of severity and impact; (c) upon request \u2014 intermediate report; (d) within one month \u2014 final report with detailed description, root cause, and mitigation measures.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A09",
        "question": "What high-risk AI use cases does Annex III define for employment and workers management?",
        "gold_answer": "Annex III point 4 classifies as high-risk: AI systems for recruitment or selection of natural persons, particularly for placing targeted job advertisements, screening or filtering applications, evaluating candidates in interviews or tests. Also: AI for making decisions affecting terms of work-related relationships, promotion or termination, task allocation based on individual behaviour or personal traits, and monitoring and evaluating performance and behaviour of persons in work relationships.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      },
      {
        "question_id": "MG-A10",
        "question": "What is the role and scope of AI regulatory sandboxes under the EU AI Act?",
        "gold_answer": "Articles 57-58 require Member States to ensure at least one AI regulatory sandbox is operational at national level by August 2026. Sandboxes provide a controlled environment for development, testing and validation of innovative AI systems under strict oversight. They include real-world testing conditions. Personal data may be processed in sandboxes for developing AI in the public interest, with specific safeguards. SMEs and startups have priority access. Sandbox results are considered by market surveillance authorities.",
        "predicted_answer": "ERROR: [Errno 22] Invalid argument",
        "exact_match": 0.0,
        "f1": 0.0,
        "latency_ms": 0.0,
        "cost_usd": 0.0,
        "total_tokens": 0,
        "convergence_rounds": 0,
        "active_nodes": 0,
        "method": "cognigraph-pcst-v2",
        "shacl_pass": 0,
        "shacl_fail": 0,
        "constraint_propagations": 0,
        "observer_redirects": 0,
        "ontology_route_filtered": 0
      }
    ]
  }
}