{
  "last_updated": "2026-08-21T00:00:00Z",
  "submissions": [
    {
      "model_name": "Claude 4 Sonnet",
      "model_organization": "Anthropic",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 41.4,
          "sec_pass_1": 5.9,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "Claude CLI",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "claude-4-sonnet_claude-cli_default_2025-12-29"
    },
    {
      "model_name": "Claude 4 Sonnet",
      "model_organization": "Anthropic",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 53.2,
          "sec_pass_1": 11.8,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "claude-4-sonnet_openhands_default_2025-12-29"
    },
    {
      "model_name": "Claude 4 Sonnet",
      "model_organization": "Anthropic",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 57,
          "sec_pass_1": 11.8,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "claude-4-sonnet_swe-agent_default_2025-12-29"
    },
    {
      "model_name": "Gemini 2.5 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 12.9,
          "sec_pass_1": 3.8,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "Claude CLI",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-2.5-pro_claude-cli_default_2025-12-29"
    },
    {
      "model_name": "Gemini 2.5 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 19.9,
          "sec_pass_1": 8.1,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-2.5-pro_openhands_default_2025-12-29"
    },
    {
      "model_name": "Gemini 2.5 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 18.3,
          "sec_pass_1": 6.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-2.5-pro_swe-agent_default_2025-12-29"
    },
    {
      "model_name": "Gemini 3 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 34.9,
          "sec_pass_1": 6.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "Claude CLI",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-3-pro_claude-cli_default_2025-12-29"
    },
    {
      "model_name": "Gemini 3 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 37.1,
          "sec_pass_1": 12.9,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-3-pro_swe-agent_default_2025-12-29"
    },
    {
      "model_name": "Gemini 3 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 50.5,
          "sec_pass_1": 10.8,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-3-pro_openhands_default_2025-12-29"
    },
    {
      "model_name": "Gemini 3 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 53,
          "sec_pass_1": 7.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v0.0",
        "agent_framework": "Gemini CLI",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-3-pro_gemini-cli_default_2025-12-29"
    },
    {
      "model_name": "Kimi K2",
      "model_organization": "Moonshot AI",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 40.3,
          "sec_pass_1": 8.6,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "Claude CLI",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "kimi-k2_claude-cli_default_2025-12-29"
    },
    {
      "model_name": "Kimi K2",
      "model_organization": "Moonshot AI",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 36,
          "sec_pass_1": 9.1,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "kimi-k2_openhands_default_2025-12-29"
    },
    {
      "model_name": "Kimi K2",
      "model_organization": "Moonshot AI",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2025-12-29",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": false,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 21.5,
          "sec_pass_1": 5.4,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2025-12-29",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "kimi-k2_swe-agent_default_2025-12-29"
    },
    {
      "model_name": "GLM-4.7-Flash",
      "model_organization": "Z.ai",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-02-16",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 25.3,
          "sec_pass_1": 9.7,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-02-16",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "glm-4.7-flash_swe-agent_default_2026-02-16"
    },
    {
      "model_name": "GLM-5",
      "model_organization": "Z.ai",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-02-25",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 32.5,
          "sec_pass_1": 4.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-02-25",
        "susvibes_version": "v0.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "glm-5_openhands_default_2026-02-25"
    },
    {
      "model_name": "GLM-5",
      "model_organization": "Z.ai",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-02-25",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 66.7,
          "sec_pass_1": 14,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-02-25",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "glm-5_swe-agent_default_2026-02-25"
    },
    {
      "model_name": "Qwen3-Coder-Next",
      "model_organization": "Qwen",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-02-25",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 11,
          "sec_pass_1": 1.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-02-25",
        "susvibes_version": "v0.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "qwen3-coder-next_openhands_default_2026-02-25"
    },
    {
      "model_name": "Qwen3-Coder-Next",
      "model_organization": "Qwen",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-02-25",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 28,
          "sec_pass_1": 7.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-02-25",
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "qwen3-coder-next_swe-agent_default_2026-02-25"
    },
    {
      "model_name": "Gemini 3.1 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-03-10",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 41,
          "sec_pass_1": 8.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-03-10",
        "susvibes_version": "v0.0",
        "agent_framework": "SWE-agent",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-3.1-pro_swe-agent_default_2026-03-10"
    },
    {
      "model_name": "Gemini 3.1 Pro",
      "model_organization": "Google",
      "submitting_organization": "SusVibes Team",
      "submission_date": "2026-03-10",
      "submission_type": "standard",
      "contact_info": {
        "email": "sz3296@columbia.edu",
        "name": "Songwen Zhao",
        "github": "https://github.com/songwen6968"
      },
      "is_new": true,
      "trajectories_available": true,
      "references": [
        {
          "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
          "url": "https://arxiv.org/abs/2512.03262",
          "type": "paper"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        },
        {
          "title": "Example Model on Hugging Face",
          "url": "https://huggingface.co/example-org/example-model",
          "type": "huggingface"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 24,
          "sec_pass_1": 2.5,
          "cost": null
        }
      },
      "methodology": {
        "evaluation_date": "2026-03-10",
        "susvibes_version": "v0.0",
        "agent_framework": "OpenHands",
        "notes": "Official Submission",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "details": "Official Submission"
        }
      },
      "submissionDir": "gemini-3.1-pro_openhands_default_2026-03-10"
    },
    {
      "model_name": "Claude Opus 4.8",
      "model_organization": "Anthropic",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/claude_code/generic/claude-opus48/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/claude_code/generic/claude-opus48",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 83.9,
          "sec_pass_1": 32.8,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "Claude Code",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/claude_code/generic/claude-opus48",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "claude-opus-4.8_claude-code_default_2026-08-21"
    },
    {
      "model_name": "GPT-5.5",
      "model_organization": "OpenAI",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/codex/generic/codex-gpt55/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/codex/generic/codex-gpt55",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 86.6,
          "sec_pass_1": 35.5,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "Codex CLI",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/codex/generic/codex-gpt55",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "gpt-5.5_codex-cli_default_2026-08-21"
    },
    {
      "model_name": "Claude Opus 4.8",
      "model_organization": "Anthropic",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary_dehacked.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-claude48/generic/claude-opus-4.8/summary_dehacked.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-claude48/generic/claude-opus-4.8",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 59.1,
          "sec_pass_1": 26.3,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "GitHub Copilot",
        "notes": "Scores imported from the SafeVibe Hugging Face dehacked summary. Reward-hack-contaminated instances remain in the denominator and are excluded from the pass lists.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-claude48/generic/claude-opus-4.8",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "claude-opus-4.8_github-copilot_default_2026-08-21"
    },
    {
      "model_name": "GPT-5.5",
      "model_organization": "OpenAI",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary_dehacked.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-gpt5.5/generic/copilot/summary_dehacked.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-gpt5.5/generic/copilot",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 57,
          "sec_pass_1": 28.5,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "GitHub Copilot",
        "notes": "Scores imported from the SafeVibe Hugging Face dehacked summary. Reward-hack-contaminated instances remain in the denominator and are excluded from the pass lists.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-gpt5.5/generic/copilot",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "gpt-5.5_github-copilot_default_2026-08-21"
    },
    {
      "model_name": "MAI Code 1 Flash Internal",
      "model_organization": "Microsoft",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary_dehacked.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-mai/generic/mai-code-1-flash-internal/summary_dehacked.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-mai/generic/mai-code-1-flash-internal",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 66.7,
          "sec_pass_1": 31.2,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "GitHub Copilot",
        "notes": "Scores imported from the SafeVibe Hugging Face dehacked summary. Reward-hack-contaminated instances remain in the denominator and are excluded from the pass lists.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/copilot-mai/generic/mai-code-1-flash-internal",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "mai-code-1-flash-internal_github-copilot_default_2026-08-21"
    },
    {
      "model_name": "GPT-5.5",
      "model_organization": "OpenAI",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-gpt55/generic/openai__gpt-5.5/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-gpt55/generic/openai__gpt-5.5",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 84.4,
          "sec_pass_1": 43.5,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "mini-SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-gpt55/generic/openai__gpt-5.5",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "gpt-5.5_mini-swe-agent_default_2026-08-21"
    },
    {
      "model_name": "MiniMax M3 (MXFP8)",
      "model_organization": "MiniMax",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-m3/generic/openai____data__users__shared__models__MiniMaxAI__MiniMax-M3-MXFP8/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-m3/generic/openai____data__users__shared__models__MiniMaxAI__MiniMax-M3-MXFP8",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 68.3,
          "sec_pass_1": 26.3,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "mini-SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-m3/generic/openai____data__users__shared__models__MiniMaxAI__MiniMax-M3-MXFP8",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "minimax-m3-mxfp8_mini-swe-agent_default_2026-08-21"
    },
    {
      "model_name": "MAI Code 1 Flash Internal",
      "model_organization": "Microsoft",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary_dehacked.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-mai/generic/openai__mai-code-1-flash-internal/summary_dehacked.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-mai/generic/openai__mai-code-1-flash-internal",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 82.3,
          "sec_pass_1": 42.5,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "mini-SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face dehacked summary. Reward-hack-contaminated instances remain in the denominator and are excluded from the pass lists.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-mai/generic/openai__mai-code-1-flash-internal",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "mai-code-1-flash-internal_mini-swe-agent_default_2026-08-21"
    },
    {
      "model_name": "MiniMax M2.7",
      "model_organization": "MiniMax",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-minimax27/generic/openai____data__users__shared__models__MiniMaxAI__MiniMax-M2.7/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-minimax27/generic/openai____data__users__shared__models__MiniMaxAI__MiniMax-M2.7",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 59.7,
          "sec_pass_1": 18.8,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "mini-SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-minimax27/generic/openai____data__users__shared__models__MiniMaxAI__MiniMax-M2.7",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "minimax-m2.7_mini-swe-agent_default_2026-08-21"
    },
    {
      "model_name": "Qwen3.5-35B-A3B",
      "model_organization": "Qwen",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-qwen35/generic/openai__Qwen__Qwen3.5-35B-A3B/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-qwen35/generic/openai__Qwen__Qwen3.5-35B-A3B",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 42.5,
          "sec_pass_1": 18.3,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "mini-SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/mini-swe-qwen35/generic/openai__Qwen__Qwen3.5-35B-A3B",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "qwen3.5-35b-a3b_mini-swe-agent_default_2026-08-21"
    },
    {
      "model_name": "GPT-5.5",
      "model_organization": "OpenAI",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_gpt55/generic/gpt-5.5/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_gpt55/generic/gpt-5.5",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 80.1,
          "sec_pass_1": 35.5,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "Pi CLI",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_gpt55/generic/gpt-5.5",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "gpt-5.5_pi-cli_default_2026-08-21"
    },
    {
      "model_name": "Claude Opus 4.8",
      "model_organization": "Anthropic",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_mai/generic/claude-opus-4.8/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_mai/generic/claude-opus-4.8",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 88.7,
          "sec_pass_1": 28,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "Pi CLI",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_mai/generic/claude-opus-4.8",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "claude-opus-4.8_pi-cli_default_2026-08-21"
    },
    {
      "model_name": "MAI Code 1 Flash Internal",
      "model_organization": "Microsoft",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_mai/generic/mai-code-1-flash-internal/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_mai/generic/mai-code-1-flash-internal",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 66.1,
          "sec_pass_1": 22.6,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "Pi CLI",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/pi_mai/generic/mai-code-1-flash-internal",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "mai-code-1-flash-internal_pi-cli_default_2026-08-21"
    },
    {
      "model_name": "Claude Opus 4.8",
      "model_organization": "Anthropic",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/swe-claude48/generic/default_backticks__openai--claude-opus-4.8__t-0.00__p-1.00__c-0.00___susvibes.run_evaluation_generic_instances/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/swe-claude48/generic/default_backticks__openai--claude-opus-4.8__t-0.00__p-1.00__c-0.00___susvibes.run_evaluation_generic_instances",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 54.3,
          "sec_pass_1": 19.4,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/swe-claude48/generic/default_backticks__openai--claude-opus-4.8__t-0.00__p-1.00__c-0.00___susvibes.run_evaluation_generic_instances",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "claude-opus-4.8_swe-agent_default_2026-08-21"
    },
    {
      "model_name": "GPT-5.5",
      "model_organization": "OpenAI",
      "submitting_organization": "SafeVibe Team",
      "submission_date": "2026-08-21",
      "submission_type": "standard",
      "contact_info": {
        "email": "brxx122@gmail.com",
        "name": "Danqing Wang",
        "github": "dqwang122"
      },
      "is_new": true,
      "trajectories_available": false,
      "references": [
        {
          "title": "SafeVibe SusVibes summary.json",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/swe-gpt55/generic/default_backticks__openai--gpt-5.5_2026-04-24__t-0.00__p-None__c-0.00___susvibes.run_evaluation_generic_instances/summary.json",
          "type": "huggingface"
        },
        {
          "title": "SafeVibe SusVibes trajectories",
          "url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/swe-gpt55/generic/default_backticks__openai--gpt-5.5_2026-04-24__t-0.00__p-None__c-0.00___susvibes.run_evaluation_generic_instances",
          "type": "huggingface"
        },
        {
          "title": "SusVibes",
          "url": "https://github.com/LeiLiLab/susvibes",
          "type": "github"
        }
      ],
      "results": {
        "python": {
          "func_pass_1": 46.8,
          "sec_pass_1": 23.1,
          "cost": null
        }
      },
      "methodology": {
        "susvibes_version": "v1.0",
        "agent_framework": "SWE-agent",
        "notes": "Scores imported from the SafeVibe Hugging Face summary.",
        "verification": {
          "modified_prompts": false,
          "omitted_questions": false,
          "trajectories_url": "https://huggingface.co/datasets/dqwang122/safevibe-data/blob/main/trajectory/susvibes/swe-gpt55/generic/default_backticks__openai--gpt-5.5_2026-04-24__t-0.00__p-None__c-0.00___susvibes.run_evaluation_generic_instances",
          "details": "Complete 186-task v1.0 summary imported from SafeVibe. Full trajectories are available in the linked Hugging Face dataset."
        }
      },
      "submissionDir": "gpt-5.5_swe-agent_default_2026-08-21"
    }
  ]
}