{
  "model": "OpenSML-JSONLite-175M-Instruct",
  "selected_stage": "sft-100",
  "step": 469,
  "model_sha256": "656f45015aab548465d93bed0b57e687497163741885da7364872851fa17e9a4",
  "output_mode": "native-json",
  "development": {
    "metrics": {
      "all": {
        "absent_accuracy": 0.9294117647058824,
        "count": 252,
        "counts": {
          "absent_correct": 237,
          "absent_total": 255,
          "exact": 200,
          "false_abstention": 25,
          "field_correct": 546,
          "fields": 604,
          "fn": 40,
          "fp": 29,
          "json_parse_ok": 251,
          "missed_abstention": 18,
          "natural_stop": 252,
          "nonrepetitive_nonempty": 252,
          "predicted_values": 339,
          "present_correct": 309,
          "present_total": 349,
          "schema_ok": 251,
          "token_limited": 0,
          "tp": 310,
          "unsupported_values": 4
        },
        "exact": 0.7936507936507936,
        "f1": 0.899854862119013,
        "false_abstention_rate": 0.07163323782234957,
        "field_accuracy": 0.9039735099337748,
        "json_parse_rate": 0.996031746031746,
        "missed_abstention_rate": 0.07058823529411765,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.03873514435736791,
        "precision": 0.9144542772861357,
        "present_accuracy": 0.8853868194842407,
        "recall": 0.8857142857142857,
        "schema_valid_rate": 0.996031746031746,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.011799410029498525
      },
      "domain/AddToPlaylist": {
        "absent_accuracy": 0.9024390243902439,
        "count": 36,
        "counts": {
          "absent_correct": 37,
          "absent_total": 41,
          "exact": 20,
          "false_abstention": 9,
          "field_correct": 72,
          "fields": 91,
          "fn": 15,
          "fp": 7,
          "json_parse_ok": 35,
          "missed_abstention": 4,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 42,
          "present_correct": 35,
          "present_total": 50,
          "schema_ok": 35,
          "token_limited": 0,
          "tp": 35,
          "unsupported_values": 2
        },
        "exact": 0.5555555555555556,
        "f1": 0.7608695652173915,
        "false_abstention_rate": 0.18,
        "field_accuracy": 0.7912087912087912,
        "json_parse_rate": 0.9722222222222222,
        "missed_abstention_rate": 0.0975609756097561,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.08232390080795288,
        "precision": 0.8333333333333334,
        "present_accuracy": 0.7,
        "recall": 0.7,
        "schema_valid_rate": 0.9722222222222222,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.047619047619047616
      },
      "domain/BookRestaurant": {
        "absent_accuracy": 0.8604651162790697,
        "count": 36,
        "counts": {
          "absent_correct": 37,
          "absent_total": 43,
          "exact": 24,
          "false_abstention": 4,
          "field_correct": 82,
          "fields": 95,
          "fn": 7,
          "fp": 9,
          "json_parse_ok": 36,
          "missed_abstention": 6,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 54,
          "present_correct": 45,
          "present_total": 52,
          "schema_ok": 36,
          "token_limited": 0,
          "tp": 45,
          "unsupported_values": 2
        },
        "exact": 0.6666666666666666,
        "f1": 0.8490566037735849,
        "false_abstention_rate": 0.07692307692307693,
        "field_accuracy": 0.8631578947368421,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.13953488372093023,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.0473247706718717,
        "precision": 0.8333333333333334,
        "present_accuracy": 0.8653846153846154,
        "recall": 0.8653846153846154,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.037037037037037035
      },
      "domain/GetWeather": {
        "absent_accuracy": 0.925,
        "count": 36,
        "counts": {
          "absent_correct": 37,
          "absent_total": 40,
          "exact": 28,
          "false_abstention": 5,
          "field_correct": 85,
          "fields": 94,
          "fn": 6,
          "fp": 3,
          "json_parse_ok": 36,
          "missed_abstention": 3,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 52,
          "present_correct": 48,
          "present_total": 54,
          "schema_ok": 36,
          "token_limited": 0,
          "tp": 49,
          "unsupported_values": 0
        },
        "exact": 0.7777777777777778,
        "f1": 0.9158878504672897,
        "false_abstention_rate": 0.09259259259259259,
        "field_accuracy": 0.9042553191489362,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.075,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.044021505151047476,
        "precision": 0.9423076923076923,
        "present_accuracy": 0.8888888888888888,
        "recall": 0.8909090909090909,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "domain/PlayMusic": {
        "absent_accuracy": 0.9767441860465116,
        "count": 36,
        "counts": {
          "absent_correct": 42,
          "absent_total": 43,
          "exact": 30,
          "false_abstention": 6,
          "field_correct": 81,
          "fields": 88,
          "fn": 6,
          "fp": 1,
          "json_parse_ok": 36,
          "missed_abstention": 1,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 40,
          "present_correct": 39,
          "present_total": 45,
          "schema_ok": 36,
          "token_limited": 0,
          "tp": 39,
          "unsupported_values": 0
        },
        "exact": 0.8333333333333334,
        "f1": 0.9176470588235294,
        "false_abstention_rate": 0.13333333333333333,
        "field_accuracy": 0.9204545454545454,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.023255813953488372,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.04131003013268734,
        "precision": 0.975,
        "present_accuracy": 0.8666666666666667,
        "recall": 0.8666666666666667,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "domain/RateBook": {
        "absent_accuracy": 0.9767441860465116,
        "count": 36,
        "counts": {
          "absent_correct": 42,
          "absent_total": 43,
          "exact": 34,
          "false_abstention": 0,
          "field_correct": 94,
          "fields": 96,
          "fn": 1,
          "fp": 2,
          "json_parse_ok": 36,
          "missed_abstention": 1,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 54,
          "present_correct": 52,
          "present_total": 53,
          "schema_ok": 36,
          "token_limited": 0,
          "tp": 52,
          "unsupported_values": 0
        },
        "exact": 0.9444444444444444,
        "f1": 0.9719626168224299,
        "false_abstention_rate": 0.0,
        "field_accuracy": 0.9791666666666666,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.023255813953488372,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.01588757585947557,
        "precision": 0.9629629629629629,
        "present_accuracy": 0.9811320754716981,
        "recall": 0.9811320754716981,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "domain/SearchCreativeWork": {
        "absent_accuracy": 1.0,
        "count": 36,
        "counts": {
          "absent_correct": 4,
          "absent_total": 4,
          "exact": 33,
          "false_abstention": 1,
          "field_correct": 44,
          "fields": 47,
          "fn": 3,
          "fp": 2,
          "json_parse_ok": 36,
          "missed_abstention": 0,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 42,
          "present_correct": 40,
          "present_total": 43,
          "schema_ok": 36,
          "token_limited": 0,
          "tp": 40,
          "unsupported_values": 0
        },
        "exact": 0.9166666666666666,
        "f1": 0.9411764705882352,
        "false_abstention_rate": 0.023255813953488372,
        "field_accuracy": 0.9361702127659575,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.0,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.017151051737730287,
        "precision": 0.9523809523809523,
        "present_accuracy": 0.9302325581395349,
        "recall": 0.9302325581395349,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "domain/SearchScreeningEvent": {
        "absent_accuracy": 0.926829268292683,
        "count": 36,
        "counts": {
          "absent_correct": 38,
          "absent_total": 41,
          "exact": 31,
          "false_abstention": 0,
          "field_correct": 88,
          "fields": 93,
          "fn": 2,
          "fp": 5,
          "json_parse_ok": 36,
          "missed_abstention": 3,
          "natural_stop": 36,
          "nonrepetitive_nonempty": 36,
          "predicted_values": 55,
          "present_correct": 50,
          "present_total": 52,
          "schema_ok": 36,
          "token_limited": 0,
          "tp": 50,
          "unsupported_values": 0
        },
        "exact": 0.8611111111111112,
        "f1": 0.9345794392523366,
        "false_abstention_rate": 0.0,
        "field_accuracy": 0.946236559139785,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.07317073170731707,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.023127176140810154,
        "precision": 0.9090909090909091,
        "present_accuracy": 0.9615384615384616,
        "recall": 0.9615384615384616,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "macro_exact": 0.7936507936507936,
      "paired_requests": {
        "accuracy": 0.7040816326530612,
        "both_correct": 69,
        "count": 98
      },
      "variant/alias": {
        "absent_accuracy": 0.96875,
        "count": 28,
        "counts": {
          "absent_correct": 31,
          "absent_total": 32,
          "exact": 24,
          "false_abstention": 3,
          "field_correct": 64,
          "fields": 68,
          "fn": 3,
          "fp": 1,
          "json_parse_ok": 28,
          "missed_abstention": 1,
          "natural_stop": 28,
          "nonrepetitive_nonempty": 28,
          "predicted_values": 34,
          "present_correct": 33,
          "present_total": 36,
          "schema_ok": 28,
          "token_limited": 0,
          "tp": 33,
          "unsupported_values": 0
        },
        "exact": 0.8571428571428571,
        "f1": 0.9428571428571428,
        "false_abstention_rate": 0.08333333333333333,
        "field_accuracy": 0.9411764705882353,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.03125,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.058719543459509235,
        "precision": 0.9705882352941176,
        "present_accuracy": 0.9166666666666666,
        "recall": 0.9166666666666666,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "variant/canonical": {
        "absent_accuracy": 0.9319371727748691,
        "count": 196,
        "counts": {
          "absent_correct": 178,
          "absent_total": 191,
          "exact": 157,
          "false_abstention": 17,
          "field_correct": 426,
          "fields": 468,
          "fn": 29,
          "fp": 24,
          "json_parse_ok": 196,
          "missed_abstention": 13,
          "natural_stop": 196,
          "nonrepetitive_nonempty": 196,
          "predicted_values": 273,
          "present_correct": 248,
          "present_total": 277,
          "schema_ok": 196,
          "token_limited": 0,
          "tp": 249,
          "unsupported_values": 3
        },
        "exact": 0.8010204081632653,
        "f1": 0.9038112522686025,
        "false_abstention_rate": 0.061371841155234655,
        "field_accuracy": 0.9102564102564102,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.06806282722513089,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.02958408025570323,
        "precision": 0.9120879120879121,
        "present_accuracy": 0.8953068592057761,
        "recall": 0.89568345323741,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.01098901098901099
      },
      "variant/nullable": {
        "absent_accuracy": 0.875,
        "count": 28,
        "counts": {
          "absent_correct": 28,
          "absent_total": 32,
          "exact": 19,
          "false_abstention": 5,
          "field_correct": 56,
          "fields": 68,
          "fn": 8,
          "fp": 4,
          "json_parse_ok": 27,
          "missed_abstention": 4,
          "natural_stop": 28,
          "nonrepetitive_nonempty": 28,
          "predicted_values": 32,
          "present_correct": 28,
          "present_total": 36,
          "schema_ok": 27,
          "token_limited": 0,
          "tp": 28,
          "unsupported_values": 1
        },
        "exact": 0.6785714285714286,
        "f1": 0.823529411764706,
        "false_abstention_rate": 0.1388888888888889,
        "field_accuracy": 0.8235294117647058,
        "json_parse_rate": 0.9642857142857143,
        "missed_abstention_rate": 0.125,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.08280819396687937,
        "precision": 0.875,
        "present_accuracy": 0.7777777777777778,
        "recall": 0.7777777777777778,
        "schema_valid_rate": 0.9642857142857143,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.03125
      }
    },
    "protocol": {
      "constrained_decoding": false,
      "json_assembly": false,
      "max_tokens": 256,
      "model_calls_per_request": 1,
      "output_repair": false,
      "prompt_truncation": false,
      "repetition_penalty": 1,
      "scorer": "strict-raw-json100-v1",
      "serialization": "native-sft-chat-v1",
      "temperature": 0
    },
    "scope": "Raw model-generated JSON, source-span annotation agreement. Duplicate keys, nonfinite JSON, fences, trailing prose and schema violations fail exact correctness. Flat string arrays/nullable strings in seven SNIPS domains; observed research panels, not a new blind or external benchmark."
  },
  "test": {
    "metrics": {
      "all": {
        "absent_accuracy": 0.9246395806028833,
        "count": 1444,
        "counts": {
          "absent_correct": 1411,
          "absent_total": 1526,
          "exact": 1191,
          "false_abstention": 111,
          "field_correct": 3193,
          "fields": 3497,
          "fn": 189,
          "fp": 186,
          "json_parse_ok": 1442,
          "missed_abstention": 115,
          "natural_stop": 1444,
          "nonrepetitive_nonempty": 1444,
          "predicted_values": 1968,
          "present_correct": 1782,
          "present_total": 1971,
          "schema_ok": 1441,
          "token_limited": 0,
          "tp": 1782,
          "unsupported_values": 30
        },
        "exact": 0.824792243767313,
        "f1": 0.9047981721249048,
        "false_abstention_rate": 0.0563165905631659,
        "field_accuracy": 0.9130683442951101,
        "json_parse_rate": 0.9986149584487535,
        "missed_abstention_rate": 0.07536041939711664,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.028334847962383555,
        "precision": 0.9054878048780488,
        "present_accuracy": 0.9041095890410958,
        "recall": 0.9041095890410958,
        "schema_valid_rate": 0.9979224376731302,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.01524390243902439
      },
      "domain/AddToPlaylist": {
        "absent_accuracy": 0.9110169491525424,
        "count": 206,
        "counts": {
          "absent_correct": 215,
          "absent_total": 236,
          "exact": 168,
          "false_abstention": 13,
          "field_correct": 475,
          "fields": 526,
          "fn": 30,
          "fp": 38,
          "json_parse_ok": 206,
          "missed_abstention": 21,
          "natural_stop": 206,
          "nonrepetitive_nonempty": 206,
          "predicted_values": 298,
          "present_correct": 260,
          "present_total": 290,
          "schema_ok": 206,
          "token_limited": 0,
          "tp": 260,
          "unsupported_values": 13
        },
        "exact": 0.8155339805825242,
        "f1": 0.8843537414965987,
        "false_abstention_rate": 0.04482758620689655,
        "field_accuracy": 0.903041825095057,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.08898305084745763,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.04326067907575776,
        "precision": 0.87248322147651,
        "present_accuracy": 0.896551724137931,
        "recall": 0.896551724137931,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0436241610738255
      },
      "domain/BookRestaurant": {
        "absent_accuracy": 0.908,
        "count": 208,
        "counts": {
          "absent_correct": 227,
          "absent_total": 250,
          "exact": 156,
          "false_abstention": 21,
          "field_correct": 506,
          "fields": 569,
          "fn": 40,
          "fp": 42,
          "json_parse_ok": 208,
          "missed_abstention": 23,
          "natural_stop": 208,
          "nonrepetitive_nonempty": 208,
          "predicted_values": 321,
          "present_correct": 279,
          "present_total": 319,
          "schema_ok": 208,
          "token_limited": 0,
          "tp": 279,
          "unsupported_values": 4
        },
        "exact": 0.75,
        "f1": 0.8718750000000001,
        "false_abstention_rate": 0.06583072100313479,
        "field_accuracy": 0.8892794376098418,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.092,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.032639438736133834,
        "precision": 0.8691588785046729,
        "present_accuracy": 0.8746081504702194,
        "recall": 0.8746081504702194,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.012461059190031152
      },
      "domain/GetWeather": {
        "absent_accuracy": 0.9349593495934959,
        "count": 206,
        "counts": {
          "absent_correct": 230,
          "absent_total": 246,
          "exact": 168,
          "false_abstention": 15,
          "field_correct": 479,
          "fields": 525,
          "fn": 30,
          "fp": 27,
          "json_parse_ok": 205,
          "missed_abstention": 16,
          "natural_stop": 206,
          "nonrepetitive_nonempty": 206,
          "predicted_values": 276,
          "present_correct": 249,
          "present_total": 279,
          "schema_ok": 204,
          "token_limited": 0,
          "tp": 249,
          "unsupported_values": 2
        },
        "exact": 0.8155339805825242,
        "f1": 0.8972972972972973,
        "false_abstention_rate": 0.053763440860215055,
        "field_accuracy": 0.9123809523809524,
        "json_parse_rate": 0.9951456310679612,
        "missed_abstention_rate": 0.06504065040650407,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.028848352069651157,
        "precision": 0.9021739130434783,
        "present_accuracy": 0.8924731182795699,
        "recall": 0.8924731182795699,
        "schema_valid_rate": 0.9902912621359223,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.007246376811594203
      },
      "domain/PlayMusic": {
        "absent_accuracy": 0.94140625,
        "count": 208,
        "counts": {
          "absent_correct": 241,
          "absent_total": 256,
          "exact": 149,
          "false_abstention": 44,
          "field_correct": 451,
          "fields": 523,
          "fn": 57,
          "fp": 28,
          "json_parse_ok": 208,
          "missed_abstention": 15,
          "natural_stop": 208,
          "nonrepetitive_nonempty": 208,
          "predicted_values": 238,
          "present_correct": 210,
          "present_total": 267,
          "schema_ok": 208,
          "token_limited": 0,
          "tp": 210,
          "unsupported_values": 2
        },
        "exact": 0.7163461538461539,
        "f1": 0.8316831683168316,
        "false_abstention_rate": 0.1647940074906367,
        "field_accuracy": 0.8623326959847036,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.05859375,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.04051304714644525,
        "precision": 0.8823529411764706,
        "present_accuracy": 0.7865168539325843,
        "recall": 0.7865168539325843,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.008403361344537815
      },
      "domain/RateBook": {
        "absent_accuracy": 0.9552845528455285,
        "count": 204,
        "counts": {
          "absent_correct": 235,
          "absent_total": 246,
          "exact": 188,
          "false_abstention": 2,
          "field_correct": 548,
          "fields": 567,
          "fn": 8,
          "fp": 14,
          "json_parse_ok": 203,
          "missed_abstention": 11,
          "natural_stop": 204,
          "nonrepetitive_nonempty": 204,
          "predicted_values": 327,
          "present_correct": 313,
          "present_total": 321,
          "schema_ok": 203,
          "token_limited": 0,
          "tp": 313,
          "unsupported_values": 2
        },
        "exact": 0.9215686274509803,
        "f1": 0.9660493827160493,
        "false_abstention_rate": 0.006230529595015576,
        "field_accuracy": 0.9664902998236331,
        "json_parse_rate": 0.9950980392156863,
        "missed_abstention_rate": 0.044715447154471545,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.011863740803485545,
        "precision": 0.9571865443425076,
        "present_accuracy": 0.9750778816199377,
        "recall": 0.9750778816199377,
        "schema_valid_rate": 0.9950980392156863,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0061162079510703364
      },
      "domain/SearchCreativeWork": {
        "absent_accuracy": 0.9230769230769231,
        "count": 208,
        "counts": {
          "absent_correct": 36,
          "absent_total": 39,
          "exact": 197,
          "false_abstention": 2,
          "field_correct": 275,
          "fields": 286,
          "fn": 8,
          "fp": 9,
          "json_parse_ok": 208,
          "missed_abstention": 3,
          "natural_stop": 208,
          "nonrepetitive_nonempty": 208,
          "predicted_values": 248,
          "present_correct": 239,
          "present_total": 247,
          "schema_ok": 208,
          "token_limited": 0,
          "tp": 239,
          "unsupported_values": 0
        },
        "exact": 0.9471153846153846,
        "f1": 0.9656565656565657,
        "false_abstention_rate": 0.008097165991902834,
        "field_accuracy": 0.9615384615384616,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.07692307692307693,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.019352024208483972,
        "precision": 0.9637096774193549,
        "present_accuracy": 0.9676113360323887,
        "recall": 0.9676113360323887,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.0
      },
      "domain/SearchScreeningEvent": {
        "absent_accuracy": 0.8972332015810277,
        "count": 204,
        "counts": {
          "absent_correct": 227,
          "absent_total": 253,
          "exact": 165,
          "false_abstention": 14,
          "field_correct": 459,
          "fields": 501,
          "fn": 16,
          "fp": 28,
          "json_parse_ok": 204,
          "missed_abstention": 26,
          "natural_stop": 204,
          "nonrepetitive_nonempty": 204,
          "predicted_values": 260,
          "present_correct": 232,
          "present_total": 248,
          "schema_ok": 204,
          "token_limited": 0,
          "tp": 232,
          "unsupported_values": 7
        },
        "exact": 0.8088235294117647,
        "f1": 0.9133858267716536,
        "false_abstention_rate": 0.056451612903225805,
        "field_accuracy": 0.9161676646706587,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.10276679841897234,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.021568229504291415,
        "precision": 0.8923076923076924,
        "present_accuracy": 0.9354838709677419,
        "recall": 0.9354838709677419,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.026923076923076925
      },
      "macro_exact": 0.8249888080699045,
      "paired_requests": {
        "accuracy": 0.7219020172910663,
        "both_correct": 501,
        "count": 694
      },
      "variant/alias": {
        "absent_accuracy": 0.8275862068965517,
        "count": 28,
        "counts": {
          "absent_correct": 24,
          "absent_total": 29,
          "exact": 19,
          "false_abstention": 2,
          "field_correct": 62,
          "fields": 75,
          "fn": 8,
          "fp": 7,
          "json_parse_ok": 27,
          "missed_abstention": 5,
          "natural_stop": 28,
          "nonrepetitive_nonempty": 28,
          "predicted_values": 45,
          "present_correct": 38,
          "present_total": 46,
          "schema_ok": 26,
          "token_limited": 0,
          "tp": 38,
          "unsupported_values": 2
        },
        "exact": 0.6785714285714286,
        "f1": 0.8351648351648352,
        "false_abstention_rate": 0.043478260869565216,
        "field_accuracy": 0.8266666666666667,
        "json_parse_rate": 0.9642857142857143,
        "missed_abstention_rate": 0.1724137931034483,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.05925792032836138,
        "precision": 0.8444444444444444,
        "present_accuracy": 0.8260869565217391,
        "recall": 0.8260869565217391,
        "schema_valid_rate": 0.9285714285714286,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.044444444444444446
      },
      "variant/canonical": {
        "absent_accuracy": 0.9305177111716622,
        "count": 1388,
        "counts": {
          "absent_correct": 1366,
          "absent_total": 1468,
          "exact": 1156,
          "false_abstention": 103,
          "field_correct": 3074,
          "fields": 3347,
          "fn": 171,
          "fp": 170,
          "json_parse_ok": 1388,
          "missed_abstention": 102,
          "natural_stop": 1388,
          "nonrepetitive_nonempty": 1388,
          "predicted_values": 1878,
          "present_correct": 1708,
          "present_total": 1879,
          "schema_ok": 1388,
          "token_limited": 0,
          "tp": 1708,
          "unsupported_values": 24
        },
        "exact": 0.8328530259365994,
        "f1": 0.909236092627096,
        "false_abstention_rate": 0.05481639169771155,
        "field_accuracy": 0.9184344188825814,
        "json_parse_rate": 1.0,
        "missed_abstention_rate": 0.06948228882833787,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.02666785328274954,
        "precision": 0.9094781682641108,
        "present_accuracy": 0.9089941458222459,
        "recall": 0.9089941458222459,
        "schema_valid_rate": 1.0,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.012779552715654952
      },
      "variant/nullable": {
        "absent_accuracy": 0.7241379310344828,
        "count": 28,
        "counts": {
          "absent_correct": 21,
          "absent_total": 29,
          "exact": 16,
          "false_abstention": 6,
          "field_correct": 57,
          "fields": 75,
          "fn": 10,
          "fp": 9,
          "json_parse_ok": 27,
          "missed_abstention": 8,
          "natural_stop": 28,
          "nonrepetitive_nonempty": 28,
          "predicted_values": 45,
          "present_correct": 36,
          "present_total": 46,
          "schema_ok": 27,
          "token_limited": 0,
          "tp": 36,
          "unsupported_values": 4
        },
        "exact": 0.5714285714285714,
        "f1": 0.7912087912087912,
        "false_abstention_rate": 0.13043478260869565,
        "field_accuracy": 0.76,
        "json_parse_rate": 0.9642857142857143,
        "missed_abstention_rate": 0.27586206896551724,
        "natural_stop_rate": 1.0,
        "nonrepetitive_nonempty_rate": 1.0,
        "output_nll": 0.08004708328683462,
        "precision": 0.8,
        "present_accuracy": 0.782608695652174,
        "recall": 0.782608695652174,
        "schema_valid_rate": 0.9642857142857143,
        "token_limited_rate": 0.0,
        "unsupported_value_rate": 0.08888888888888889
      }
    },
    "protocol": {
      "constrained_decoding": false,
      "json_assembly": false,
      "max_tokens": 256,
      "model_calls_per_request": 1,
      "output_repair": false,
      "prompt_truncation": false,
      "repetition_penalty": 1,
      "scorer": "strict-raw-json100-v1",
      "serialization": "native-sft-chat-v1",
      "temperature": 0
    },
    "scope": "Raw model-generated JSON, source-span annotation agreement. Duplicate keys, nonfinite JSON, fences, trailing prose and schema violations fail exact correctness. Flat string arrays/nullable strings in seven SNIPS domains; observed research panels, not a new blind or external benchmark."
  },
  "playground": {
    "metrics": {
      "alias": {
        "absent_accuracy": 0.875,
        "count": 7,
        "counts": {
          "absent_correct": 7,
          "absent_total": 8,
          "exact": 6,
          "false_abstention": 0,
          "field_correct": 16,
          "fields": 17,
          "fn": 0,
          "fp": 1,
          "json_parse_ok": 7,
          "missed_abstention": 1,
          "natural_stop": 7,
          "nonrepetitive_nonempty": 7,
          "predicted_values": 10,
          "present_correct": 9,
          "present_total": 9,
          "schema_ok": 7,
          "token_limited": 0,
          "tp": 9,
          "unsupported_values": 0
        },
        "exact": 0.8571428571428571,
        "f1": 0.9473684210526316,
        "field_accuracy": 0.9411764705882353,
        "json_parse": 1.0,
        "natural_stop_rate": 1.0,
        "precision": 0.9,
        "present_accuracy": 1.0,
        "recall": 1.0,
        "schema_valid": 1.0,
        "unsupported_value_rate": 0.0
      },
      "all": {
        "absent_accuracy": 0.9056603773584906,
        "count": 54,
        "counts": {
          "absent_correct": 48,
          "absent_total": 53,
          "exact": 42,
          "false_abstention": 3,
          "field_correct": 118,
          "fields": 135,
          "fn": 13,
          "fp": 7,
          "json_parse_ok": 52,
          "missed_abstention": 5,
          "natural_stop": 54,
          "nonrepetitive_nonempty": 54,
          "predicted_values": 78,
          "present_correct": 70,
          "present_total": 82,
          "schema_ok": 52,
          "token_limited": 0,
          "tp": 71,
          "unsupported_values": 2
        },
        "exact": 0.7777777777777778,
        "f1": 0.8765432098765431,
        "field_accuracy": 0.8740740740740741,
        "json_parse": 0.9629629629629629,
        "natural_stop_rate": 1.0,
        "precision": 0.9102564102564102,
        "present_accuracy": 0.8536585365853658,
        "recall": 0.8452380952380952,
        "schema_valid": 0.9629629629629629,
        "unsupported_value_rate": 0.02564102564102564
      },
      "authored_diagnostics": {
        "absent_accuracy": 1.0,
        "count": 12,
        "counts": {
          "absent_correct": 11,
          "absent_total": 11,
          "exact": 7,
          "false_abstention": 2,
          "field_correct": 29,
          "fields": 34,
          "fn": 6,
          "fp": 2,
          "json_parse_ok": 12,
          "missed_abstention": 0,
          "natural_stop": 12,
          "nonrepetitive_nonempty": 12,
          "predicted_values": 21,
          "present_correct": 18,
          "present_total": 23,
          "schema_ok": 12,
          "token_limited": 0,
          "tp": 19,
          "unsupported_values": 1
        },
        "exact": 0.5833333333333334,
        "f1": 0.8260869565217391,
        "field_accuracy": 0.8529411764705882,
        "json_parse": 1.0,
        "natural_stop_rate": 1.0,
        "precision": 0.9047619047619048,
        "present_accuracy": 0.782608695652174,
        "recall": 0.76,
        "schema_valid": 1.0,
        "unsupported_value_rate": 0.047619047619047616
      },
      "nullable": {
        "absent_accuracy": 0.625,
        "count": 7,
        "counts": {
          "absent_correct": 5,
          "absent_total": 8,
          "exact": 4,
          "false_abstention": 0,
          "field_correct": 12,
          "fields": 17,
          "fn": 2,
          "fp": 2,
          "json_parse_ok": 6,
          "missed_abstention": 3,
          "natural_stop": 7,
          "nonrepetitive_nonempty": 7,
          "predicted_values": 9,
          "present_correct": 7,
          "present_total": 9,
          "schema_ok": 6,
          "token_limited": 0,
          "tp": 7,
          "unsupported_values": 1
        },
        "exact": 0.5714285714285714,
        "f1": 0.7777777777777778,
        "field_accuracy": 0.7058823529411765,
        "json_parse": 0.8571428571428571,
        "natural_stop_rate": 1.0,
        "precision": 0.7777777777777778,
        "present_accuracy": 0.7777777777777778,
        "recall": 0.7777777777777778,
        "schema_valid": 0.8571428571428571,
        "unsupported_value_rate": 0.1111111111111111
      },
      "observed_originals": {
        "absent_accuracy": 0.9615384615384616,
        "count": 28,
        "counts": {
          "absent_correct": 25,
          "absent_total": 26,
          "exact": 25,
          "false_abstention": 1,
          "field_correct": 61,
          "fields": 67,
          "fn": 5,
          "fp": 2,
          "json_parse_ok": 27,
          "missed_abstention": 1,
          "natural_stop": 28,
          "nonrepetitive_nonempty": 28,
          "predicted_values": 38,
          "present_correct": 36,
          "present_total": 41,
          "schema_ok": 27,
          "token_limited": 0,
          "tp": 36,
          "unsupported_values": 0
        },
        "exact": 0.8928571428571429,
        "f1": 0.9113924050632912,
        "field_accuracy": 0.9104477611940298,
        "json_parse": 0.9642857142857143,
        "natural_stop_rate": 1.0,
        "precision": 0.9473684210526315,
        "present_accuracy": 0.8780487804878049,
        "recall": 0.8780487804878049,
        "schema_valid": 0.9642857142857143,
        "unsupported_value_rate": 0.0
      },
      "schema_stress": {
        "absent_accuracy": 0.75,
        "count": 14,
        "counts": {
          "absent_correct": 12,
          "absent_total": 16,
          "exact": 10,
          "false_abstention": 0,
          "field_correct": 28,
          "fields": 34,
          "fn": 2,
          "fp": 3,
          "json_parse_ok": 13,
          "missed_abstention": 4,
          "natural_stop": 14,
          "nonrepetitive_nonempty": 14,
          "predicted_values": 19,
          "present_correct": 16,
          "present_total": 18,
          "schema_ok": 13,
          "token_limited": 0,
          "tp": 16,
          "unsupported_values": 1
        },
        "exact": 0.7142857142857143,
        "f1": 0.8648648648648649,
        "field_accuracy": 0.8235294117647058,
        "json_parse": 0.9285714285714286,
        "natural_stop_rate": 1.0,
        "precision": 0.8421052631578947,
        "present_accuracy": 0.8888888888888888,
        "recall": 0.8888888888888888,
        "schema_valid": 0.9285714285714286,
        "unsupported_value_rate": 0.05263157894736842
      }
    },
    "protocol": {
      "constrained": false,
      "json_assembly": false,
      "max_tokens": 256,
      "model_calls_per_request": 1,
      "output_repair": false,
      "raw_text_preserved": true,
      "repetition_penalty": 1,
      "route": "/api/json/generate",
      "seed": 2026,
      "temperature": 0,
      "top_k": 40
    },
    "scope": "All actual playground source/schema fixtures, with their displayed schema property ordering and instructions unchanged. Observed original DEV cases, previously authored diagnostics, and observed schema-stress views; local diagnostic, not a fresh blind benchmark. NLL not measured through generation API."
  },
  "historical_protocol_note": "SFT-093 results used labeled fieldwise generation plus JSON assembly; they do not describe the current raw JSON model."
}