Setting the file. One moment. 10
Setup DevOps Agent
24.7
Code Output Guide · references
RDS Oracle
references/model-evaluation/scripts/validate_custom_metrics.py
Python·120 lines·4 KB
class
RatingValue
(
BaseModel
):
16 floatValue: Optional[float] = None
17 stringValue: Optional[str] = None
18
19 @model_validator(mode="after")
20 def exactly_one_value(self):
21 has_float = self.floatValue is not None
22 has_string = self.stringValue is not None
23 if has_float == has_string: # both set or neither set
24 raise ValueError("Exactly one of 'floatValue' or 'stringValue' must be set.")
25 return self
26
27
28class RatingScaleEntry(BaseModel):
29 definition: str
30 value: RatingValue
31
32 @field_validator("definition")
33 @classmethod
34 def definition_length(cls, v):
35 if len(v) > 100:
36 raise ValueError(f"Definition exceeds 100 chars ({len(v)}).")
37 return v
38
39
40class CustomMetricDefinition(BaseModel):
41 name: str
42 instructions: str
43 ratingScale: Optional[list[RatingScaleEntry]] = None
44
45 @model_validator(mode="after")
46 def check_instructions(self):
47 if len(self.instructions) > 5000:
48 raise ValueError(f"Instructions exceed 5000 char limit ({len(self.instructions)}).")
49 if "{{prediction}}" not in self.instructions and "{{prompt}}" not in self.instructions:
50 raise ValueError("Instructions must contain at least {{prompt}} or {{prediction}}.")
51 return self
52
53 @model_validator(mode="after")
54 def consistent_scale_types(self):
55 if not self.ratingScale:
56 return self
57 types = set()
58 for entry in self.ratingScale:
59 if entry.value.floatValue is not None:
60 types.add("float")
61 if entry.value.stringValue is not None:
62 types.add("string")
63 if len(types) > 1:
64 raise ValueError("ratingScale mixes float and string values. Use one type.")
65 return self
66
67
68class CustomMetric(BaseModel):
69 customMetricDefinition: CustomMetricDefinition
70
71
72def validate(raw: str) -> tuple[bool, list[str]]:
73 """Validate a JSON string of custom metrics. Returns (ok, errors)."""
74 try:
75 data = json.loads(raw)
76 except json.JSONDecodeError as e:
77 return False, [f"Invalid JSON: {e}"]
78
79 if not isinstance(data, list):
80 return False, ["Must be a JSON array of metric definitions."]
81 if len(data) == 0:
82 return False, ["Array is empty — need at least one metric."]
83 if len(data) > 10:
84 return False, [f"Too many metrics ({len(data)}). Maximum is 10."]
85
86 errors = []
87 for i, item in enumerate(data):
88 try:
89 CustomMetric.model_validate(item)
90 except Exception as e:
91 errors.append(f"Metric [{i}]: {e}")
92
93 return len(errors) == 0, errors
94
95
96def main():
97 if len(sys.argv) < 2:
98 print("Usage: python validate_custom_metrics.py '<json>' | file.json")
99 sys.exit(1)
100
101 arg = sys.argv[1]
102 try:
103 with open(arg, encoding="utf-8") as f:
104 raw = f.read()
105 except (FileNotFoundError, IsADirectoryError):
106 raw = arg
107
108 ok, errors = validate(raw)
109 if ok:
110 count = len(json.loads(raw))
111 print(f"✅ Valid — {count} custom metric{'s' if count != 1 else ''} defined.")
112 else:
113 print("❌ Validation failed:")
114 for err in errors:
115 print(f" - {err}")
116 sys.exit(1)
117
118
119if __name__ == "__main__":
120 main()