validation_result.proto

// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package tensorflow_model_analysis;

import "tensorflow_model_analysis/proto/config.proto";
import "tensorflow_model_analysis/proto/metrics_for_slice.proto";

// Information about failure per metric.
message ValidationFailure {
  MetricKey metric_key = 1;
  MetricThreshold metric_threshold = 2;
  MetricValue metric_value = 3;
  string message = 4;
}

// Information about slices matched.
message SlicingDetails {
  oneof slicing_spec_oneof {
    SlicingSpec slicing_spec = 1;
    CrossSlicingSpec cross_slicing_spec = 3;
  }
  int32 num_matching_slices = 2;
}

// Extra details about validation.
message ValidationDetails {
  repeated SlicingDetails slicing_details = 1;
}

message MetricsValidationForSlice {
  oneof slice_key_oneof {
    // SliceKey of a given slice.
    SliceKey slice_key = 2;

    // CrossSliceKey for cross slice validations.
    CrossSliceKey cross_slice_key = 4;
  }
  // All failures under a slice.
  repeated ValidationFailure failures = 3;
}

message ValidationResult {
  // True if there are no metric validation failures or missing slices, else
  // false.
  bool validation_ok = 1;

  // True if failure due to missing thresholds.
  bool missing_thresholds = 6;

  // Information about which threshold is blocking which metric.
  repeated MetricsValidationForSlice metric_validations_per_slice = 2;

  // Information about missing slices.
  repeated SlicingSpec missing_slices = 3;

  // Information about missing cross slices.
  repeated CrossSlicingSpec missing_cross_slices = 5;

  // Extra details about validation performed.
  ValidationDetails validation_details = 4;

  // True if this run is rubbertamped. A rubberstamped validation is one in
  // which there was no baseline model, so diff thresholds were ignored, but in
  // which the non-diff thresholds are still checked.
  bool rubber_stamp = 7;
}