# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test for using the model_eval_lib API."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import tempfile


import tensorflow as tf
from tensorflow_model_analysis.api import model_eval_lib
from tensorflow_model_analysis.eval_saved_model import testutil
from tensorflow_model_analysis.eval_saved_model.example_trainers import csv_linear_classifier
from tensorflow_model_analysis.eval_saved_model.example_trainers import custom_estimator
from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator
from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_classifier
from tensorflow_model_analysis.eval_saved_model.post_export_metrics import metric_keys
from tensorflow_model_analysis.eval_saved_model.post_export_metrics import post_export_metrics
from tensorflow_model_analysis.slicer import slicer


class EvaluateTest(testutil.TensorflowModelAnalysisTest):

  def setUp(self):
    self.longMessage = True

  def _getTempDir(self):
    return tempfile.mkdtemp()

  def _exportEvalSavedModel(self, classifier):
    temp_model_location = os.path.join(self._getTempDir(), 'eval_export_dir')
    _, model_location = classifier(None, temp_model_location)
    return model_location

  def _writeTFExamplesToTFRecords(self, examples):
    data_location = os.path.join(self._getTempDir(), 'input_data.rio')
    with tf.python_io.TFRecordWriter(data_location) as writer:
      for example in examples:
        writer.write(example.SerializeToString())
    return data_location

  def _writeCSVToTextFile(self, examples):
    data_location = os.path.join(self._getTempDir(), 'input_data.csv')
    with open(data_location, 'w') as writer:
      for example in examples:
        writer.write(example + '\n')
    return data_location

  def assertMetricsAlmostEqual(self, got_value, expected_value):
    for (s, m) in got_value:
      self.assertIn(s, expected_value)
      self.assertDictElementsAlmostEqual(m, expected_value[s])

  def assertPlotsAlmostEqual(self, got_value, expected_value):
    for (s, m) in got_value:
      self.assertIn(s, expected_value)
      self.assertDictMatrixRowsAlmostEqual(m, expected_value[s])

  def testRunModelAnalysis(self):
    model_location = self._exportEvalSavedModel(
        linear_classifier.simple_linear_classifier)
    examples = [
        self._makeExample(age=3.0, language='english', label=1.0),
        self._makeExample(age=3.0, language='chinese', label=0.0),
        self._makeExample(age=4.0, language='english', label=1.0),
        self._makeExample(age=5.0, language='chinese', label=1.0)
    ]
    data_location = self._writeTFExamplesToTFRecords(examples)
    slice_spec = [slicer.SingleSliceSpec(columns=['language'])]
    eval_result = model_eval_lib.run_model_analysis(
        model_location,
        data_location,
        slice_spec=slice_spec,
        example_weight_key='age')
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected = {
        (('language', 'chinese'),): {
            'accuracy': 0.5,
            'my_mean_label': 0.5,
            metric_keys.EXAMPLE_WEIGHT: 8.0,
            metric_keys.EXAMPLE_COUNT: 2.0,
        },
        (('language', 'english'),): {
            'accuracy': 1.0,
            'my_mean_label': 1.0,
            metric_keys.EXAMPLE_WEIGHT: 7.0,
            metric_keys.EXAMPLE_COUNT: 2.0,
        }
    }
    self.assertEqual(eval_result.config.model_location, model_location)
    self.assertEqual(eval_result.config.data_location, data_location)
    self.assertEqual(eval_result.config.slice_spec, slice_spec)
    self.assertMetricsAlmostEqual(eval_result.slicing_metrics, expected)
    self.assertFalse(eval_result.plots)

  def testRunModelAnalysisWithPlots(self):
    model_location = self._exportEvalSavedModel(
        fixed_prediction_estimator.simple_fixed_prediction_estimator)
    examples = [
        self._makeExample(prediction=0, label=1.0),
        self._makeExample(prediction=0.7, label=0.0),
        self._makeExample(prediction=0.8, label=1.0),
        self._makeExample(prediction=1.0, label=1.0)
    ]
    data_location = self._writeTFExamplesToTFRecords(examples)
    eval_result = model_eval_lib.run_model_analysis(
        model_location,
        data_location,
        add_metrics_callbacks=[post_export_metrics.auc_plots()])
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected_metrics = {(): {metric_keys.EXAMPLE_COUNT: 4.0,}}
    expected_plots = {
        (): {
            metric_keys.AUC_PLOTS_MATRICES: [(8001, [
                2, 1, 0, 1, 1.0 / 1.0, 1.0 / 3.0
            ])],
        }
    }
    self.assertMetricsAlmostEqual(eval_result.slicing_metrics, expected_metrics)
    self.assertPlotsAlmostEqual(eval_result.plots, expected_plots)

  def testRunModelAnalysisForCSVText(self):
    model_location = self._exportEvalSavedModel(
        csv_linear_classifier.simple_csv_linear_classifier)
    examples = [
        '3.0,english,1.0', '3.0,chinese,0.0', '4.0,english,1.0',
        '5.0,chinese,1.0'
    ]
    data_location = self._writeCSVToTextFile(examples)
    eval_result = model_eval_lib.run_model_analysis(
        model_location, data_location, file_format='text')
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected = {(): {'accuracy': 0.75, metric_keys.EXAMPLE_COUNT: 4.0}}
    self.assertMetricsAlmostEqual(eval_result.slicing_metrics, expected)

  def testMultipleModelAnalysis(self):
    model_location_1 = self._exportEvalSavedModel(
        linear_classifier.simple_linear_classifier)
    model_location_2 = self._exportEvalSavedModel(
        custom_estimator.simple_custom_estimator)
    examples = [
        self._makeExample(age=3.0, language='english', label=1.0),
        self._makeExample(age=3.0, language='chinese', label=0.0),
        self._makeExample(age=4.0, language='english', label=1.0),
        self._makeExample(age=5.0, language='chinese', label=1.0)
    ]
    data_location = self._writeTFExamplesToTFRecords(examples)
    eval_results = model_eval_lib.multiple_model_analysis(
        [model_location_1, model_location_2],
        data_location,
        slice_spec=[slicer.SingleSliceSpec(features=[('language', 'english')])])
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    self.assertEqual(2, len(eval_results._results))
    expected_result_1 = {
        (('language', 'english'),): {
            'my_mean_label': 1.0,
            metric_keys.EXAMPLE_COUNT: 2.0
        }
    }
    expected_result_2 = {
        (('language', 'english'),): {
            'mean_label': 1.0,
            metric_keys.EXAMPLE_COUNT: 2.0
        }
    }
    self.assertMetricsAlmostEqual(eval_results._results[0].slicing_metrics,
                                  expected_result_1)
    self.assertMetricsAlmostEqual(eval_results._results[1].slicing_metrics,
                                  expected_result_2)

  def testMultipleDataAnalysis(self):
    model_location = self._exportEvalSavedModel(
        linear_classifier.simple_linear_classifier)
    data_location_1 = self._writeTFExamplesToTFRecords([
        self._makeExample(age=3.0, language='english', label=1.0),
        self._makeExample(age=3.0, language='english', label=0.0),
        self._makeExample(age=5.0, language='chinese', label=1.0)
    ])
    data_location_2 = self._writeTFExamplesToTFRecords(
        [self._makeExample(age=4.0, language='english', label=1.0)])
    eval_results = model_eval_lib.multiple_data_analysis(
        model_location, [data_location_1, data_location_2],
        slice_spec=[slicer.SingleSliceSpec(features=[('language', 'english')])])
    self.assertEqual(2, len(eval_results._results))
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected_result_1 = {
        (('language', 'english'),): {
            metric_keys.EXAMPLE_COUNT: 2.0,
        }
    }
    expected_result_2 = {
        (('language', 'english'),): {
            metric_keys.EXAMPLE_COUNT: 1.0,
        }
    }
    self.assertMetricsAlmostEqual(eval_results._results[0].slicing_metrics,
                                  expected_result_1)
    self.assertMetricsAlmostEqual(eval_results._results[1].slicing_metrics,
                                  expected_result_2)


if __name__ == '__main__':
  tf.test.main()

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test for using the model_eval_lib API."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import tempfile


import tensorflow as tf
from tensorflow_model_analysis.api import model_eval_lib
from tensorflow_model_analysis.eval_saved_model import testutil
from tensorflow_model_analysis.eval_saved_model.example_trainers import csv_linear_classifier
from tensorflow_model_analysis.eval_saved_model.example_trainers import custom_estimator
from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator
from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_classifier
from tensorflow_model_analysis.eval_saved_model.post_export_metrics import metric_keys
from tensorflow_model_analysis.eval_saved_model.post_export_metrics import post_export_metrics
from tensorflow_model_analysis.slicer import slicer


class EvaluateTest(testutil.TensorflowModelAnalysisTest):

  def setUp(self):
    self.longMessage = True

  def _getTempDir(self):
    return tempfile.mkdtemp()

  def _exportEvalSavedModel(self, classifier):
    temp_model_location = os.path.join(self._getTempDir(), 'eval_export_dir')
    _, model_location = classifier(None, temp_model_location)
    return model_location

  def _writeTFExamplesToTFRecords(self, examples):
    data_location = os.path.join(self._getTempDir(), 'input_data.rio')
    with tf.python_io.TFRecordWriter(data_location) as writer:
      for example in examples:
        writer.write(example.SerializeToString())
    return data_location

  def _writeCSVToTextFile(self, examples):
    data_location = os.path.join(self._getTempDir(), 'input_data.csv')
    with open(data_location, 'w') as writer:
      for example in examples:
        writer.write(example + '\n')
    return data_location

  def assertMetricsAlmostEqual(self, got_value, expected_value):
    for (s, m) in got_value:
      self.assertIn(s, expected_value)
      self.assertDictElementsAlmostEqual(m, expected_value[s])

  def assertPlotsAlmostEqual(self, got_value, expected_value):
    for (s, m) in got_value:
      self.assertIn(s, expected_value)
      self.assertDictMatrixRowsAlmostEqual(m, expected_value[s])

  def testRunModelAnalysis(self):
    model_location = self._exportEvalSavedModel(
        linear_classifier.simple_linear_classifier)
    examples = [
        self._makeExample(age=3.0, language='english', label=1.0),
        self._makeExample(age=3.0, language='chinese', label=0.0),
        self._makeExample(age=4.0, language='english', label=1.0),
        self._makeExample(age=5.0, language='chinese', label=1.0)
    ]
    data_location = self._writeTFExamplesToTFRecords(examples)
    slice_spec = [slicer.SingleSliceSpec(columns=['language'])]
    eval_result = model_eval_lib.run_model_analysis(
        model_location,
        data_location,
        slice_spec=slice_spec,
        example_weight_key='age')
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected = {
        (('language', 'chinese'),): {
            'accuracy': 0.5,
            'my_mean_label': 0.5,
            metric_keys.EXAMPLE_WEIGHT: 8.0,
            metric_keys.EXAMPLE_COUNT: 2.0,
        },
        (('language', 'english'),): {
            'accuracy': 1.0,
            'my_mean_label': 1.0,
            metric_keys.EXAMPLE_WEIGHT: 7.0,
            metric_keys.EXAMPLE_COUNT: 2.0,
        }
    }
    self.assertEqual(eval_result.config.model_location, model_location)
    self.assertEqual(eval_result.config.data_location, data_location)
    self.assertEqual(eval_result.config.slice_spec, slice_spec)
    self.assertMetricsAlmostEqual(eval_result.slicing_metrics, expected)
    self.assertFalse(eval_result.plots)

  def testRunModelAnalysisWithPlots(self):
    model_location = self._exportEvalSavedModel(
        fixed_prediction_estimator.simple_fixed_prediction_estimator)
    examples = [
        self._makeExample(prediction=0, label=1.0),
        self._makeExample(prediction=0.7, label=0.0),
        self._makeExample(prediction=0.8, label=1.0),
        self._makeExample(prediction=1.0, label=1.0)
    ]
    data_location = self._writeTFExamplesToTFRecords(examples)
    eval_result = model_eval_lib.run_model_analysis(
        model_location,
        data_location,
        add_metrics_callbacks=[post_export_metrics.auc_plots()])
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected_metrics = {(): {metric_keys.EXAMPLE_COUNT: 4.0,}}
    expected_plots = {
        (): {
            metric_keys.AUC_PLOTS_MATRICES: [(8001, [
                2, 1, 0, 1, 1.0 / 1.0, 1.0 / 3.0
            ])],
        }
    }
    self.assertMetricsAlmostEqual(eval_result.slicing_metrics, expected_metrics)
    self.assertPlotsAlmostEqual(eval_result.plots, expected_plots)

  def testRunModelAnalysisForCSVText(self):
    model_location = self._exportEvalSavedModel(
        csv_linear_classifier.simple_csv_linear_classifier)
    examples = [
        '3.0,english,1.0', '3.0,chinese,0.0', '4.0,english,1.0',
        '5.0,chinese,1.0'
    ]
    data_location = self._writeCSVToTextFile(examples)
    eval_result = model_eval_lib.run_model_analysis(
        model_location, data_location, file_format='text')
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected = {(): {'accuracy': 0.75, metric_keys.EXAMPLE_COUNT: 4.0}}
    self.assertMetricsAlmostEqual(eval_result.slicing_metrics, expected)

  def testMultipleModelAnalysis(self):
    model_location_1 = self._exportEvalSavedModel(
        linear_classifier.simple_linear_classifier)
    model_location_2 = self._exportEvalSavedModel(
        custom_estimator.simple_custom_estimator)
    examples = [
        self._makeExample(age=3.0, language='english', label=1.0),
        self._makeExample(age=3.0, language='chinese', label=0.0),
        self._makeExample(age=4.0, language='english', label=1.0),
        self._makeExample(age=5.0, language='chinese', label=1.0)
    ]
    data_location = self._writeTFExamplesToTFRecords(examples)
    eval_results = model_eval_lib.multiple_model_analysis(
        [model_location_1, model_location_2],
        data_location,
        slice_spec=[slicer.SingleSliceSpec(features=[('language', 'english')])])
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    self.assertEqual(2, len(eval_results._results))
    expected_result_1 = {
        (('language', 'english'),): {
            'my_mean_label': 1.0,
            metric_keys.EXAMPLE_COUNT: 2.0
        }
    }
    expected_result_2 = {
        (('language', 'english'),): {
            'mean_label': 1.0,
            metric_keys.EXAMPLE_COUNT: 2.0
        }
    }
    self.assertMetricsAlmostEqual(eval_results._results[0].slicing_metrics,
                                  expected_result_1)
    self.assertMetricsAlmostEqual(eval_results._results[1].slicing_metrics,
                                  expected_result_2)

  def testMultipleDataAnalysis(self):
    model_location = self._exportEvalSavedModel(
        linear_classifier.simple_linear_classifier)
    data_location_1 = self._writeTFExamplesToTFRecords([
        self._makeExample(age=3.0, language='english', label=1.0),
        self._makeExample(age=3.0, language='english', label=0.0),
        self._makeExample(age=5.0, language='chinese', label=1.0)
    ])
    data_location_2 = self._writeTFExamplesToTFRecords(
        [self._makeExample(age=4.0, language='english', label=1.0)])
    eval_results = model_eval_lib.multiple_data_analysis(
        model_location, [data_location_1, data_location_2],
        slice_spec=[slicer.SingleSliceSpec(features=[('language', 'english')])])
    self.assertEqual(2, len(eval_results._results))
    # We only check some of the metrics to ensure that the end-to-end
    # pipeline works.
    expected_result_1 = {
        (('language', 'english'),): {
            metric_keys.EXAMPLE_COUNT: 2.0,
        }
    }
    expected_result_2 = {
        (('language', 'english'),): {
            metric_keys.EXAMPLE_COUNT: 1.0,
        }
    }
    self.assertMetricsAlmostEqual(eval_results._results[0].slicing_metrics,
                                  expected_result_1)
    self.assertMetricsAlmostEqual(eval_results._results[1].slicing_metrics,
                                  expected_result_2)


if __name__ == '__main__':
  tf.test.main()

# Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. Test for using the model_eval_lib API. # We only check some of the metrics to ensure that the end-to-end # pipeline works. # We only check some of the metrics to ensure that the end-to-end # pipeline works. # We only check some of the metrics to ensure that the end-to-end # pipeline works. # We only check some of the metrics to ensure that the end-to-end # pipeline works. # We only check some of the metrics to ensure that the end-to-end # pipeline works.