From 5f55a5f98129e36f99b34c1d66706f66d3db47ef Mon Sep 17 00:00:00 2001 From: Hiroyuki Eguchi Date: Fri, 20 Jan 2017 10:02:46 +0900 Subject: [PATCH] Model Evaluation Prediction model needs to be evaluated accuracy. Add a "meteos model-evaluation" command to allow users to evaluate their models which they created. implements blueprint evaluate-model Change-Id: I4d54d789298f88251ddbd2dabab357318052cc40 --- meteosclient/api/client.py | 2 + meteosclient/api/model_evaluations.py | 68 +++++++++++++++++++++++++++ meteosclient/api/shell.py | 55 ++++++++++++++++++++++ sample/json/model_evaluation.json | 8 ++++ 4 files changed, 133 insertions(+) create mode 100644 meteosclient/api/model_evaluations.py create mode 100644 sample/json/model_evaluation.json diff --git a/meteosclient/api/client.py b/meteosclient/api/client.py index be5a6c0..9fabe2b 100644 --- a/meteosclient/api/client.py +++ b/meteosclient/api/client.py @@ -26,6 +26,7 @@ from meteosclient.api import templates from meteosclient.api import experiments from meteosclient.api import datasets from meteosclient.api import models +from meteosclient.api import model_evaluations from meteosclient.api import learnings @@ -119,6 +120,7 @@ class Client(object): self.experiments = experiments.ExperimentManager(client) self.datasets = datasets.DatasetManager(client) self.models = models.ModelManager(client) + self.model_evaluations = model_evaluations.ModelEvaluationManager(client) self.learnings = learnings.LearningManager(client) def _get_keystone_auth(self, username=None, api_key=None, auth_url=None, diff --git a/meteosclient/api/model_evaluations.py b/meteosclient/api/model_evaluations.py new file mode 100644 index 0000000..950a545 --- /dev/null +++ b/meteosclient/api/model_evaluations.py @@ -0,0 +1,68 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +from six.moves.urllib import parse + +from meteosclient.api import base + + +class ModelEvaluation(base.Resource): + resource_name = 'ModelEvaluation' + + +class ModelEvaluationManager(base.ResourceManager): + resource_class = ModelEvaluation + NotUpdated = base.NotUpdated() + + def create(self, display_name=None, model_id=None, + source_dataset_url=None, dataset_format=None, + swift_tenant=None, swift_username=None, + swift_password=None): + """Create a ModelEvaluation.""" + + data = { + 'display_name': display_name, + 'model_id': model_id, + 'source_dataset_url': source_dataset_url, + 'dataset_format': dataset_format, + 'swift_tenant': swift_tenant, + 'swift_username': swift_username, + 'swift_password': swift_password, + } + + body = {'model_evaluation': data} + + return self._create('/model_evaluations', body, 'model_evaluation') + + def list(self, search_opts=None, limit=None, marker=None, + sort_by=None, reverse=None): + """Get a list of ModelEvaluations.""" + query = base.get_query_string(search_opts, limit=limit, marker=marker, + sort_by=sort_by, reverse=reverse) + url = "/model_evaluations%s" % query + return self._page(url, 'model_evaluations', limit) + + def get(self, model_evaluation_id, show_progress=False): + """Get information about a ModelEvaluation.""" + url = ('/model_evaluations/%(model_evaluation_id)s?%(params)s' % + {"model_evaluation_id": model_evaluation_id, + "params": parse.urlencode({"show_progress": show_progress})}) + + return self._get(url, 'model_evaluation') + + def delete(self, model_evaluation_id): + """Delete a ModelEvaluation.""" + self._delete('/model_evaluations/%s' % model_evaluation_id) diff --git a/meteosclient/api/shell.py b/meteosclient/api/shell.py index 4bf204c..ad7fc75 100644 --- a/meteosclient/api/shell.py +++ b/meteosclient/api/shell.py @@ -288,6 +288,61 @@ def do_model_unload(cs, args): cs.models.unload(args.id) +# +# Model_Evaluations +# ~~~~~~~~ +# model_evaluation-list +# +# model_evaluation-show +# +# model_evaluation-create [--json ] +# +# model_evaluation-delete +# + +def do_model_evaluation_list(cs, args): + """Print a list of available model_evaluations.""" + model_evaluations = cs.model_evaluations.list() + + columns = ('id', + 'name', + 'status', + 'model_id', + 'model_type', + 'source_dataset_url', + 'stdout') + utils.print_list(model_evaluations, columns) + + +@utils.arg('id', + metavar='', + help='ID of the model_evaluation to show.') +def do_model_evaluation_show(cs, args): + """Show details of a model_evaluation.""" + model_evaluation = cs.model_evaluations.get(args.id) + _show_dict(model_evaluation) + + +@utils.arg('--json', + default=sys.stdin, + type=argparse.FileType('r'), + help='JSON representation of model_evaluation.') +def do_model_evaluation_create(cs, args): + """Create a model_evaluation.""" + model_evaluation = json.loads(args.json.read()) + + _filter_call_args(model_evaluation, cs.model_evaluations.create) + _show_dict(cs.model_evaluations.create(**model_evaluation)) + + +@utils.arg('id', + metavar='', + help='ID of the model_evaluation to delete.') +def do_model_evaluation_delete(cs, args): + """Delete a model_evaluation.""" + cs.model_evaluations.delete(args.id) + + # # Learnings # ~~~~~~~~ diff --git a/sample/json/model_evaluation.json b/sample/json/model_evaluation.json new file mode 100644 index 0000000..e5c7c19 --- /dev/null +++ b/sample/json/model_evaluation.json @@ -0,0 +1,8 @@ +{ + "display_name": "sample-evaluation", + "source_dataset_url": "", + "model_id": "", + "swift_tenant": "demo", + "swift_username": "demo", + "swift_password": "nova" +}