From 7885a1b7c6a7bd747b2298c44fbde3280da45b6d Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Sat, 16 Jun 2018 05:02:34 +0200 Subject: [PATCH] Remove code that is no longer used. (#681) * tf_job - This is a helm chart for TFJobs; we no longer use helm. This has long since been replaced by ksonnet. * gke - Looks like some old datalab notebook. This is no longer used * tensorflow-models - Looks like an old attempt at an example that was never quite finished. --- examples/gke/notebook_image/Dockerfile | 42 ----- examples/gke/notebook_image/Makefile | 30 --- examples/gke/test_notebook.py | 173 ------------------ .../tensorflow-models/Dockerfile.template | 19 -- examples/tensorflow-models/README.md | 2 - examples/tf_job/.helmignore | 21 --- examples/tf_job/Chart.yaml | 4 - examples/tf_job/templates/_helpers.tpl | 16 -- examples/tf_job/templates/tf_job.yaml | 24 --- examples/tf_job/values.yaml | 4 - 10 files changed, 335 deletions(-) delete mode 100644 examples/gke/notebook_image/Dockerfile delete mode 100644 examples/gke/notebook_image/Makefile delete mode 100644 examples/gke/test_notebook.py delete mode 100644 examples/tensorflow-models/Dockerfile.template delete mode 100644 examples/tensorflow-models/README.md delete mode 100644 examples/tf_job/.helmignore delete mode 100644 examples/tf_job/Chart.yaml delete mode 100644 examples/tf_job/templates/_helpers.tpl delete mode 100644 examples/tf_job/templates/tf_job.yaml delete mode 100644 examples/tf_job/values.yaml diff --git a/examples/gke/notebook_image/Dockerfile b/examples/gke/notebook_image/Dockerfile deleted file mode 100644 index 7dbb0fb9cf..0000000000 --- a/examples/gke/notebook_image/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A Dockerfile suitable for building a docker container suitable for running -# the GKE example notebook. - -FROM gcr.io/cloud-datalab/datalab:local-20171023 -LABEL authors="Jeremy Lewi " - -# Install Python K8s client library -RUN pip install kubernetes - -# Upgrade some of the GCP client librarys -RUN pip install --upgrade google-cloud-logging==1.3.0 - -# Install docker -# Docker is used when running locally to build the images. -# Note: 1.11+ changes the tarball format -# TODO(jlewi): I think it should be fine to upgrade to newer version of docker. -RUN curl -L "https://get.docker.com/builds/Linux/x86_64/docker-1.9.1.tgz" \ - | tar -C /usr/bin -xvzf- --strip-components=3 usr/local/bin/docker - -# Install kubectl -RUN gcloud components install -q kubectl - -# Install Helm -RUN wget -O /tmp/get_helm.sh \ - https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get && \ - chmod 700 /tmp/get_helm.sh && \ - /tmp/get_helm.sh && \ - rm /tmp/get_helm.sh diff --git a/examples/gke/notebook_image/Makefile b/examples/gke/notebook_image/Makefile deleted file mode 100644 index 3fa67925ce..0000000000 --- a/examples/gke/notebook_image/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -IMG = gcr.io/tf-on-k8s-dogfood/gke-datalab -TAG = $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty) - -all: build - -build: - @echo {\"image\": \"$(IMG):$(TAG)\"} > version.json - docker build -t $(IMG):$(TAG) . - docker tag $(IMG):$(TAG) $(IMG):latest - @echo Built $(IMG):$(TAG) and tagged with latest - rm -f version.json - -push: build - gcloud docker -- push $(IMG):$(TAG) - gcloud docker -- push $(IMG):latest - @echo Pushed $(IMG) with :latest and :$(TAG) tags diff --git a/examples/gke/test_notebook.py b/examples/gke/test_notebook.py deleted file mode 100644 index 4885612f83..0000000000 --- a/examples/gke/test_notebook.py +++ /dev/null @@ -1,173 +0,0 @@ -import collections -import datetime -import logging -import os -import re -import runpy -import tempfile -import unittest -import uuid - -import nbconvert -import nbformat -import six -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials -from py import util - -SubTuple = collections.namedtuple("SubTuple", ("name", "value", "pattern")) - -def replace_vars(lines, new_values): - """Substitutite in the values of the variables. - - This function assumes there is a single substitution for each variable. - The function will return as soon as all variables have matched once. - - Args: - lines: List of lines comprising the notebook. - new_values: Key value pairs containing values to set different variables to. - """ - - # List of matches variables. - matches = [] - matched = set() - for k, v in new_values.iteritems(): - # Format the values appropriately - new_value = v - if isinstance(v, six.string_types): - new_value = "\"{0}\"".format(v) - matches.append(SubTuple(k, new_value, - re.compile(r"\s*{0}\s*=.*".format(k)))) - - for i, l in enumerate(lines): - for p in matches: - # Check if this line matches - if p.pattern.match(l): - # Replace this line with this text - # We need to preserve leading white space for indentation - pieces = l.split("=", 1) - lines[i] = "{0}={1}".format(pieces[0], p.value) - matched.add(p.name) - - unmatched = [] - for k, _ in new_values.iteritems(): - if not k in matched: - unmatched.append(k) - if unmatched: - raise ValueError("No matches for variables {0}".format(", ".join(unmatched))) - - return lines - -def strip_appendix(lines): - """Remove all code in the appendix.""" - - p = re.compile(r"#\s*#*\s*Appendix") - for i in range(len(lines) - 1, 0, -1): - if p.match(lines[i], re.IGNORECASE): - return lines[0:i] - raise ValueError("Could not find Appendix") - -def strip_unexecutable(lines): - """Remove all code that we can't execute""" - - valid = [] - for l in lines: - if l.startswith("get_ipython"): - continue - valid.append(l) - return valid - -class TestNotebook(unittest.TestCase): - @staticmethod - def run_test(project, zone, cluster, new_values): # pylint: disable=too-many-locals - # TODO(jeremy@lewi.us): Need to configure the notebook and test to build - # using GCB. - dirname = os.path.dirname(__file__) - if not dirname: - logging.info("__file__ doesn't apper to be absolute path.") - dirname = os.getcwd() - notebook_path = os.path.join(dirname, "TF on GKE.ipynb") - logging.info("Reading notebook %s", notebook_path) - if not os.path.exists(notebook_path): - raise ValueError("%s does not exist" % notebook_path) - - with open(notebook_path) as hf: - node = nbformat.read(hf, nbformat.NO_CONVERT) - exporter = nbconvert.PythonExporter() - raw, _ = nbconvert.export(exporter, node) - - credentials = GoogleCredentials.get_application_default() - gke = discovery.build("container", "v1", credentials=credentials) - - lines = raw.splitlines() - - modified = replace_vars(lines, new_values) - - modified = strip_appendix(modified) - - modified = strip_unexecutable(modified) - - with tempfile.NamedTemporaryFile(suffix="notebook.py", prefix="tmpGke", - mode="w", delete=False) as hf: - code_path = hf.name - hf.write("\n".join(modified)) - logging.info("Wrote notebook to: %s", code_path) - - with open(code_path) as hf: - contents = hf.read() - logging.info("Notebook contents:\n%s", contents) - - try: - runpy.run_path(code_path) - finally: - logging.info("Deleting cluster; project=%s, zone=%s, name=%s", project, - zone, cluster) - util.delete_cluster(gke, cluster, project, zone) - - def testCpu(self): - """Test using CPU only.""" - now = datetime.datetime.now() - project = "kubeflow-ci" - cluster = ("gke-nb-test-" + now.strftime("v%Y%m%d") + "-" - + uuid.uuid4().hex[0:4]) - zone = "us-east1-d" - new_values = { - "project": project, - "cluster_name": cluster, - "zone": zone, - "registry": "gcr.io/kubeflow-ci", - "data_dir": "gs://kubeflow-ci_temp/cifar10/data", - "job_dirs": "gs://kubeflow-ci_temp/cifar10/jobs", - "num_steps": 10, - "use_gpu": False, - } - self.run_test(project, zone, cluster, new_values) - - def testGpu(self): - """Test using CPU only.""" - now = datetime.datetime.now() - project = "kubeflow-ci" - cluster = ("gke-nb-test-" + now.strftime("v%Y%m%d") + "-" - + uuid.uuid4().hex[0:4]) - zone = "us-east1-c" - new_values = { - "project": project, - "cluster_name": cluster, - "zone": zone, - "registry": "gcr.io/kubeflow-ci", - "data_dir": "gs://kubeflow-ci_temp/cifar10/data", - "job_dirs": "gs://kubeflow-ci_temp/cifar10/jobs", - "num_steps": 10, - "use_gpu": True, - "accelerator": "nvidia-tesla-k80", - "accelerator_count": 1, - } - self.run_test(project, zone, cluster, new_values) - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, - format=('%(levelname)s|%(asctime)s' - '|%(pathname)s|%(lineno)d| %(message)s'), - datefmt='%Y-%m-%dT%H:%M:%S', - ) - unittest.main() diff --git a/examples/tensorflow-models/Dockerfile.template b/examples/tensorflow-models/Dockerfile.template deleted file mode 100644 index 055509fbfb..0000000000 --- a/examples/tensorflow-models/Dockerfile.template +++ /dev/null @@ -1,19 +0,0 @@ -# Docker image for running examples in Tensorflow models. -# base_image depends on whether we are running on GPUs or non-GPUs -FROM {{base_image}} - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - build-essential \ - git - -# TODO(jlewi): Stop using my branch once we merge the PR to support writing -# to GCS -# Checkout tensorflow/models and pin to a particular version -# RUN git clone https://github.com/tensorflow/models.git /tensorflow_models && \ -# cd /tensorflow_models && \ -# git checkout d86aa76 - -RUN git clone https://github.com/jlewi/models.git /tensorflow_models && \ - cd /tensorflow_models && \ - git checkout generate_records diff --git a/examples/tensorflow-models/README.md b/examples/tensorflow-models/README.md deleted file mode 100644 index 37556d8bd7..0000000000 --- a/examples/tensorflow-models/README.md +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile for building an image containing the tensorflow/models repository -suitable for running the examples. diff --git a/examples/tf_job/.helmignore b/examples/tf_job/.helmignore deleted file mode 100644 index f0c1319444..0000000000 --- a/examples/tf_job/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/examples/tf_job/Chart.yaml b/examples/tf_job/Chart.yaml deleted file mode 100644 index 20caa83216..0000000000 --- a/examples/tf_job/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: A Helm chart for Kubernetes -name: tf-job -version: 0.1.0 diff --git a/examples/tf_job/templates/_helpers.tpl b/examples/tf_job/templates/_helpers.tpl deleted file mode 100644 index f0d83d2edb..0000000000 --- a/examples/tf_job/templates/_helpers.tpl +++ /dev/null @@ -1,16 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/examples/tf_job/templates/tf_job.yaml b/examples/tf_job/templates/tf_job.yaml deleted file mode 100644 index 40e159b08d..0000000000 --- a/examples/tf_job/templates/tf_job.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: "kubeflow.org/v1alpha1" -kind: "TFJob" -metadata: - name: {{ .Release.Name }} -spec: - replica_specs: - - replicas: 1 - tfPort: 2222 - tfReplicaType: MASTER - template: - spec: - containers: - - image: {{ .Values.image }} - name: tensorflow - restartPolicy: OnFailure - - replicas: 1 - tfPort: 2222 - tfReplicaType: WORKER - template: - spec: - containers: - - image: {{ .Values.image }} - name: tensorflow - restartPolicy: OnFailure diff --git a/examples/tf_job/values.yaml b/examples/tf_job/values.yaml deleted file mode 100644 index 088e76f4e6..0000000000 --- a/examples/tf_job/values.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# Default values for tf_job. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. -image: gcr.io/tf-on-k8s-dogfood/tf_sample:latest