Add --verbose flag to show puller progress

Add minimal prints to indicate puller progress if --verbose
flag is passed to fast puller.  The intention is that the bazel
container_pull workspace rules could eventually output something,
instead of bazel getting seemingly stuck for 10 minutes when
pulling an image that is many gigabytes in size.

We are intentionally not using the logging facility and the existing
--stderrthreshold argument to display these prints, because that
will produce log-formatted output that looks too detailed when inlined
with other bazel output.  The intention is to show user-friendly
progress instead:

   Downloading from gcr.io/tensorflow/tensorflow:latest (1/12)
   Downloading from gcr.io/tensorflow/tensorflow:latest (2/12)
   Downloading from gcr.io/tensorflow/tensorflow:latest (3/12)
   Downloading from gcr.io/tensorflow/tensorflow:latest (4/12)
   Downloading from gcr.io/tensorflow/tensorflow:latest (5/12)
   ...

Author: scele

client/v2_2/docker_image_.py

@@ -260,6 +260,9 @@ def _tags(self):
   def tags(self):
     return self._tags().get('tags', [])
 
+  def name(self):
+    return self._name
+
   def manifests(self):
     payload = self._tags()
     if 'manifest' not in payload:

client/v2_2/save_.py

@@ -20,6 +20,7 @@
 import hashlib
 import json
 import os
+import sys
 import tarfile
 
 import concurrent.futures
@@ -121,7 +122,8 @@ def tarball(
 def fast(
     image,
     directory,
-    threads = 1
+    threads = 1,
+    verbose = False
 ):
   """Produce a FromDisk compatible file layout under the provided directory.
 
@@ -151,8 +153,11 @@ def fast(
   def write_file(
       name,
       accessor,
-      arg
+      arg,
+      message=None
   ):
+    if verbose and message is not None:
+      sys.stderr.write(message + "\n")
     with open(name, 'wb') as f:
       f.write(accessor(arg))
 
@@ -164,6 +169,7 @@ def write_file(
     future_to_params[f] = config_file
 
     idx = 0
+    num_layers = len(image.fs_layers())
     layers = []
     for blob in reversed(image.fs_layers()):
       # Create a local copy
@@ -174,7 +180,8 @@ def write_file(
       future_to_params[f] = digest_name
 
       layer_name = os.path.join(directory, '%03d.tar.gz' % idx)
-      f = executor.submit(write_file, layer_name, image.blob, blob)
+      message = 'Downloading from {} ({}/{})'.format(image.name(), idx+1, num_layers)
+      f = executor.submit(write_file, layer_name, image.blob, blob, message)
       future_to_params[f] = layer_name
 
       layers.append((digest_name, layer_name))

tools/fast_puller_.py

@@ -48,6 +48,9 @@
 parser.add_argument('--directory', action='store',
                     help='Where to save the image\'s files.')
 
+parser.add_argument('--verbose', action='store_true',
+                    help='Print something as the pull progresses.')
+
 _THREADS = 8
 
 _PROCESSOR_ARCHITECTURE = 'amd64'
@@ -107,13 +110,13 @@ def main():
     logging.info('Pulling v2.2 image from %r ...', name)
     with v2_2_image.FromRegistry(name, creds, transport, accept) as v2_2_img:
       if v2_2_img.exists():
-        save.fast(v2_2_img, args.directory, threads=_THREADS)
+        save.fast(v2_2_img, args.directory, threads=_THREADS, verbose=args.verbose)
         return
 
     logging.info('Pulling v2 image from %r ...', name)
     with v2_image.FromRegistry(name, creds, transport) as v2_img:
       with v2_compat.V22FromV2(v2_img) as v2_2_img:
-        save.fast(v2_2_img, args.directory, threads=_THREADS)
+        save.fast(v2_2_img, args.directory, threads=_THREADS, verbose=args.verbose)
         return
   # pylint: disable=broad-except
   except Exception as e: