Skip to content
This repository was archived by the owner on Jun 4, 2021. It is now read-only.

Add --print-progress flag to show puller progress #66

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add --print-progress flag to show puller progress #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions client/v2_2/docker_image_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,9 @@ def _tags(self):
def tags(self):
return self._tags().get('tags', [])

def name(self):
return self._name

def manifests(self):
payload = self._tags()
if 'manifest' not in payload:
Expand Down
19 changes: 13 additions & 6 deletions client/v2_2/save_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io
import json
import os
import sys
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use logging instead, you can check fast_flatten or fast_pusher for reference.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See my commit message for explanation why I don't want to use logging.
Also, I would like the print to go to stderr, not stdout, because other bazel build status messages also go to stderr:

Loading: 
Loading: 0 packages loaded
    currently loading: foo
Analyzing: target //foo:bar (45 packages loaded)
...

I think the pull status progresses should appear together with these messages in stderr like so:

Loading: 
Loading: 0 packages loaded
    currently loading: foo
Analyzing: target //foo:bar (45 packages loaded)
Downloading from gcr.io/tensorflow/tensorflow:latest (1/12)
Downloading from gcr.io/tensorflow/tensorflow:latest (2/12)
Downloading from gcr.io/tensorflow/tensorflow:latest (3/12)
...

If the output goes to stdout, then these status messages would clutter run_log.txt in the following invocation:

bazel run //foo:bar 2>build_log.txt >run_log.txt

import tarfile

import concurrent.futures
Expand Down Expand Up @@ -144,7 +145,8 @@ def tarball(name, image,
def fast(image,
directory,
threads = 1,
cache_directory = None):
cache_directory = None,
print_progress = False):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

update Args section below.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

"""Produce a FromDisk compatible file layout under the provided directory.

After calling this, the following filesystem will exist:
Expand All @@ -166,6 +168,7 @@ def fast(image,
directory: an existing empty directory under which to save the layout.
threads: the number of threads to use when performing the upload.
cache_directory: directory that stores file cache.
print_progress: whether to print pull status messages to stderr.

Returns:
A tuple whose first element is the path to the config file, and whose second
Expand All @@ -174,13 +177,15 @@ def fast(image,
"""

def write_file(name, accessor,
arg):
arg, message = None):
if print_progress and message is not None:
sys.stderr.write(message + "\n")
with io.open(name, u'wb') as f:
f.write(accessor(arg))

def write_file_and_store(name, accessor,
arg, cached_layer):
write_file(cached_layer, accessor, arg)
arg, cached_layer, message = None):
write_file(cached_layer, accessor, arg, message)
link(cached_layer, name)

def link(source, dest):
Expand Down Expand Up @@ -225,11 +230,13 @@ def valid(cached_layer, digest):
'unused')

idx = 0
num_layers = len(image.fs_layers())
layers = []
for blob in reversed(image.fs_layers()):
# Create a local copy
layer_name = os.path.join(directory, '%03d.tar.gz' % idx)
digest_name = os.path.join(directory, '%03d.sha256' % idx)
message = 'Downloading from {} (layer {}/{})'.format(image.name(), idx+1, num_layers)
# Strip the sha256: prefix
digest = blob[7:].encode('utf8')
f = executor.submit(
Expand All @@ -247,10 +254,10 @@ def valid(cached_layer, digest):
future_to_params[f] = layer_name
else:
f = executor.submit(write_file_and_store, layer_name, image.blob,
blob, cached_layer)
blob, cached_layer, message)
future_to_params[f] = layer_name
else:
f = executor.submit(write_file, layer_name, image.blob, blob)
f = executor.submit(write_file, layer_name, image.blob, blob, message)
future_to_params[f] = layer_name

layers.append((digest_name, layer_name))
Expand Down
9 changes: 7 additions & 2 deletions tools/fast_puller_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@
'located. Overiddes the value from DOCKER_CONFIG')
parser.add_argument(
'--cache', action='store', help='Image\'s files cache directory.')
parser.add_argument(
'--print-progress', action='store_true', help='Print pull progresses to stderr.')


_THREADS = 8

Expand Down Expand Up @@ -125,7 +128,8 @@ def main():
v2_2_img,
args.directory,
threads=_THREADS,
cache_directory=args.cache)
cache_directory=args.cache,
print_progress=args.print_progress)
return

logging.info('Pulling v2 image from %r ...', name)
Expand All @@ -135,7 +139,8 @@ def main():
v2_2_img,
args.directory,
threads=_THREADS,
cache_directory=args.cache)
cache_directory=args.cache,
print_progress=args.print_progress)
return
# pylint: disable=broad-except
except Exception as e:
Expand Down