Merge pull request #918 from confluentinc/v1.5.0rc

edenhill · web-flow · commit 5aa6439df081 · 2020-07-22T07:39:35.000+02:00
v1.5.0rc
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -1,6 +1,6 @@
 environment:
   global:
-    LIBRDKAFKA_NUGET_VERSION: 1.4.2
+    LIBRDKAFKA_NUGET_VERSION: 1.5.0
     CIBW_SKIP: cp33-* cp34-*
     CIBW_TEST_REQUIRES: pytest pytest-timeout requests trivup
     # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,6 @@
 env:
  global:
-  - LIBRDKAFKA_VERSION=v1.4.2
+  - LIBRDKAFKA_VERSION=v1.5.0
 jobs:
  include:
   # Source package verification with Python 2.7
@@ -45,7 +45,7 @@ jobs:
 # Install cibuildwheel if this is a tagged PR
 before_install:
   - if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
-  - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -r tools/requirements-manylinux.txt --force-reinstall; fi
+  - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -U -r tools/requirements-manylinux.txt --force-reinstall ; fi
 
 # Install interceptors
 # Install librdkafka if not CIBW_BEFORE_BUILD
@@ -69,8 +69,7 @@ after_script:
  - ldd staging/libs/* || otool -L staging/libs/* || true
  - if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "osx" ]]; then DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs py.test --timeout=60 --ignore=tmp-build --import-mode append; fi
  - if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs py.test --timeout=60 --ignore=tmp-build --import-mode append; fi
- - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == linux && -n $CIBW_BEFORE_BUILD ]];then tools/test-manylinux.sh; fi
- - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-osx.sh; fi
+ - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-wheel.sh wheelhouse; fi
  - if [[ $MK_DOCS == y ]]; then make docs; fi
 
 deploy:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,15 @@
+# Confluent's Python client for Apache Kafka
+
+## v1.5.0
+
+v1.5.0 is a maintenance release with the following fixes and enhancements:
+
+ - Bundles librdkafka v1.5.0 - see release notes for all enhancements and fixes.
+ - Documentation fixes
+ - [Dockerfile examples](examples/docker)
+ - [List offsets example](examples/list_offsets.py)
+
+confluent-kafka-python is based on librdkafka v1.5.0, see the
+[librdkafka release notes](https://github.com/edenhill/librdkafka/releases/tag/v1.5.0)
+for a complete list of changes, enhancements, fixes and upgrade considerations.
+
diff --git a/confluent_kafka/avro/requirements.txt b/confluent_kafka/avro/requirements.txt
@@ -1,4 +1,4 @@
 fastavro>=0.23.0
 requests
-avro==1.9.2;python_version<='3.0'
-avro-python3==1.9.2.1;python_version>='3.0'
+avro==1.10.0;python_version<='3.0'
+avro-python3==1.10.0;python_version>='3.0'
diff --git a/confluent_kafka/src/confluent_kafka.h b/confluent_kafka/src/confluent_kafka.h
@@ -41,8 +41,8 @@
  *  0xMMmmRRPP
  *  MM=major, mm=minor, RR=revision, PP=patchlevel (not used)
  */
-#define CFL_VERSION     0x01040200
-#define CFL_VERSION_STR "1.4.2"
+#define CFL_VERSION     0x01050000
+#define CFL_VERSION_STR "1.5.0"
 
 /**
  * Minimum required librdkafka version. This is checked both during
diff --git a/docs/conf.py b/docs/conf.py
@@ -57,9 +57,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.4.2'
+version = '1.5.0'
 # The full version, including alpha/beta/rc tags.
-release = '1.4.2'
+release = '1.5.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/examples/docker/Dockerfile.alpine b/examples/docker/Dockerfile.alpine
@@ -30,7 +30,7 @@ FROM alpine:3.12
 
 COPY . /usr/src/confluent-kafka-python
 
-ENV LIBRDKAFKA_VERSION v1.4.2
+ENV LIBRDKAFKA_VERSION v1.5.0
 ENV KAFKACAT_VERSION master
 
 
diff --git a/setup.py b/setup.py
@@ -26,8 +26,8 @@
 SCHEMA_REGISTRY_REQUIRES = ['requests']
 
 AVRO_REQUIRES = ['fastavro>=0.23.0',
-                 'avro==1.9.2;python_version<"3.0"',
-                 'avro-python3==1.9.2.1;python_version>"3.0"'
+                 'avro==1.10.0;python_version<"3.0"',
+                 'avro-python3==1.10.0;python_version>"3.0"'
                  ] + SCHEMA_REGISTRY_REQUIRES
 
 JSON_REQUIRES = ['jsonschema'] + SCHEMA_REGISTRY_REQUIRES
@@ -63,7 +63,7 @@ def get_install_requirements(path):
 setup(name='confluent-kafka',
       # Make sure to bump CFL_VERSION* in confluent_kafka/src/confluent_kafka.h
       # and version and release in docs/conf.py.
-      version='1.4.2',
+      version='1.5.0',
       description='Confluent\'s Python client for Apache Kafka',
       author='Confluent Inc',
       author_email='support@confluent.io',
diff --git a/tools/README.md b/tools/README.md
@@ -32,3 +32,13 @@ This example only builds for Python 2.7 x64:
     $ CIBW_SKIP="cp3* cp*i686*" tools/cibuildwheel-build.sh wheelhouse
 
 Packages will now be available in wheelhouse/
+
+
+
+## How to test wheels
+
+After wheels have been downloaded with `download-s3.py`, perform smoketests
+by running `tools/test-wheels.sh <download-dir>`, e.g.,
+`tools/test-wheels.sh tools/dl-v1.5.0rc1`.
+This script preferably be run on OSX (with Docker installed) so that
+both OSX and Linux wheels are tested.
diff --git a/tools/RELEASE.md b/tools/RELEASE.md
@@ -70,8 +70,8 @@ previous version, e.g.: `git grep 0.11`
 The version should be set to the final release version, even when doing
 RCs, so only needs to be set once for each release.
 
- * `confluent_kafka/src/confluent_kafka.c` - in the `version()` function,
-    change both the string and the hex-representation.
+ * `confluent_kafka/src/confluent_kafka.h` - update the `CFL_VERSION`
+    `CFL_VERSION_STR`, change both the string and the hex-representation.
  * `docs/conf.py` - change `release` and `version` variables.
  * `setup.py` - change `version` variable.
 
@@ -107,13 +107,13 @@ the version.
 Perform a dry-run push first to make sure the correct branch and only our tag
 is pushed.
 
-    $ git push --dry-run --tags origin master
+    $ git push --dry-run --tags origin v0.11.4rc  # tags and branch
 
 Remove `--dry-run` when you're happy with the results.
 
 An alternative is to push branch and tags separately:
 
-    $ git push --dry-run origin master
+    $ git push --dry-run origin v0.11.4rc  # the branch
     $ git push --dry-run --tags origin v0.11.4rc1
 
 
@@ -136,25 +136,10 @@ artifacts from S3 using:
 
 The artifacts will be downloaded to `dl-<tag>/`.
 
-
 ## 8. Verify packages
 
-Create a new virtualenv:
-
-    $ rm -rf mytestenv2
-    $ virtualenv mytestenv2
-    $ source mytestenv2/bin/activate
-
-Install the relevant package for your platform:
+    $ tools/test-wheels.sh tools/dl-v0.11.4rc1
 
-    $ pip install --no-cache-dir --find-links dl-v0.11.4rc1/ confluent-kafka
-
-Verify that the package works, should print the expected Python client
-and librdkafka versions:
-
-    $ python -c 'import confluent_kafka as ck ; print "py: {} c: {}" \
-                 .format(ck.version(), ck.libversion())'
-    py: ('0.11.4', 721920) c: ('0.11.4-RC1', 722121)
 
 ## 10. Open a release PR
 
diff --git a/tools/smoketest.sh b/tools/smoketest.sh
@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+
+#
+# Run smoke tests to verify a vanilla installation of confluent-kafka.
+#
+# If a wheeldir is supplied confluent-kafka is first installed.
+#
+# Must be run from the top-level project directory.
+#
+
+if [[ ! -f tools/$(basename $0) ]]; then
+    echo "$0: Must be run from the top-level project directory"
+    exit 1
+fi
+
+set -e
+
+wheeldir=$1
+testdir=
+all_fails=
+
+if [[ -n $wheeldir ]]; then
+    if [[ ! -d $wheeldir ]]; then
+        echo "$0: wheeldir $wheeldir does not exist"
+        exit 1
+    fi
+    python -m pip install virtualenv
+fi
+
+pyvers_tested=
+
+# Run tests with both python2 and python3 (whatever versions the OS provides)
+for py in 2.7 3.8 3.7 3.6 3.5 ; do
+    echo "$0: # Smoketest with Python$py"
+
+    if ! python$py -V ; then
+        echo "$0: python$py not available: skipping"
+        continue
+    fi
+
+    pyvers_tested="$pyvers_tested $py"
+
+    if [[ -n $wheeldir ]]; then
+        venvdir=$(mktemp -d /tmp/_venvXXXXXX)
+
+        function cleanup () {
+            set +e
+            deactivate
+            rm -rf "$venvdir"
+            [[ -d $testdir ]] && rm -rf "$testdir"
+            set -e
+        }
+
+        trap cleanup EXIT
+
+        virtualenv -p python$py $venvdir
+        source $venvdir/bin/activate
+        hash -r
+
+        pip install -U pip pkginfo
+        pip install -r tests/requirements.txt
+
+        # Get the packages version so we can pin the install
+        # command to this version (which hopefully loads it from the wheeldir
+        # rather than PyPi) while still allowing dependencies to be installed
+        # from PyPi.
+        # Assuming that the wheeldirectory only contains wheels for a single
+        # version we can pick any wheel file.
+        version=$(pkginfo -f version $(ls $wheeldir/confluent_kafka*.whl | head -1) | sed -e 's/^version: //')
+        if [[ -z $version ]]; then
+            echo "$0: Unable to parse version from wheel files in $wheeldir"
+            exit 1
+        fi
+
+        pip install --find-links "$wheeldir" confluent-kafka==$version
+        pip install --find-links "$wheeldir" confluent-kafka[avro]==$version
+        pip install --find-links "$wheeldir" confluent-kafka[protobuf]==$version
+        pip install --find-links "$wheeldir" confluent-kafka[json]==$version
+    fi
+
+
+    # Copy unit tests to temporary directory to avoid any conflicting __pycache__
+    # directories from the source tree.
+    testdir=$(mktemp -d /tmp/_testdirXXXXXX)
+    cp tests/*.py $testdir/
+
+    # Change to a neutral path where there is no confluent_kafka sub-directory
+    # that might interfere with module load.
+    pushd $testdir
+    echo "$0: Running unit tests"
+    pytest
+
+    fails=""
+
+    echo "$0: Verifying OpenSSL"
+    python -c "
+import confluent_kafka
+confluent_kafka.Producer({'ssl.cipher.suites':'DEFAULT'})
+" || fails="$fails OpenSSL"
+
+    for compr in gzip lz4 snappy zstd; do
+        echo "$0: Verifying $compr"
+        python -c "
+import confluent_kafka
+confluent_kafka.Producer({'compression.codec':'$compr'})
+" || fails="$fails $compr"
+    done
+
+    echo "$0: Verifying Interceptor installation"
+    echo "$0: Note: Requires protobuf-c to be installed on the system."
+    python -c '
+from confluent_kafka import Consumer
+
+c = Consumer({"group.id": "test-linux", "plugin.library.paths": "monitoring-interceptor"})
+' || echo "$0: Warning: interceptor test failed, which we ignore"
+
+
+    if [[ -n $fails ]]; then
+        echo "$0: SMOKE TEST FAILS FOR python$py: $fails"
+        all_fails="$all_fails \[py$py: $fails\]"
+    fi
+
+    popd # $testdir
+done
+
+if [[ -z $pyvers_tested ]]; then
+    echo "$0: NO PYTHON VERSIONS TESTED"
+    exit 1
+fi
+
+if [[ -z $all_fails ]]; then
+    echo "$0: ALL SMOKE TESTS PASSED (python versions:$pyvers_tested)"
+    exit 0
+fi
+
+echo "$0: SMOKE TEST FAILURES: $all_fails"
+exit 1
+
+
diff --git a/tools/test-manylinux.sh b/tools/test-manylinux.sh
diff --git a/tools/test-wheels.sh b/tools/test-wheels.sh
