Skip to content

Commit 3f2fa03

Browse files
wesmwesm
committed
ARROW-1430: [Python] Python CI build outside of a bash function scope, enable flake8 to fail build
This way `set -e` will have effect on any failures during the Python build. This should result in a failing build from flake8 warnings, which I will then fix Author: Wes McKinney <wes.mckinney@twosigma.com> Closes apache#1059 from wesm/ARROW-1430 and squashes the following commits: 33e9d6e [Wes McKinney] Also suppress E999 in Cython linting ffbee54 [Wes McKinney] Do not lint Cython in Python 2.7 3cf5d25 [Wes McKinney] fix another warning 4149705 [Wes McKinney] Fix flake8 warnings 06b44b6 [Wes McKinney] Move flake8 checks to correct place b26ddff [Wes McKinney] Remove unused lines bbc3aeb [Wes McKinney] Split parquet-cpp install into separate script, invoke travis_script_python twice
1 parent b0b125f commit 3f2fa03

8 files changed

Lines changed: 114 additions & 99 deletions

File tree

‎.travis.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,9 @@ matrix:
6161
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
6262
script:
6363
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
64-
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
64+
- $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
65+
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
66+
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
6567
- compiler: clang
6668
osx_image: xcode6.4
6769
os: osx
@@ -73,7 +75,9 @@ matrix:
7375
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
7476
script:
7577
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
76-
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
78+
- $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
79+
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
80+
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
7781
- language: cpp
7882
before_script:
7983
- docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest

‎ci/travis_build_parquet_cpp.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
set -e
20+
21+
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
22+
23+
export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
24+
25+
# $CPP_TOOLCHAIN set up in before_script_cpp
26+
export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
27+
28+
PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
29+
mkdir -p $PARQUET_DIR
30+
31+
git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
32+
33+
pushd $PARQUET_DIR
34+
mkdir build-dir
35+
cd build-dir
36+
37+
cmake \
38+
-GNinja \
39+
-DCMAKE_BUILD_TYPE=debug \
40+
-DCMAKE_INSTALL_PREFIX=$ARROW_PYTHON_PARQUET_HOME \
41+
-DPARQUET_BOOST_USE_SHARED=off \
42+
-DPARQUET_BUILD_BENCHMARKS=off \
43+
-DPARQUET_BUILD_EXECUTABLES=off \
44+
-DPARQUET_BUILD_TESTS=off \
45+
..
46+
47+
ninja
48+
ninja install
49+
50+
popd

‎ci/travis_env_common.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
3434
export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
3535
export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
3636

37+
export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
38+
3739
export CMAKE_EXPORT_COMPILE_COMMANDS=1
3840

3941
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then

‎ci/travis_script_python.sh‎

Lines changed: 48 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -21,114 +21,72 @@ set -e
2121
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
2222

2323
export ARROW_HOME=$ARROW_CPP_INSTALL
24-
export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
24+
export PARQUET_HOME=$ARROW_PYTHON_PARQUET_HOME
2525
export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
2626
export PYARROW_CXXFLAGS="-Werror"
2727

28-
build_parquet_cpp() {
29-
export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
28+
PYTHON_VERSION=$1
29+
CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
3030

31-
# $CPP_TOOLCHAIN set up in before_script_cpp
32-
export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
31+
conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
32+
source activate $CONDA_ENV_DIR
3333

34-
PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
35-
mkdir -p $PARQUET_DIR
34+
python --version
35+
which python
3636

37-
git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
37+
# faster builds, please
38+
conda install -y -q nomkl
3839

39-
pushd $PARQUET_DIR
40-
mkdir build-dir
41-
cd build-dir
40+
# Expensive dependencies install from Continuum package repo
41+
conda install -y -q pip numpy pandas cython flake8
4242

43-
cmake \
44-
-GNinja \
45-
-DCMAKE_BUILD_TYPE=debug \
46-
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
47-
-DPARQUET_BOOST_USE_SHARED=off \
48-
-DPARQUET_BUILD_BENCHMARKS=off \
49-
-DPARQUET_BUILD_EXECUTABLES=off \
50-
-DPARQUET_BUILD_TESTS=off \
51-
..
43+
# Build C++ libraries
44+
pushd $ARROW_CPP_BUILD_DIR
5245

53-
ninja
54-
ninja install
46+
# Clear out prior build files
47+
rm -rf *
5548

56-
popd
57-
}
58-
59-
build_parquet_cpp
60-
61-
function rebuild_arrow_libraries() {
62-
pushd $ARROW_CPP_BUILD_DIR
63-
64-
# Clear out prior build files
65-
rm -rf *
66-
67-
cmake -GNinja \
68-
-DARROW_BUILD_TESTS=off \
69-
-DARROW_BUILD_UTILITIES=off \
70-
-DARROW_PLASMA=on \
71-
-DARROW_PYTHON=on \
72-
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
73-
$ARROW_CPP_DIR
74-
75-
ninja
76-
ninja install
49+
cmake -GNinja \
50+
-DARROW_BUILD_TESTS=off \
51+
-DARROW_BUILD_UTILITIES=off \
52+
-DARROW_PLASMA=on \
53+
-DARROW_PYTHON=on \
54+
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
55+
$ARROW_CPP_DIR
7756

78-
popd
79-
}
57+
ninja
58+
ninja install
8059

81-
python_version_tests() {
82-
PYTHON_VERSION=$1
83-
CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
60+
popd
8461

85-
conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
86-
source activate $CONDA_ENV_DIR
62+
# Other stuff pip install
63+
pushd $ARROW_PYTHON_DIR
8764

88-
python --version
89-
which python
65+
# Fail fast on style checks
66+
flake8 --count pyarrow
9067

91-
# faster builds, please
92-
conda install -y -q nomkl
68+
# Check Cython files with some checks turned off
69+
flake8 --count --config=.flake8.cython pyarrow
9370

94-
# Expensive dependencies install from Continuum package repo
95-
conda install -y -q pip numpy pandas cython flake8
71+
pip install -r requirements.txt
72+
python setup.py build_ext --with-parquet --with-plasma \
73+
install --single-version-externally-managed --record=record.text
74+
popd
9675

97-
# Fail fast on style checks
98-
flake8 --count pyarrow
76+
python -c "import pyarrow.parquet"
77+
python -c "import pyarrow.plasma"
9978

100-
# Check Cython files with some checks turned off
101-
flake8 --count --config=.flake8.cython pyarrow
79+
if [ $TRAVIS_OS_NAME == "linux" ]; then
80+
export PLASMA_VALGRIND=1
81+
fi
10282

103-
# Build C++ libraries
104-
rebuild_arrow_libraries
83+
PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
84+
python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
10585

106-
# Other stuff pip install
107-
pushd $ARROW_PYTHON_DIR
108-
pip install -r requirements.txt
109-
python setup.py build_ext --with-parquet --with-plasma \
110-
install --single-version-externally-managed --record=record.text
86+
if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
87+
# Build documentation once
88+
pushd $ARROW_PYTHON_DIR/doc
89+
conda install -y -q --file=requirements.txt
90+
sphinx-build -b html -d _build/doctrees -W source _build/html
11191
popd
112-
113-
python -c "import pyarrow.parquet"
114-
python -c "import pyarrow.plasma"
115-
116-
if [ $TRAVIS_OS_NAME == "linux" ]; then
117-
export PLASMA_VALGRIND=1
118-
fi
119-
120-
PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
121-
python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
122-
123-
if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
124-
# Build documentation once
125-
pushd $ARROW_PYTHON_DIR/doc
126-
conda install -y -q --file=requirements.txt
127-
sphinx-build -b html -d _build/doctrees -W source _build/html
128-
popd
129-
fi
130-
}
131-
132-
# run tests for python 2.7 and 3.6
133-
python_version_tests 2.7
134-
python_version_tests 3.6
92+
fi

‎python/.flake8.cython‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717

1818
[flake8]
1919
filename = *.pyx,*.pxd,*.pxi
20-
ignore = E211,E901,E225,E226,E227
20+
ignore = E211,E901,E999,E225,E226,E227

‎python/pyarrow/includes/libarrow.pxd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
243243
shared_ptr[CSchema] RemoveMetadata()
244244

245245
cdef cppclass PrettyPrintOptions:
246-
int indent;
246+
int indent
247247

248248
CStatus PrettyPrint(const CSchema& schema,
249249
const PrettyPrintOptions& options,

‎python/pyarrow/tests/test_serialization.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,9 @@ def assert_equal(obj1, obj2):
7777
for i in range(len(obj1)):
7878
assert_equal(obj1[i], obj2[i])
7979
else:
80-
assert obj1 == obj2, "Objects {} and {} are different.".format(obj1,
81-
obj2)
80+
assert obj1 == obj2, ("Objects {} and {} are different."
81+
.format(obj1, obj2))
82+
8283

8384
PRIMITIVE_OBJECTS = [
8485
0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999,

‎python/pyarrow/tests/test_table.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,9 @@ def test_recordbatch_from_to_pandas():
135135
data = pd.DataFrame({
136136
'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
137137
'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
138-
'c2': np.random.randn(5),
139-
'c3': ['foo', 'bar', None, 'baz', 'qux'],
140-
'c4': [False, True, False, True, False]
138+
'c3': np.random.randn(5),
139+
'c4': ['foo', 'bar', None, 'baz', 'qux'],
140+
'c5': [False, True, False, True, False]
141141
})
142142

143143
batch = pa.RecordBatch.from_pandas(data)

0 commit comments

Comments
 (0)