Skip to content

Commit de2edc8

Browse files
wesmwesm
committed
ARROW-1156: [C++/Python] Expand casting API, add UnaryKernel callable. Use Cast in appropriate places when converting from pandas
cc @cloud-fan With this patch we now try to cast to indicated type on ingest of objects from pandas: ``` In [3]: arr = np.array([None] * 5) In [4]: pa.Array.from_pandas(arr) Out[4]: <pyarrow.lib.NullArray object at 0x7f6cf1485d18> [ NA, NA, NA, NA, NA ] In [5]: pa.Array.from_pandas(arr, type=pa.int32()) Out[5]: <pyarrow.lib.Int32Array object at 0x7f6cf1485d68> [ NA, NA, NA, NA, NA ] ``` I also added zero-copy casts from integers of the right size to each of the date and time types. Includes refactoring for ARROW-1481. Author: Wes McKinney <wes.mckinney@twosigma.com> Closes apache#1063 from wesm/ARROW-1156 and squashes the following commits: 166d1a5 [Wes McKinney] iwyu 34f5c9d [Wes McKinney] Harden default cast options, fix unsafe Python case 1d07b75 [Wes McKinney] Add some basic casting unit tests in Python c1b4570 [Wes McKinney] Expose arrow::compute::Cast in Python as Array.cast. Still need to write tests a9a04c9 [Wes McKinney] UnaryKernel::Call returns Status for now for simplicity. Support pre-allocated memory 8903709 [Wes McKinney] Implement casts from null to numbers. Try to cast for types where we do not have an inference rule when converting from arrays of Python objects a22dd20 [Wes McKinney] Add test to assert zero copy for compatible integer to date/time a14b83f [Wes McKinney] Create callable CastKernel object. Add zero-copy cast rules for date/time types
1 parent fe45c2b commit de2edc8

16 files changed

Lines changed: 687 additions & 217 deletions

File tree

‎cpp/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,9 @@ interfaces in this library as needed.
126126
The CUDA toolchain used to build the library can be customized by using the
127127
`$CUDA_HOME` environment variable.
128128

129+
This library is still in Alpha stages, and subject to API changes without
130+
deprecation warnings.
131+
129132
### API documentation
130133

131134
To generate the (html) API documentation, run the following command in the apidoc

‎cpp/build-support/iwyu/mappings/arrow-misc.imp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@
2424
{ symbol: ["uint8_t", private, "<cstdint>", public ] },
2525
{ symbol: ["_Node_const_iterator", private, "<flatbuffers/flatbuffers.h>", public ] },
2626
{ symbol: ["unordered_map<>::mapped_type", private, "<flatbuffers/flatbuffers.h>", public ] },
27-
{ symbol: ["__alloc_traits<>::value_type", private, "<vector>", public ] },
2827
{ symbol: ["move", private, "<utility>", public ] },
2928
{ symbol: ["pair", private, "<utility>", public ] },
3029
{ symbol: ["errno", private, "<cerrno>", public ] },
31-
{ symbol: ["posix_memalign", private, "<cstdlib>", public ] }
30+
{ symbol: ["posix_memalign", private, "<cstdlib>", public ] },
31+
{ include: ["<ext/alloc_traits.h>", private, "<vector>", public ] },
32+
{ include: ["<string.h>", private, "<cstring>", public ] }
3233
]

‎cpp/src/arrow/array.h‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -87,48 +87,48 @@ namespace internal {
8787
struct ARROW_EXPORT ArrayData {
8888
ArrayData() {}
8989

90+
ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
91+
int64_t null_count = kUnknownNullCount, int64_t offset = 0)
92+
: type(type), length(length), null_count(null_count), offset(offset) {}
93+
9094
ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
9195
const std::vector<std::shared_ptr<Buffer>>& buffers,
9296
int64_t null_count = kUnknownNullCount, int64_t offset = 0)
93-
: type(type),
94-
length(length),
95-
buffers(buffers),
96-
null_count(null_count),
97-
offset(offset) {}
97+
: ArrayData(type, length, null_count, offset) {
98+
this->buffers = buffers;
99+
}
98100

99101
ArrayData(const std::shared_ptr<DataType>& type, int64_t length,
100102
std::vector<std::shared_ptr<Buffer>>&& buffers,
101103
int64_t null_count = kUnknownNullCount, int64_t offset = 0)
102-
: type(type),
103-
length(length),
104-
buffers(std::move(buffers)),
105-
null_count(null_count),
106-
offset(offset) {}
104+
: ArrayData(type, length, null_count, offset) {
105+
this->buffers = std::move(buffers);
106+
}
107107

108108
// Move constructor
109109
ArrayData(ArrayData&& other) noexcept
110110
: type(std::move(other.type)),
111111
length(other.length),
112-
buffers(std::move(other.buffers)),
113112
null_count(other.null_count),
114113
offset(other.offset),
114+
buffers(std::move(other.buffers)),
115115
child_data(std::move(other.child_data)) {}
116116

117117
ArrayData(const ArrayData& other) noexcept
118118
: type(other.type),
119119
length(other.length),
120-
buffers(other.buffers),
121120
null_count(other.null_count),
122121
offset(other.offset),
122+
buffers(other.buffers),
123123
child_data(other.child_data) {}
124124

125125
// Move assignment
126126
ArrayData& operator=(ArrayData&& other) {
127127
type = std::move(other.type);
128128
length = other.length;
129-
buffers = std::move(other.buffers);
130129
null_count = other.null_count;
131130
offset = other.offset;
131+
buffers = std::move(other.buffers);
132132
child_data = std::move(other.child_data);
133133
return *this;
134134
}
@@ -139,9 +139,9 @@ struct ARROW_EXPORT ArrayData {
139139

140140
std::shared_ptr<DataType> type;
141141
int64_t length;
142-
std::vector<std::shared_ptr<Buffer>> buffers;
143142
int64_t null_count;
144143
int64_t offset;
144+
std::vector<std::shared_ptr<Buffer>> buffers;
145145
std::vector<std::shared_ptr<ArrayData>> child_data;
146146
};
147147

‎cpp/src/arrow/compute/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
# Headers: top level
1919
install(FILES
20+
api.h
2021
cast.h
2122
context.h
23+
kernel.h
2224
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute")
2325

2426
#######################################

‎cpp/src/arrow/compute/api.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#ifndef ARROW_COMPUTE_API_H
19+
#define ARROW_COMPUTE_API_H
20+
21+
#include "arrow/compute/cast.h"
22+
#include "arrow/compute/context.h"
23+
#include "arrow/compute/kernel.h"
24+
25+
#endif // ARROW_COMPUTE_API_H

0 commit comments

Comments
 (0)